NanamiAdmin
3ce076b8dc
Implement error handling mechanism with retry logic for failed services. Includes: - ServiceControl struct to track service state - HandleErrorProcess function to attempt service restarts - Integration with monitor to automatically recover services - Enhanced exception handling with status checks and monitoring restart
37 lines
1.1 KiB
Go
37 lines
1.1 KiB
Go
package errorHandle
|
|
|
|
import (
|
|
"fmt"
|
|
"os/exec"
|
|
"time"
|
|
|
|
"Watchdog_Linux-systemd/postLog"
|
|
)
|
|
|
|
type ServiceStatusChecker func(serviceName string) bool
|
|
|
|
func HandleErrorProcess(serviceName string, isServiceRunning ServiceStatusChecker) error {
|
|
postLog.Debug(fmt.Sprintf("[HandleErrorProcess] Start handle error process for service: %s", serviceName))
|
|
serviceControl := &ServiceControl{
|
|
ServiceName: serviceName,
|
|
RetryCount: 0,
|
|
}
|
|
|
|
for i := 0; i < 5; i++ {
|
|
serviceControl.RetryCount++
|
|
postLog.Debug(fmt.Sprintf("[HandleErrorProcess] Try to restart service '%s', retry count: %d", serviceName, serviceControl.RetryCount))
|
|
cmd := exec.Command("systemctl", "restart", serviceName)
|
|
err := cmd.Run()
|
|
if err == nil {
|
|
if isServiceRunning != nil && isServiceRunning(serviceName) {
|
|
return nil
|
|
}
|
|
}
|
|
time.Sleep(time.Duration(i+1) * time.Second)
|
|
}
|
|
serviceControl.ErrorType = "restart"
|
|
serviceControl.ErrorMsg = fmt.Sprintf("Failed to recover service '%s', retry count: %d", serviceName, serviceControl.RetryCount)
|
|
serviceControl.ErrorTime = time.Now()
|
|
return fmt.Errorf(serviceControl.ErrorMsg)
|
|
}
|