feat(watchdog): implement keepalive mechanism and enhance connection handling

README.md

@@ -69,13 +69,12 @@ For detailed API documentation, please see [docs/api.md](docs/api.md)
 - [x] Add frpc instance running status management API
 - [x] Add frpc instance log display API
 - [x] Fix random database lock when processing logs
-- [ ] Add frpc createdBy storage and display
+- [ ] Add frpc createdBy display
 - [x] Add frpc proxy management API
 - [x] Fix backend can still start frpc instance when it is already running
 - [ ] Develop an agent software to handle windows service management
 - [ ] Refactor all log output level to be more clear
-- [ ] Add global websocket endpoint for posting notifications
-- [ ] Add frpc instance watchdog
+- [x] Add frpc instance watchdog
 
 ## License
 

main.go

@@ -12,8 +12,8 @@ import (
 	"super-frpc/frpLogger"
 	"super-frpc/global"
 	"super-frpc/postLog"
-	"super-frpc/sys"
 	"super-frpc/session"
+	"super-frpc/sys"
 	"super-frpc/utils"
 	"super-frpc/watchdog"
 	"syscall"
@@ -81,6 +81,13 @@ func main() {
 		} else {
 			postLog.Info(fmt.Sprintf("Connected to Watchdog at %s:%d", "127.0.0.1", global.CurrentConfig.Watchdog.Port))
 			global.Is.WatchdogConnected = true
+
+			go func() {
+				if err := watchdog.StartKeepAlive(); err != nil {
+					postLog.Error(fmt.Sprintf("Watchdog keepalive stopped: %v", err))
+					global.Is.WatchdogConnected = false
+				}
+			}()
 		}
 	}
 

watchdog/Init&tcpClient.go

@@ -9,9 +9,9 @@ import (
 	"path/filepath"
 	"strings"
 	"super-frpc/global"
+	"super-frpc/postLog"
 	"super-frpc/sys"
 	"super-frpc/utils"
-	"super-frpc/postLog"
 	"sync"
 	"time"
 )
@@ -26,6 +26,17 @@ var (
 )
 
 func Init() error {
+	tcpConnMutex.Lock()
+	if recvChan != nil || stopRecvChan != nil {
+		tcpConnMutex.Unlock()
+		return nil
+	}
+	if tcpConn != nil {
+		tcpConnMutex.Unlock()
+		return fmt.Errorf("TCP client already initialized")
+	}
+	tcpConnMutex.Unlock()
+
 	if global.CurrentConfig.Watchdog.Enabled {
 		if err := ensureWatchdogProcess(); err != nil {
 			postLog.Error(fmt.Sprintf("Failed to boot watchdog program: %v", err))
@@ -36,14 +47,6 @@ func Init() error {
 	tcpConnMutex.Lock()
 	defer tcpConnMutex.Unlock()
 
-	if recvChan != nil || stopRecvChan != nil {
-		return nil
-	}
-
-	if tcpConn != nil {
-		return fmt.Errorf("TCP client already initialized")
-	}
-
 	recvChan = make(chan string, 100)
 	stopRecvChan = make(chan struct{})
 	isConnected = false
@@ -114,14 +117,15 @@ func isWatchdogProcessRunning(watchdogName string) bool {
 		outputStr := strings.ToLower(string(output))
 		return strings.Contains(outputStr, strings.ToLower(watchdogName)) && !strings.Contains(outputStr, "no tasks are running")
 	case "systemd", "init.d":
-		cmd := exec.Command("ps", "-A", "-o", "comm=")
+		cmd := exec.Command("ps", "-A", "-o", "args=")
 		output, err := cmd.CombinedOutput()
 		if err != nil {
 			return false
 		}
 
 		for _, line := range strings.Split(string(output), "\n") {
-			if strings.TrimSpace(line) == watchdogName {
+			fields := strings.Fields(strings.TrimSpace(line))
+			if len(fields) > 0 && filepath.Base(fields[0]) == watchdogName {
 				return true
 			}
 		}

watchdog/connect.go

@@ -1,6 +1,8 @@
 package watchdog
 
 import (
+	"fmt"
+	"super-frpc/postLog"
 	"time"
 )
 
@@ -13,17 +15,32 @@ func Connect(ipaddr string, port int) bool {
 		return false
 	}
 
-	if err := tcpConnect(ipaddr, port); err != nil {
+	deadline := time.Now().Add(5 * time.Second)
+	var lastErr error
+	for {
+		if err := tcpConnect(ipaddr, port); err == nil {
+			break
+		} else {
+			lastErr = err
+		}
+
+		if time.Now().After(deadline) {
+			postLog.Error(fmt.Sprintf("[watchdog] failed to connect before timeout: %v", lastErr))
 			return false
 		}
 
+		time.Sleep(200 * time.Millisecond)
+	}
+
 	response, err := sendMsg("watchdogAgentConnectionTest", 3)
 	if err != nil {
+		postLog.Error(fmt.Sprintf("[watchdog] connection test failed: %v", err))
 		Destroy()
 		return false
 	}
 
 	if response != "success" {
+		postLog.Error(fmt.Sprintf("[watchdog] connection test returned unexpected response: %s", response))
 		Destroy()
 		return false
 	}

watchdog/keepalive.go

@@ -1,2 +1,62 @@
 package watchdog
 
+import (
+	"fmt"
+	"time"
+
+	"super-frpc/global"
+	"super-frpc/postLog"
+)
+
+func StartKeepAlive() error {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		<-ticker.C
+
+		if err := isProcessAlive(); err != nil {
+			global.Is.WatchdogConnected = false
+			postLog.Warning(fmt.Sprintf("[watchdog] keepalive check failed: %v", err))
+
+			_ = Disconnect()
+
+			var lastErr error
+			for i := 0; i < 5; i++ {
+				if err := ensureWatchdogProcess(); err != nil {
+					lastErr = err
+					time.Sleep(500 * time.Millisecond)
+					continue
+				}
+
+				if Connect("127.0.0.1", global.CurrentConfig.Watchdog.Port) {
+					global.Is.WatchdogConnected = true
+					postLog.Info("[watchdog] successfully reconnected to watchdog")
+					lastErr = nil
+					break
+				}
+
+				lastErr = fmt.Errorf("failed to connect to watchdog")
+				time.Sleep(500 * time.Millisecond)
+			}
+
+			if lastErr != nil {
+				postLog.Error(fmt.Sprintf("[watchdog] failed to recover watchdog connection: %v", lastErr))
+				return fmt.Errorf("watchdog recovery failed after 5 attempts: %w", lastErr)
+			}
+		}
+	}
+}
+
+func isProcessAlive() error {
+	if watchdogName, err := getWatchdogBinaryName(); err == nil && isWatchdogProcessRunning(watchdogName) {
+		resp, err := sendMsg("watchdogAgentConnectionTest", 3)
+		if err != nil || resp != "success" {
+			return fmt.Errorf("watchdog not responding to connection test")
+		}
+	} else {
+		return fmt.Errorf("watchdog process not running")
+	}
+
+	return nil
+}