feat: cronjob to monitor server status (#421)

Author: tanmoysrt

swiftwave_service/core/server.operations.go

@@ -3,6 +3,7 @@ package core
 import (
 	"errors"
 	"gorm.io/gorm"
+	"time"
 )
 
 // CreateServer creates a new server in the database
@@ -13,6 +14,7 @@ func CreateServer(db *gorm.DB, server *Server) error {
 	if server.User == "" {
 		return errors.New("user is required")
 	}
+	server.LastPing = time.Now()
 	return db.Create(server).Error
 }
 
@@ -109,3 +111,13 @@ func FetchAllProxyServersIrrespectiveOfStatus(db *gorm.DB) ([]Server, error) {
 	err := db.Where("proxy_enabled = ?", true).Find(&servers).Error
 	return servers, err
 }
+
+// MarkServerAsOnline marks a server as online in the database
+func MarkServerAsOnline(db *gorm.DB, server *Server) error {
+	return db.Model(server).Updates(map[string]interface{}{"status": ServerOnline, "last_ping": time.Now()}).Error
+}
+
+// MarkServerAsOffline marks a server as offline in the database
+func MarkServerAsOffline(db *gorm.DB, server *Server) error {
+	return db.Model(server).Update("status", ServerOffline).Error
+}

swiftwave_service/cronjob/init.go

@@ -28,6 +28,8 @@ func (m Manager) Start(nowait bool) {
 	go m.SyncProxy()
 	m.wg.Add(1)
 	go m.SyncBackupProxyServer()
+	m.wg.Add(1)
+	go m.MonitorServerStatus()
 	if !nowait {
 		m.wg.Wait()
 	}

swiftwave_service/cronjob/server_status_monitor.go

@@ -0,0 +1,65 @@
+package cronjob
+
+import (
+	"bytes"
+	"github.com/swiftwave-org/swiftwave/ssh_toolkit"
+	"github.com/swiftwave-org/swiftwave/swiftwave_service/core"
+	"github.com/swiftwave-org/swiftwave/swiftwave_service/logger"
+	"strings"
+	"time"
+)
+
+func (m Manager) MonitorServerStatus() {
+	for {
+		m.monitorServerStatus()
+		time.Sleep(5 * time.Minute)
+	}
+}
+
+func (m Manager) monitorServerStatus() {
+	logger.CronJobLogger.Println("Triggering Server Status Monitor Job")
+	// Fetch all servers
+	servers, err := core.FetchAllServers(&m.ServiceManager.DbClient)
+	if err != nil {
+		logger.CronJobLoggerError.Println("Failed to fetch server list")
+		logger.CronJobLoggerError.Println(err)
+		return
+	}
+	if len(servers) == 0 {
+		logger.CronJobLogger.Println("Skipping ! No server found")
+		return
+	} else {
+		for _, server := range servers {
+			go func(server core.Server) {
+				if m.isServerOnline(server) {
+					err = core.MarkServerAsOnline(&m.ServiceManager.DbClient, &server)
+					if err != nil {
+						logger.CronJobLoggerError.Println("DB Error : Failed to mark server as online > ", server.HostName)
+					} else {
+						logger.CronJobLogger.Println("Server marked as online > ", server.HostName)
+					}
+				} else {
+					err = core.MarkServerAsOffline(&m.ServiceManager.DbClient, &server)
+					if err != nil {
+						logger.CronJobLoggerError.Println("DB Error : Failed to mark server as offline > ", server.HostName)
+					} else {
+						logger.CronJobLogger.Println("Server marked as offline > ", server.HostName)
+					}
+				}
+			}(server)
+		}
+	}
+}
+
+func (m Manager) isServerOnline(server core.Server) bool {
+	cmd := "echo ok"
+	stdoutBuf := new(bytes.Buffer)
+	err := ssh_toolkit.ExecCommandOverSSH(cmd, stdoutBuf, nil, 5, server.IP, 22, server.User, m.Config.SystemConfig.SshPrivateKey, 30)
+	if err != nil {
+		return false
+	}
+	if strings.Compare(stdoutBuf.String(), "ok") == 0 {
+		return false
+	}
+	return true
+}