wip

Author: mark-dell-usa

controllers/tests/controller/selfnoderemediation_controller_test.go

@@ -579,6 +579,9 @@ var _ = Describe("SNR Controller", func() {
 			})
 
 			Context("no peer found", func() {
+				BeforeEach(func() {
+					clearSimulatedPeerResponses()
+				})
 				It("Verify that watchdog is not triggered", func() {
 					verifyWatchdogNotTriggered()
 				})
@@ -1298,6 +1301,7 @@ func configureSimulatedPeerResponses(simulateResponses bool) {
 		DeferCleanup(func() {
 			apiCheck.ShouldSimulatePeerResponses = orgValue
 			apiCheck.RestoreSimulatedPeerResponses(orgResponses)
+			apiCheck.RememberSimulatedPeerResponses()
 			if !simulateResponses {
 				apiCheck.SetPeersOverride(nil)
 			}
@@ -1400,13 +1404,28 @@ func addNodes(nodes []newNodeConfig) {
 
 		}
 
+		apiCheck.RememberSimulatedPeerResponses()
+
 	})
 }
 
 func resetWatchdogTimer() {
 	By("Resetting watchdog timer", func() {
 		dummyDog.Reset()
 		apiCheck.ResetPeerTimers()
+		if apiCheck.ShouldSimulatePeerResponses {
+			apiCheck.RestoreBaselineSimulatedResponses()
+			apiCheck.RememberSimulatedPeerResponses()
+		} else {
+			apiCheck.ClearBaselineSimulatedResponses()
+		}
+	})
+}
+
+func clearSimulatedPeerResponses() {
+	By("Clearing simulated peer responses", func() {
+		apiCheck.ClearSimulatedPeerResponses()
+		apiCheck.ClearBaselineSimulatedResponses()
 	})
 }
 

controllers/tests/shared/shared.go

@@ -67,6 +67,7 @@ type ApiConnectivityCheckWrapper struct {
 	peersOverride            PeersOverrideFunc
 	workerLastResponse       time.Time
 	controlPlaneLastResponse time.Time
+	baselineResponses        []selfNodeRemediation.HealthCheckResponseCode
 }
 
 // PeersOverrideFunc allows tests to supply synthetic peer address lists without
@@ -186,6 +187,33 @@ func (ckw *ApiConnectivityCheckWrapper) RestoreSimulatedPeerResponses(codes []se
 	copy(ckw.simulatedPeerResponses, codes)
 }
 
+func (ckw *ApiConnectivityCheckWrapper) RememberSimulatedPeerResponses() {
+	ckw.responsesMu.Lock()
+	defer ckw.responsesMu.Unlock()
+	if len(ckw.simulatedPeerResponses) == 0 {
+		ckw.baselineResponses = nil
+		return
+	}
+	ckw.baselineResponses = make([]selfNodeRemediation.HealthCheckResponseCode, len(ckw.simulatedPeerResponses))
+	copy(ckw.baselineResponses, ckw.simulatedPeerResponses)
+}
+
+func (ckw *ApiConnectivityCheckWrapper) RestoreBaselineSimulatedResponses() {
+	ckw.responsesMu.Lock()
+	defer ckw.responsesMu.Unlock()
+	if len(ckw.baselineResponses) == 0 {
+		return
+	}
+	ckw.simulatedPeerResponses = make([]selfNodeRemediation.HealthCheckResponseCode, len(ckw.baselineResponses))
+	copy(ckw.simulatedPeerResponses, ckw.baselineResponses)
+}
+
+func (ckw *ApiConnectivityCheckWrapper) ClearBaselineSimulatedResponses() {
+	ckw.responsesMu.Lock()
+	defer ckw.responsesMu.Unlock()
+	ckw.baselineResponses = nil
+}
+
 // SetPeersOverride registers a custom provider for peer address lists.
 func (ckw *ApiConnectivityCheckWrapper) SetPeersOverride(fn PeersOverrideFunc) {
 	ckw.responsesMu.Lock()

pkg/apicheck/check.go

@@ -280,6 +280,9 @@ func (c *ApiConnectivityCheck) evaluateWorker(now time.Time) controlplane.Evalua
 	}
 
 	if !summary.responded || !summary.hadMinimum {
+		if !summary.responded && c.config.MinPeersForRemediation == 0 {
+			return controlplane.EvaluationIsolation
+		}
 		if c.peerTimeoutExceeded(peers.Worker, now) {
 			return controlplane.EvaluationIsolation
 		}
@@ -410,6 +413,8 @@ func (c *ApiConnectivityCheck) gatherPeerResponses(role peers.Role, now time.Tim
 		c.recordPeerSilence(role, now)
 	}
 
+	c.config.Log.Info("peer summary", "role", role, "total", summary.totalPeers(), "healthy", summary.healthy, "unhealthy", summary.unhealthy, "apiErrors", summary.apiErrors, "failures", summary.failures, "responded", summary.responded, "hadMinimum", summary.hadMinimum)
+
 	return summary
 }