Add support for checking Machine conditions in MachineHealthCheck

MachineHealthCheck currently only allows checking Node conditions to
validate if a machine is healthy. However, machine conditions capture
conditions that do not exist on nodes, for example, control plane node
conditions such as EtcdPodHealthy, SchedulerPodHealthy that can indicate
if a controlplane machine has been created correctly.

Adding support for Machine conditions enables us to perform remediation
during control plane upgrades.

This PR introduces a new field as part of the MachineHealthCheckChecks:
  - `UnhealthyMachineConditions`

This will mirror the behavior of `UnhealthyNodeConditions` but the
MachineHealthCheck controller will instead check the machine conditions.

This reimplements and extends the work originally proposed by @justinmir in PR #12275.

Co-authored-by: Justin Miron <[email protected]>
Signed-off-by: Furkat Gofurov <[email protected]>

Author: furkatgofurov7

api/core/v1beta1/conversion_test.go

@@ -99,6 +99,7 @@ func ClusterFuzzFuncs(_ runtimeserializer.CodecFactory) []interface{} {
 		hubClusterVariable,
 		hubFailureDomain,
 		hubUnhealthyNodeCondition,
+		hubUnhealthyMachineCondition,
 		spokeCluster,
 		spokeClusterTopology,
 		spokeObjectReference,
@@ -126,6 +127,17 @@ func hubClusterSpec(in *clusterv1.ClusterSpec, c randfill.Continue) {
 		in.ControlPlaneRef.APIGroup = gvk.Group
 		in.ControlPlaneRef.Kind = gvk.Kind
 	}
+
+	// remove MachineHealthCheck.UnhealthyMachineConditions as it does not exist in v1beta1.
+	if in.Topology.IsDefined() && in.Topology.ControlPlane.HealthCheck.IsDefined() {
+		in.Topology.ControlPlane.HealthCheck.Checks.UnhealthyMachineConditions = nil
+	}
+
+	if in.Topology.IsDefined() && len(in.Topology.Workers.MachineDeployments) > 0 {
+		for i := range in.Topology.Workers.MachineDeployments {
+			in.Topology.Workers.MachineDeployments[i].HealthCheck.Checks.UnhealthyMachineConditions = nil
+		}
+	}
 }
 
 func hubClusterStatus(in *clusterv1.ClusterStatus, c randfill.Continue) {
@@ -177,6 +189,14 @@ func hubUnhealthyNodeCondition(in *clusterv1.UnhealthyNodeCondition, c randfill.
 	}
 }
 
+func hubUnhealthyMachineCondition(in *clusterv1.UnhealthyMachineCondition, c randfill.Continue) {
+	c.FillNoCustom(in)
+
+	if in.TimeoutSeconds == nil {
+		in.TimeoutSeconds = ptr.To(int32(0)) // TimeoutSeconds is a required field and nil does not round trip
+	}
+}
+
 func spokeCluster(in *Cluster, c randfill.Continue) {
 	c.FillNoCustom(in)
 
@@ -267,12 +287,14 @@ func spokeClusterVariable(in *ClusterVariable, c randfill.Continue) {
 
 func ClusterClassFuncs(_ runtimeserializer.CodecFactory) []interface{} {
 	return []interface{}{
+		hubClusterClassSpec,
 		hubClusterClassVariable,
 		hubClusterClassStatusVariableDefinition,
 		hubClusterClassStatus,
 		hubJSONPatch,
 		hubJSONSchemaProps,
 		hubUnhealthyNodeCondition,
+		hubUnhealthyMachineCondition,
 		spokeClusterClass,
 		spokeObjectReference,
 		spokeClusterClassStatus,
@@ -287,6 +309,21 @@ func ClusterClassFuncs(_ runtimeserializer.CodecFactory) []interface{} {
 	}
 }
 
+func hubClusterClassSpec(in *clusterv1.ClusterClassSpec, c randfill.Continue) {
+	c.FillNoCustom(in)
+
+	// remove MachineHealthCheck.UnhealthyMachineConditions as it does not exist in v1beta1.
+	if in.ControlPlane.HealthCheck.IsDefined() && in.ControlPlane.HealthCheck.Checks.UnhealthyMachineConditions != nil {
+		in.ControlPlane.HealthCheck.Checks.UnhealthyMachineConditions = nil
+	}
+
+	if len(in.Workers.MachineDeployments) > 0 {
+		for i := range in.Workers.MachineDeployments {
+			in.Workers.MachineDeployments[i].HealthCheck.Checks.UnhealthyMachineConditions = nil
+		}
+	}
+}
+
 func hubClusterClassVariable(in *clusterv1.ClusterClassVariable, c randfill.Continue) {
 	c.FillNoCustom(in)
 
@@ -728,7 +765,9 @@ func spokeMachineDeploymentStatus(in *MachineDeploymentStatus, c randfill.Contin
 func MachineHealthCheckFuzzFuncs(_ runtimeserializer.CodecFactory) []interface{} {
 	return []interface{}{
 		hubUnhealthyNodeCondition,
+		hubUnhealthyMachineCondition,
 		hubMachineHealthCheckStatus,
+		hubMachineHealthCheckSpec,
 		spokeMachineHealthCheck,
 		spokeMachineHealthCheckSpec,
 		spokeObjectReference,
@@ -737,6 +776,14 @@ func MachineHealthCheckFuzzFuncs(_ runtimeserializer.CodecFactory) []interface{}
 	}
 }
 
+func hubMachineHealthCheckSpec(in *clusterv1.MachineHealthCheckSpec, c randfill.Continue) {
+	c.FillNoCustom(in)
+
+	if in.Checks.UnhealthyMachineConditions != nil {
+		in.Checks.UnhealthyMachineConditions = nil
+	}
+}
+
 func hubMachineHealthCheckStatus(in *clusterv1.MachineHealthCheckStatus, c randfill.Continue) {
 	c.FillNoCustom(in)
 	// Drop empty structs with only omit empty fields.

api/core/v1beta2/cluster_types.go

@@ -725,6 +725,16 @@ type ControlPlaneTopologyHealthCheckChecks struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
+
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a machine is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the machine is unhealthy.
+	//
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
 }
 
 // ControlPlaneTopologyHealthCheckRemediation configures if and how remediations are triggered if a control plane Machine is unhealthy.
@@ -975,6 +985,16 @@ type MachineDeploymentTopologyHealthCheckChecks struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
+
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a machine is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the machine is unhealthy.
+	//
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
 }
 
 // MachineDeploymentTopologyHealthCheckRemediation configures if and how remediations are triggered if a MachineDeployment Machine is unhealthy.

api/core/v1beta2/clusterclass_types.go

@@ -281,6 +281,16 @@ type ControlPlaneClassHealthCheckChecks struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
+
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a machine is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the machine is unhealthy.
+	//
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
 }
 
 // ControlPlaneClassHealthCheckRemediation configures if and how remediations are triggered if a control plane Machine is unhealthy.
@@ -542,6 +552,16 @@ type MachineDeploymentClassHealthCheckChecks struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
+
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a machine is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the machine is unhealthy.
+	//
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
 }
 
 // MachineDeploymentClassHealthCheckRemediation configures if and how remediations are triggered if a MachineDeployment Machine is unhealthy.

api/core/v1beta2/machine_types.go

@@ -276,6 +276,10 @@ const (
 	// defined by a MachineHealthCheck object.
 	MachineHealthCheckUnhealthyNodeReason = "UnhealthyNode"
 
+	// MachineHealthCheckUnhealthyMachineReason surfaces when the machine does not pass the health checks
+	// defined by a MachineHealthCheck object.
+	MachineHealthCheckUnhealthyMachineReason = "UnhealthyMachine"
+
 	// MachineHealthCheckNodeStartupTimeoutReason surfaces when the node hosted on the machine does not appear within
 	// the timeout defined by a MachineHealthCheck object.
 	MachineHealthCheckNodeStartupTimeoutReason = "NodeStartupTimeout"

api/core/v1beta2/machinehealthcheck_types.go

@@ -111,6 +111,16 @@ type MachineHealthCheckChecks struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=100
 	UnhealthyNodeConditions []UnhealthyNodeCondition `json:"unhealthyNodeConditions,omitempty"`
+
+	// unhealthyMachineConditions contains a list of the machine conditions that determine
+	// whether a machine is considered unhealthy.  The conditions are combined in a
+	// logical OR, i.e. if any of the conditions is met, the machine is unhealthy.
+	//
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	UnhealthyMachineConditions []UnhealthyMachineCondition `json:"unhealthyMachineConditions,omitempty"`
 }
 
 // MachineHealthCheckRemediation configures if and how remediations are triggered if a Machine is unhealthy.
@@ -234,6 +244,31 @@ type UnhealthyNodeCondition struct {
 	TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"`
 }
 
+// UnhealthyMachineCondition represents a Machine condition type and value with a timeout
+// specified as a duration.  When the named condition has been in the given
+// status for at least the timeout value, a machine is considered unhealthy.
+type UnhealthyMachineCondition struct {
+	// type of Machine condition
+	// +kubebuilder:validation:Type=string
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=316
+	// +required
+	Type string `json:"type"`
+
+	// status of the condition, one of True, False, Unknown.
+	// +required
+	// +kubebuilder:validation:Enum=True;False;Unknown
+	Status metav1.ConditionStatus `json:"status"`
+
+	// timeoutSeconds is the duration that a machine must be in a given status for,
+	// after which the machine is considered unhealthy.
+	// For example, with a value of "1h", the machine must match the status
+	// for at least 1 hour before being considered unhealthy.
+	// +required
+	// +kubebuilder:validation:Minimum=0
+	TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"`
+}
+
 // MachineHealthCheckStatus defines the observed state of MachineHealthCheck.
 // +kubebuilder:validation:MinProperties=1
 type MachineHealthCheckStatus struct {

api/core/v1beta2/v1beta1_condition_consts.go

@@ -157,6 +157,9 @@ const (
 
 	// UnhealthyNodeConditionV1Beta1Reason is the reason used when a machine's node has one of the MachineHealthCheck's unhealthy conditions.
 	UnhealthyNodeConditionV1Beta1Reason = "UnhealthyNode"
+
+	// UnhealthyMachineConditionV1Beta1Reason is the reason used when a machine has one of the MachineHealthCheck's unhealthy conditions.
+	UnhealthyMachineConditionV1Beta1Reason = "UnhealthyMachine"
 )
 
 const (