minimizing reboot for scaling image mode enabled node

Author: dkhater-redhat

manifests/machineconfigserver/clusterrole.yaml

@@ -9,6 +9,9 @@ rules:
 - apiGroups: ["machineconfiguration.openshift.io"]
   resources: ["controllerconfigs"]
   verbs: ["get", "watch", "list"]
+- apiGroups: ["machineconfiguration.openshift.io"]
+  resources: ["machineosconfigs", "machineosbuilds"]
+  verbs: ["get", "list", "watch"]
 - apiGroups: ["security.openshift.io"]
   resourceNames: ["hostnetwork"]
   resources: ["securitycontextconstraints"]

pkg/daemon/daemon.go

@@ -1712,6 +1712,7 @@ func (dn *Daemon) getStateAndConfigs() (*stateAndConfigs, error) {
 		}
 		return nil, err
 	}
+	klog.Infof("[daemon.go] getStateAndConfigs: bootstrapping=%v, currentConfigName=%s, currentConfig.Spec.OSImageURL=%s (source=cluster-lister)", bootstrapping, currentConfigName, currentConfig.Spec.OSImageURL)
 	state, err := getNodeAnnotationExt(dn.node, constants.MachineConfigDaemonStateAnnotationKey, true)
 	if err != nil {
 		return nil, err
@@ -1721,11 +1722,13 @@ func (dn *Daemon) getStateAndConfigs() (*stateAndConfigs, error) {
 		klog.Infof("%s is not set. any errors? %s", constants.CurrentImageAnnotationKey, err)
 		return nil, err
 	}
+	klog.Infof("[daemon.go] getStateAndConfigs: currentImage=%s (from node annotations)", currentImage)
 	desiredImage, err := getNodeAnnotationExt(dn.node, constants.DesiredImageAnnotationKey, true)
 	if err != nil {
 		klog.Infof("%s is not set. any errors? %s", constants.DesiredImageAnnotationKey, err)
 		return nil, err
 	}
+	klog.Infof("[daemon.go] getStateAndConfigs: desiredImage=%s (from node annotations)", desiredImage)
 
 	// Temporary hack: the MCS used to not write the state=done annotation
 	// key.  If it's unset, let's write it now.
@@ -2174,8 +2177,19 @@ func (dn *Daemon) checkStateOnFirstRun() error {
 
 	// Bootstrapping state is when we have the node annotations file
 	if state.bootstrapping {
-		targetOSImageURL := state.currentConfig.Spec.OSImageURL
+		// targetOSImageURL := state.currentConfig.Spec.OSImageURL
+		// klog.Infof("[daemon.go] checkStateOnFirstRun: in bootstrap mode, bootedOSImageURL=%s, targetOSImageURL=%s (from state.currentConfig.Spec.OSImageURL)", dn.bootedOSImageURL, targetOSImageURL)
+
+		// During bootstrap, prefer the image from node annotations (if set) over the MC from cluster lister
+		targetOSImageURL := state.currentImage
+		if targetOSImageURL == "" {
+			// Fall back to MC's OSImageURL if no image annotation was provided
+			targetOSImageURL = state.currentConfig.Spec.OSImageURL
+		}
+		klog.Infof("[daemon.go] checkStateOnFirstRun: in bootstrap mode, bootedOSImageURL=%s, targetOSImageURL=%s (from state.currentImage annotation or state.currentConfig.Spec.OSImageURL)", dn.bootedOSImageURL,
+			targetOSImageURL)
 		osMatch := dn.checkOS(targetOSImageURL)
+		klog.Infof("[daemon.go] checkStateOnFirstRun: osMatch=%v (bootedOSImageURL == targetOSImageURL)", osMatch)
 		if !osMatch {
 			logSystem("Bootstrap pivot required to: %s", targetOSImageURL)
 

pkg/server/cluster_server.go

@@ -9,12 +9,15 @@ import (
 	"path/filepath"
 	"time"
 
+	ign3types "github.com/coreos/ignition/v2/config/v3_5/types"
 	yaml "github.com/ghodss/yaml"
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
 	mcfginformers "github.com/openshift/client-go/machineconfiguration/informers/externalversions"
 
 	"github.com/openshift/machine-config-operator/internal/clients"
 	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/informers"
 	corelisterv1 "k8s.io/client-go/listers/core/v1"
@@ -42,6 +45,8 @@ type clusterServer struct {
 	machineConfigLister     v1.MachineConfigLister
 	controllerConfigLister  v1.ControllerConfigLister
 	configMapLister         corelisterv1.ConfigMapLister
+	machineOSConfigLister   v1.MachineOSConfigLister
+	machineOSBuildLister    v1.MachineOSBuildLister
 
 	kubeconfigFunc kubeconfigFunc
 	apiserverURL   string
@@ -76,23 +81,27 @@ func NewClusterServer(kubeConfig, apiserverURL string) (Server, error) {
 	sharedInformerFactory := mcfginformers.NewSharedInformerFactory(machineConfigClient, resyncPeriod()())
 	kubeNamespacedSharedInformer := informers.NewFilteredSharedInformerFactory(kubeClient, resyncPeriod()(), "openshift-machine-config-operator", nil)
 
-	mcpInformer, mcInformer, ccInformer, cmInformer :=
+	mcpInformer, mcInformer, ccInformer, cmInformer, moscInformer, mosbInformer :=
 		sharedInformerFactory.Machineconfiguration().V1().MachineConfigPools(),
 		sharedInformerFactory.Machineconfiguration().V1().MachineConfigs(),
 		sharedInformerFactory.Machineconfiguration().V1().ControllerConfigs(),
-		kubeNamespacedSharedInformer.Core().V1().ConfigMaps()
-	mcpLister, mcLister, ccLister, cmLister := mcpInformer.Lister(), mcInformer.Lister(), ccInformer.Lister(), cmInformer.Lister()
-	mcpListerHasSynced, mcListerHasSynced, ccListerHasSynced, cmListerHasSynced :=
+		kubeNamespacedSharedInformer.Core().V1().ConfigMaps(),
+		sharedInformerFactory.Machineconfiguration().V1().MachineOSConfigs(),
+		sharedInformerFactory.Machineconfiguration().V1().MachineOSBuilds()
+	mcpLister, mcLister, ccLister, cmLister, moscLister, mosbLister := mcpInformer.Lister(), mcInformer.Lister(), ccInformer.Lister(), cmInformer.Lister(), moscInformer.Lister(), mosbInformer.Lister()
+	mcpListerHasSynced, mcListerHasSynced, ccListerHasSynced, cmListerHasSynced, moscListerHasSynced, mosbListerHasSynced :=
 		mcpInformer.Informer().HasSynced,
 		mcInformer.Informer().HasSynced,
 		ccInformer.Informer().HasSynced,
-		cmInformer.Informer().HasSynced
+		cmInformer.Informer().HasSynced,
+		moscInformer.Informer().HasSynced,
+		mosbInformer.Informer().HasSynced
 
 	var informerStopCh chan struct{}
 	go sharedInformerFactory.Start(informerStopCh)
 	go kubeNamespacedSharedInformer.Start(informerStopCh)
 
-	if !cache.WaitForCacheSync(informerStopCh, mcpListerHasSynced, mcListerHasSynced, ccListerHasSynced, cmListerHasSynced) {
+	if !cache.WaitForCacheSync(informerStopCh, mcpListerHasSynced, mcListerHasSynced, ccListerHasSynced, cmListerHasSynced, moscListerHasSynced, mosbListerHasSynced) {
 		return nil, errors.New("failed to wait for cache sync")
 	}
 
@@ -101,6 +110,8 @@ func NewClusterServer(kubeConfig, apiserverURL string) (Server, error) {
 		machineConfigLister:     mcLister,
 		controllerConfigLister:  ccLister,
 		configMapLister:         cmLister,
+		machineOSConfigLister:   moscLister,
+		machineOSBuildLister:    mosbLister,
 		kubeconfigFunc:          func() ([]byte, []byte, error) { return kubeconfigFromSecret(bootstrapTokenDir, apiserverURL, nil) },
 		apiserverURL:            apiserverURL,
 	}, nil
@@ -163,7 +174,27 @@ func (cs *clusterServer) GetConfig(cr poolRequest) (*runtime.RawExtension, error
 
 	addDataAndMaybeAppendToIgnition(caBundleFilePath, cc.Spec.KubeAPIServerServingCAData, &ignConf)
 	addDataAndMaybeAppendToIgnition(cloudProviderCAPath, cc.Spec.CloudProviderCAData, &ignConf)
-	appenders := getAppenders(currConf, cr.version, cs.kubeconfigFunc, []string{}, "")
+
+	desiredImage := cs.resolveDesiredImageForPool(mp)
+	klog.Infof("desiredImage is found to be %s", desiredImage)
+
+	appenders := []appenderFunc{
+		func(cfg *ign3types.Config, _ *mcfgv1.MachineConfig) error {
+			return appendNodeAnnotations(cfg, currConf, desiredImage)
+		},
+		func(cfg *ign3types.Config, _ *mcfgv1.MachineConfig) error {
+			return appendKubeConfig(cfg, cs.kubeconfigFunc)
+		},
+		appendInitialMachineConfig,
+		func(cfg *ign3types.Config, _ *mcfgv1.MachineConfig) error { return appendCerts(cfg, []string{}, "") },
+		// Inject desired image into the MC
+		appendDesiredOSImage(desiredImage),
+		// Must be last
+		func(cfg *ign3types.Config, mc *mcfgv1.MachineConfig) error {
+			return appendEncapsulated(cfg, mc, cr.version)
+		},
+	}
+
 	for _, a := range appenders {
 		if err := a(&ignConf, mc); err != nil {
 			return nil, err
@@ -224,3 +255,90 @@ func kubeconfigFromSecret(secretDir, apiserverURL string, caData []byte) ([]byte
 	}
 	return kcData, caData, nil
 }
+
+// Finds the MOSC that targets this MCO and verfies that the MOSC has an image in status
+// locates the matching MOSB for the MCP's current or next rendered MC and confirms build succeeded
+// and returns the image pullspec when its ready
+func (cs *clusterServer) resolveDesiredImageForPool(pool *mcfgv1.MachineConfigPool) string {
+	// Find MachineOSConfig for this pool
+	moscList, err := cs.machineOSConfigLister.List(labels.Everything())
+	if err != nil {
+		// MOSC resources not available
+		klog.Infof("Could not list MachineOSConfigs for pool %s: %v", pool.Name, err)
+		return ""
+	}
+
+	var mosc *mcfgv1.MachineOSConfig
+	for _, config := range moscList {
+		if config.Spec.MachineConfigPool.Name == pool.Name {
+			mosc = config
+			break
+		}
+	}
+
+	// No MOSC for this pool - not layered
+	if mosc == nil {
+		return ""
+	}
+
+	// Check if image is ready in MOSC status
+	moscState := ctrlcommon.NewMachineOSConfigState(mosc)
+	if !moscState.HasOSImage() {
+		klog.Infof("Pool %s has MachineOSConfig but image not ready yet", pool.Name)
+		return ""
+	}
+
+	// Also verify the corresponding MOSB is successful to ensure we don't serve an image that hasn't been built yet
+	mosbList, err := cs.machineOSBuildLister.List(labels.Everything())
+	if err != nil {
+		klog.Infof("Could not list MachineOSBuilds for pool %s: %v", pool.Name, err)
+		return ""
+	}
+
+	var currentConf string
+	if pool.Status.UpdatedMachineCount > 0 {
+		currentConf = pool.Spec.Configuration.Name
+	} else {
+		currentConf = pool.Status.Configuration.Name
+	}
+
+	var mosb *mcfgv1.MachineOSBuild
+	for _, build := range mosbList {
+		if build.Spec.MachineOSConfig.Name == mosc.Name &&
+			build.Spec.MachineConfig.Name == currentConf {
+			mosb = build
+			break
+		}
+	}
+
+	if mosb == nil {
+		klog.Infof("Pool %s has MachineOSConfig but no matching MachineOSBuild for MC %s", pool.Name, currentConf)
+		return ""
+	}
+
+	// Check build is successful
+	mosbState := ctrlcommon.NewMachineOSBuildState(mosb)
+	if !mosbState.IsBuildSuccess() {
+		klog.Infof("Pool %s has MachineOSBuild but build not successful yet", pool.Name)
+		return ""
+	}
+
+	imageSpec := moscState.GetOSImage()
+	klog.Infof("Resolved layered image for pool %s: %s", pool.Name, imageSpec)
+	return imageSpec
+}
+
+// appendDesiredOSImage mutates the MC to include the desired OS image.
+// This runs appendEncapsulated so mcd-firstboot sees the image on first boot.
+func appendDesiredOSImage(desired string) appenderFunc {
+	return func(_ *ign3types.Config, mc *mcfgv1.MachineConfig) error {
+		if desired == "" {
+			return nil
+		}
+		if mc.Spec.OSImageURL != desired {
+			klog.Infof("overriding MC OSImageURL: %q -> %q", mc.Spec.OSImageURL, desired)
+			mc.Spec.OSImageURL = desired
+		}
+		return nil
+	}
+}

pkg/server/server.go

@@ -50,7 +50,7 @@ func getAppenders(currMachineConfig string, version *semver.Version, f kubeconfi
 	appenders := []appenderFunc{
 		// append machine annotations file.
 		func(cfg *ign3types.Config, _ *mcfgv1.MachineConfig) error {
-			return appendNodeAnnotations(cfg, currMachineConfig)
+			return appendNodeAnnotations(cfg, currMachineConfig, "")
 		},
 		// append kubeconfig.
 		func(cfg *ign3types.Config, _ *mcfgv1.MachineConfig) error { return appendKubeConfig(cfg, f) },
@@ -153,8 +153,8 @@ func appendKubeConfig(conf *ign3types.Config, f kubeconfigFunc) error {
 	return nil
 }
 
-func appendNodeAnnotations(conf *ign3types.Config, currConf string) error {
-	anno, err := getNodeAnnotation(currConf)
+func appendNodeAnnotations(conf *ign3types.Config, currConf string, image string) error {
+	anno, err := getNodeAnnotation(currConf, image)
 	if err != nil {
 		return err
 	}
@@ -165,12 +165,17 @@ func appendNodeAnnotations(conf *ign3types.Config, currConf string) error {
 	return nil
 }
 
-func getNodeAnnotation(conf string) (string, error) {
+func getNodeAnnotation(conf string, image string) (string, error) {
 	nodeAnnotations := map[string]string{
 		daemonconsts.CurrentMachineConfigAnnotationKey:     conf,
 		daemonconsts.DesiredMachineConfigAnnotationKey:     conf,
 		daemonconsts.MachineConfigDaemonStateAnnotationKey: daemonconsts.MachineConfigDaemonStateDone,
 	}
+	// If image is provided, include image annotations
+	if image != "" {
+		nodeAnnotations[daemonconsts.CurrentImageAnnotationKey] = image
+		nodeAnnotations[daemonconsts.DesiredImageAnnotationKey] = image
+	}
 	contents, err := json.Marshal(nodeAnnotations)
 	if err != nil {
 		return "", fmt.Errorf("could not marshal node annotations, err: %w", err)

pkg/server/server_test.go

@@ -159,7 +159,7 @@ func TestBootstrapServer(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error while appending file to ignition: %v", err)
 	}
-	anno, err := getNodeAnnotation(mp.Status.Configuration.Name)
+	anno, err := getNodeAnnotation(mp.Status.Configuration.Name, "")
 	if err != nil {
 		t.Fatalf("unexpected error while creating annotations err: %v", err)
 	}
@@ -354,7 +354,7 @@ func TestClusterServer(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error while appending file to ignition: %v", err)
 	}
-	anno, err := getNodeAnnotation(mp.Status.Configuration.Name)
+	anno, err := getNodeAnnotation(mp.Status.Configuration.Name, "")
 	if err != nil {
 		t.Fatalf("unexpected error while creating annotations err: %v", err)
 	}