Add node affinity to kit substrate components (#178)

* Add node affinity to kit substrate components

* Implement review comments

Author: felix-zhe-huang

operator/charts/kit-operator/templates/controller/deployment.yaml

@@ -25,4 +25,16 @@ spec:
             memory: 20Mi
       # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074
       securityContext:
-        fsGroup: 1000
+        fsGroup: 1000
+      {{- with .Values.controller.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

operator/charts/kit-operator/templates/webhook/deployment.yaml

@@ -43,4 +43,17 @@ spec:
                   fieldPath: metadata.namespace
       # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074
       securityContext:
-        fsGroup: 1000
+        fsGroup: 1000
+      {{- with .Values.webhook.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhook.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhook.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+

operator/charts/kit-operator/values.yaml

@@ -17,4 +17,4 @@ webhook:
   tolerations: []
   affinity: {}
   # TODO this will be updated by the git actions
-  image: "public.ecr.aws/kit/kit-webhook:latest"
+  image: "public.ecr.aws/kit/kit-webhook:latest"

substrate/pkg/controller/substrate/cluster/addons/kitoperator.go

@@ -33,9 +33,13 @@ func (l *KITOperator) Create(ctx context.Context, substrate *v1alpha1.Substrate)
 	if err := helm.NewClient(*substrate.Status.Cluster.KubeConfig).Apply(ctx, &helm.Chart{
 		Namespace:       "kit",
 		Name:            "kit-operator",
-		Repository:      "https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/kit-operator-0.0.12",
-		Version:         "0.0.12",
+		Repository:      "https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/latest",
+		Version:         "latest",
 		CreateNamespace: true,
+		Values: map[string]interface{}{
+			"controller": map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+			"webhook":    map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+		},
 	}); err != nil {
 		return reconcile.Result{}, fmt.Errorf("applying chart, %w", err)
 	}

substrate/pkg/controller/substrate/cluster/addons/prometheusstack.go

@@ -44,8 +44,9 @@ func (p *PrometheusStack) Create(ctx context.Context, substrate *v1alpha1.Substr
 			"kubeApiServer":         map[string]interface{}{"enabled": false},
 			"kubeStateMetrics":      map[string]interface{}{"enabled": false},
 			"kubeControllerManager": map[string]interface{}{"enabled": false},
-			"prometheus":            map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}},
-			"prometheusOperator":    map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}},
+			"prometheus":            map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}, "prometheusSpec": map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}}},
+			"prometheusOperator":    map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}, "nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+			"grafana":               map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
 		},
 	}); err != nil {
 		return reconcile.Result{}, fmt.Errorf("applying chart, %w", err)