Merge pull request #35780 from upodroid/kops-job-patch

run the kops baremetal jobs in prow & add hyperdisk jobs

Author: k8s-ci-robot

config/jobs/kubernetes/kops/build_jobs.py

@@ -90,6 +90,7 @@ def build_test(cloud='aws',
                runs_per_day=0,
                scenario=None,
                env=None,
+               instance_groups_overrides=None,
                kubernetes_feature_gates=None,
                build_cluster=None,
                cluster_name=None,
@@ -114,6 +115,9 @@ def build_test(cloud='aws',
     if extra_flags is None:
         extra_flags = []
 
+    if instance_groups_overrides is None:
+        instance_groups_overrides = []
+
     if cloud == 'aws':
         kops_image = distro_images[distro]
         kops_ssh_user = distros_ssh_user[distro]
@@ -234,6 +238,7 @@ def build_test(cloud='aws',
         test_args=test_args,
         cluster_name=cluster_name,
         storage_e2e_cred=storage_e2e_cred,
+        instance_groups_overrides=instance_groups_overrides,
     )
 
     spec = {
@@ -318,7 +323,8 @@ def presubmit_test(branch='master',
                    cluster_name=None,
                    use_preset_for_account_creds=None,
                    alert_email=None,
-                   alert_num_failures=None):
+                   alert_num_failures=None,
+                   instance_groups_overrides=None):
     # pylint: disable=too-many-statements,too-many-branches,too-many-arguments
     if cloud == 'aws':
         if distro == "channels":
@@ -355,6 +361,8 @@ def presubmit_test(branch='master',
     if extra_flags is None:
         extra_flags = []
 
+    if instance_groups_overrides is None:
+        instance_groups_overrides = []
     # TODO: Uncomment when dynamic discovery buckets are available
     # if (irsa and cloud == "aws" and scenario is None and
     #         terraform_version is None and name != "pull-kops-aws-distro-al2023"):
@@ -414,6 +422,7 @@ def presubmit_test(branch='master',
         build_cluster=build_cluster,
         test_args=test_args,
         cluster_name=cluster_name,
+        instance_groups_overrides=instance_groups_overrides,
     )
 
     spec = {
@@ -2043,6 +2052,31 @@ def generate_presubmits_e2e():
             always_run=True,
             extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747
         ),
+        presubmit_test(
+            cloud='gce',
+            k8s_version='stable',
+            kops_channel='alpha',
+            name='pull-kops-e2e-k8s-gce-cilium-hyperdisk',
+            networking='cilium',
+            tab_name='e2e-gce-cilium-hyperdisk',
+            always_run=False,
+            extra_flags=[
+                "--zones=us-east1-b",
+                "--node-size=c4-standard-4",
+                "--master-size=c4-standard-2",
+                "--set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000",
+                "--set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000",
+                "--set spec.etcdClusters[*].etcdMembers[*].volumeSize=60",
+                "--set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced",
+                "--gce-service-account=default"
+            ],
+            instance_groups_overrides=[
+                "spec.rootVolume.type=hyperdisk-balanced",
+                "spec.rootVolume.size=60",
+                "spec.rootVolume.iops=10000",
+                "spec.rootVolume.throughput=1000",
+            ]
+        ),
         presubmit_test(
             cloud='gce',
             k8s_version='stable',

config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml

@@ -1,5 +1,5 @@
 # Test jobs generated by build_jobs.py (do not manually edit)
-# 41 jobs
+# 42 jobs
 presubmits:
   kubernetes/kops:
 
@@ -403,6 +403,80 @@ presubmits:
       testgrid-days-of-results: '90'
       testgrid-tab-name: e2e-gce-cilium
 
+# {"cloud": "gce", "distro": "u2404", "extra_flags": "--zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default", "k8s_version": "stable", "kops_channel": "alpha", "networking": "cilium"}
+  - name: pull-kops-e2e-k8s-gce-cilium-hyperdisk
+    cluster: k8s-infra-prow-build
+    branches:
+    - master
+    always_run: false
+    optional: false
+    skip_report: false
+    labels:
+      preset-k8s-ssh: "true"
+    decorate: true
+    decoration_config:
+      timeout: 90m
+    path_alias: k8s.io/kops
+    spec:
+      serviceAccountName: k8s-kops-test
+      containers:
+      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+        imagePullPolicy: Always
+        command:
+        - runner.sh
+        args:
+        - bash
+        - -c
+        - |
+            make test-e2e-install
+            kubetest2 kops \
+            -v 2 \
+            --up --build --down \
+            --cloud-provider=gce \
+            --admin-access=0.0.0.0/0 \
+            --create-args="--channel=alpha --networking=cilium --zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default" \
+            --kubernetes-version=https://dl.k8s.io/release/stable.txt \
+            --kops-binary-path=/home/prow/go/src/k8s.io/kops/.build/dist/linux/amd64/kops \
+            --control-plane-instance-group-overrides=spec.rootVolume.type=hyperdisk-balanced \
+            --node-instance-group-overrides=spec.rootVolume.type=hyperdisk-balanced \
+            --control-plane-instance-group-overrides=spec.rootVolume.size=60 \
+            --node-instance-group-overrides=spec.rootVolume.size=60 \
+            --control-plane-instance-group-overrides=spec.rootVolume.iops=10000 \
+            --node-instance-group-overrides=spec.rootVolume.iops=10000 \
+            --control-plane-instance-group-overrides=spec.rootVolume.throughput=1000 \
+            --node-instance-group-overrides=spec.rootVolume.throughput=1000 \
+            --test=kops \
+            -- \
+            --test-args="-test.timeout=60m" \
+            --test-package-marker=stable.txt \
+            --parallel=25
+        securityContext:
+          privileged: true
+        env:
+        - name: KUBE_SSH_KEY_PATH
+          value: /etc/ssh-key-secret/ssh-private
+        - name: KUBE_SSH_USER
+          value: prow
+        - name: GOPATH
+          value: /home/prow/go
+        resources:
+          limits:
+            cpu: "4"
+            memory: 6Gi
+          requests:
+            cpu: "4"
+            memory: 6Gi
+    annotations:
+      test.kops.k8s.io/cloud: gce
+      test.kops.k8s.io/distro: u2404
+      test.kops.k8s.io/extra_flags: --zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default
+      test.kops.k8s.io/k8s_version: stable
+      test.kops.k8s.io/kops_channel: alpha
+      test.kops.k8s.io/networking: cilium
+      testgrid-dashboards: kops-presubmits, presubmits-kops, sig-cluster-lifecycle-kops
+      testgrid-days-of-results: '90'
+      testgrid-tab-name: e2e-gce-cilium-hyperdisk
+
 # {"cloud": "gce", "distro": "u2404", "extra_flags": "--gce-service-account=default", "k8s_version": "stable", "kops_channel": "alpha", "networking": "cilium-etcd"}
   - name: pull-kops-e2e-k8s-gce-cilium-etcd
     cluster: k8s-infra-prow-build

config/jobs/kubernetes/kops/kops-presubmits.yaml

@@ -12,7 +12,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -40,7 +40,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -73,7 +73,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         imagePullPolicy: Always
         command:
         - runner.sh
@@ -126,7 +126,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         imagePullPolicy: Always
         command:
         - runner.sh
@@ -179,7 +179,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         imagePullPolicy: Always
         command:
         - runner.sh
@@ -226,7 +226,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -255,7 +255,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -282,7 +282,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -311,7 +311,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -338,7 +338,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -367,7 +367,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -395,7 +395,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -424,7 +424,7 @@ presubmits:
     path_alias: k8s.io/kops
     spec:
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         command:
         - runner.sh
         args:
@@ -464,7 +464,7 @@ presubmits:
     spec:
       serviceAccountName: k8s-kops-test
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         imagePullPolicy: Always
         command:
         - runner.sh
@@ -508,7 +508,7 @@ presubmits:
     spec:
       serviceAccountName: azure
       containers:
-      - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
         imagePullPolicy: Always
         command:
         - runner.sh
@@ -544,3 +544,69 @@ presubmits:
     annotations:
       testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits
       testgrid-tab-name: e2e-azure
+  - name: pull-kops-e2e-baremetal
+    branches:
+    - master
+    always_run: false
+    labels:
+      preset-dind-enabled: "true"
+    cluster: k8s-infra-prow-build
+    decorate: true
+    decoration_config:
+      timeout: 90m
+    path_alias: k8s.io/kops
+    spec:
+      nodeSelector:
+        cloud.google.com/machine-family: c4 # this node type supports nested virtualization
+      containers:
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
+        imagePullPolicy: Always
+        command:
+        - runner.sh
+        args:
+        - tests/e2e/scenarios/bare-metal/run-test
+        # docker-in-docker needs privileged mode
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "7"
+            memory: "25Gi"
+          requests:
+            cpu: "7"
+            memory: "25Gi"
+    annotations:
+      testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits
+  - name: pull-kops-e2e-baremetal-ipv6
+    branches:
+    - master
+    always_run: false
+    labels:
+      preset-dind-enabled: "true"
+    cluster: k8s-infra-prow-build
+    decorate: true
+    decoration_config:
+      timeout: 90m
+    path_alias: k8s.io/kops
+    spec:
+      nodeSelector:
+        cloud.google.com/machine-family: c4 # this node type supports nested virtualization
+      containers:
+      - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master
+        imagePullPolicy: Always
+        command:
+        - runner.sh
+        args:
+        - tests/e2e/scenarios/bare-metal/scenario-ipv6
+        # docker-in-docker needs privileged mode
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: "7"
+            memory: "25Gi"
+          requests:
+            cpu: "7"
+            memory: "25Gi"
+    annotations:
+      testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits

config/jobs/kubernetes/kops/templates/presubmit.yaml.jinja

@@ -70,6 +70,10 @@
             {%- if template_path %}
             --template-path={{template_path}} \
             {%- endif %}
+            {%- for flag in instance_groups_overrides %}
+            --control-plane-instance-group-overrides={{flag}} \
+            --node-instance-group-overrides={{flag}} \
+            {%- endfor %}
             --test=kops \
             -- \
             {%- if test_args %}