From 11cce17a7f1d1933608774a1c46f97fb6cd94415 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 7 Apr 2025 11:09:34 +0700 Subject: [PATCH] K8s: Update strategy as Recreate by default Grid components still not support well for RollingUpdate Signed-off-by: Viet Nguyen Duc --- .github/workflows/helm-chart-test.yml | 2 +- charts/selenium-grid/CONFIGURATION.md | 10 +-- charts/selenium-grid/values.yaml | 10 +-- tests/charts/ci/base-auth-ingress-values.yaml | 2 +- tests/charts/ci/base-subPath-values.yaml | 7 -- tests/charts/make/chart_test.sh | 6 ++ .../refValues/simplex-docker-desktop.yaml | 77 +++++++++++++++++++ tests/charts/refValues/simplex-minikube.yaml | 19 +---- tests/charts/templates/test.py | 7 +- 9 files changed, 102 insertions(+), 38 deletions(-) create mode 100644 tests/charts/refValues/simplex-docker-desktop.yaml diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index a26c21abdb..e2e6855b0f 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -263,7 +263,7 @@ jobs: - name: Test chart upgrade if: (matrix.test-upgrade == true) run: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SET_MAX_REPLICAS=10 TEST_NAME_OVERRIDE=true TEST_UPGRADE_CHART=${TEST_UPGRADE_CHART} make chart_test_autoscaling_${{ matrix.test-strategy }} + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} SET_MAX_REPLICAS=10 TEST_NAME_OVERRIDE=true TEST_UPGRADE_CHART=${TEST_UPGRADE_CHART} SET_UPDATE_STRATEGY=Recreate make chart_test_autoscaling_${{ matrix.test-strategy }} - name: Cleanup Kubernetes cluster if: always() run: CLUSTER=${CLUSTER} make chart_cluster_cleanup diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index c343f819d3..10a3da0038 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -44,7 +44,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | global.seleniumGrid.revisionHistoryLimit | int | `10` | Specify how many old ReplicaSets for this Deployment you want to retain. The rest will be garbage-collected in the background. | | global.seleniumGrid.structuredLogs | bool | `false` | Whether to enable structured logging | | global.seleniumGrid.httpLogs | bool | `false` | Enable http logging. Tracing should be enabled to log http logs. | -| global.seleniumGrid.updateStrategy.type | string | `"RollingUpdate"` | Specify update strategy for all components, can be overridden individually | +| global.seleniumGrid.updateStrategy.type | string | `"Recreate"` | Specify update strategy for all components, can be overridden individually | | global.seleniumGrid.updateStrategy.rollingUpdate | object | `{"maxSurge":1,"maxUnavailable":0}` | Specify for strategy RollingUpdate | | global.seleniumGrid.affinity | object | `{}` | Specify affinity for all components, can be overridden individually | | global.seleniumGrid.topologySpreadConstraints | list | `[]` | Specify topologySpreadConstraints for all components, can be overridden individually | @@ -420,7 +420,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | crossBrowsers.relayNode | list | `[{"nameOverride":null}]` | Additional release nodes, array of objects with the same structure as `relayNode` | | chromeNode.enabled | bool | `true` | Enable chrome nodes | | chromeNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long-living pods false - for provisioning your own custom type such as Jobs | -| chromeNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | +| chromeNode.updateStrategy | object | `{"type":null}` | Global update strategy will be overwritten by individual component | | chromeNode.replicas | int | `1` | Number of chrome nodes | | chromeNode.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | | chromeNode.imageName | string | `"node-chrome"` | Image of chrome nodes | @@ -478,7 +478,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | chromeNode.videoRecorder | object | `{}` | Override specific video recording settings for chrome node | | firefoxNode.enabled | bool | `true` | Enable firefox nodes | | firefoxNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long living pods false - for provisioning your own custom type such as Jobs | -| firefoxNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | +| firefoxNode.updateStrategy | object | `{"type":null}` | Global update strategy will be overwritten by individual component | | firefoxNode.replicas | int | `1` | Number of firefox nodes | | firefoxNode.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | | firefoxNode.imageName | string | `"node-firefox"` | Image of firefox nodes | @@ -536,7 +536,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | firefoxNode.videoRecorder | object | `{}` | Override specific video recording settings for firefox node | | edgeNode.enabled | bool | `true` | Enable edge nodes | | edgeNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long living pods false - for provisioning your own custom type such as Jobs | -| edgeNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | +| edgeNode.updateStrategy | object | `{"type":null}` | Global update strategy will be overwritten by individual component | | edgeNode.replicas | int | `1` | Number of edge nodes | | edgeNode.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | | edgeNode.imageName | string | `"node-edge"` | Image of edge nodes | @@ -595,7 +595,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | relayNode.enabled | bool | `false` | Enable relay nodes | | relayNode.relayUrl | string | `""` | Specify another Grid, another network, or a cloud vendor that you wish to connect to (e.g. https://ondemand.us-west-1.saucelabs.com/wd/hub) | | relayNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long-living pods false - for provisioning your own custom type such as Jobs | -| relayNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | +| relayNode.updateStrategy | object | `{"type":null}` | Global update strategy will be overwritten by individual component | | relayNode.replicas | int | `1` | Number of relay nodes | | relayNode.imageRegistry | string | `nil` | Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) | | relayNode.imageName | string | `"node-base"` | Image of relay nodes | diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 63543cad34..642920068c 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -34,7 +34,7 @@ global: httpLogs: false updateStrategy: # -- Specify update strategy for all components, can be overridden individually - type: RollingUpdate + type: Recreate # type: RollingUpdate # -- Specify for strategy RollingUpdate rollingUpdate: @@ -1124,7 +1124,7 @@ chromeNode: deploymentEnabled: true # -- Global update strategy will be overwritten by individual component updateStrategy: - type: RollingUpdate + type: # -- Number of chrome nodes replicas: 1 # -- Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) @@ -1320,7 +1320,7 @@ firefoxNode: deploymentEnabled: true # -- Global update strategy will be overwritten by individual component updateStrategy: - type: RollingUpdate + type: # -- Number of firefox nodes replicas: 1 # -- Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) @@ -1516,7 +1516,7 @@ edgeNode: deploymentEnabled: true # -- Global update strategy will be overwritten by individual component updateStrategy: - type: RollingUpdate + type: # -- Number of edge nodes replicas: 1 # -- Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) @@ -1713,7 +1713,7 @@ relayNode: deploymentEnabled: true # -- Global update strategy will be overwritten by individual component updateStrategy: - type: RollingUpdate + type: # -- Number of relay nodes replicas: 1 # -- Registry to pull the image (this overwrites global.seleniumGrid.imageRegistry parameter) diff --git a/tests/charts/ci/base-auth-ingress-values.yaml b/tests/charts/ci/base-auth-ingress-values.yaml index e208797a20..6ddc97bcb5 100644 --- a/tests/charts/ci/base-auth-ingress-values.yaml +++ b/tests/charts/ci/base-auth-ingress-values.yaml @@ -55,4 +55,4 @@ ingress-nginx: enabled: true kind: DaemonSet service: - type: ClusterIP + type: LoadBalancer diff --git a/tests/charts/ci/base-subPath-values.yaml b/tests/charts/ci/base-subPath-values.yaml index e53a80deb9..5cde10ecd8 100644 --- a/tests/charts/ci/base-subPath-values.yaml +++ b/tests/charts/ci/base-subPath-values.yaml @@ -11,13 +11,6 @@ ingress: name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' port: number: 4444 - - path: /(/?)(session/.*/element) - pathType: ImplementationSpecific - backend: - service: - name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' - port: - number: 4444 hub: subPath: *gridAppRoot diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 66cade8b9a..c083a33860 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -185,6 +185,12 @@ HELM_COMMAND_SET_IMAGES=" \ --set edgeNode.nodeMaxSessions=${MAX_SESSIONS_EDGE} \ " +if [ -n "${SET_UPDATE_STRATEGY}" ]; then + HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ + --set global.seleniumGrid.updateStrategy.type=${SET_UPDATE_STRATEGY} \ + " +fi + if [ -n "${TRACING_EXPORTER_ENDPOINT}" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ --set tracing.exporterEndpoint=\\"${TRACING_EXPORTER_ENDPOINT}\\" \ diff --git a/tests/charts/refValues/simplex-docker-desktop.yaml b/tests/charts/refValues/simplex-docker-desktop.yaml new file mode 100644 index 0000000000..783346e3f4 --- /dev/null +++ b/tests/charts/refValues/simplex-docker-desktop.yaml @@ -0,0 +1,77 @@ +# README: This is a sample values for chart deployment in K8s cluster started by Docker Desktop +# Chart dependency ingress-nginx is installed together by enabling `ingress.enableWithController` +# Chart dependency keda is installed together by enabling `autoscaling.enable` +# Enabled ingress without hostname, set the subPath `/selenium`. NGINX type LoadBalancer to expose access from `http://localhost/selenium` +global: + seleniumGrid: + logLevel: INFO + +tls: + ingress: + enabled: true + +ingress: + enableWithController: true + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/app-root: &gridAppRoot "/selenium" + className: nginx + hostname: "" + paths: + - path: /selenium(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: '{{ ternary (include "seleniumGrid.router.fullname" $ ) (include "seleniumGrid.hub.fullname" $ ) $.Values.isolateComponents }}' + port: + number: 4444 + +basicAuth: + enabled: false + +isolateComponents: true + +autoscaling: + enabled: true + scalingType: job + scaledOptions: + minReplicaCount: 0 + maxReplicaCount: 8 + pollingInterval: 15 + scaledJobOptions: + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 5 + scalingStrategy: + strategy: default + +hub: + subPath: *gridAppRoot + serviceType: NodePort + +components: + router: + subPath: *gridAppRoot + serviceType: NodePort + +chromeNode: + extraEnvironmentVariables: &extraEnvironmentVariablesNodes + - name: SE_VNC_NO_PASSWORD + value: "true" + +firefoxNode: + extraEnvironmentVariables: *extraEnvironmentVariablesNodes + +edgeNode: + extraEnvironmentVariables: *extraEnvironmentVariablesNodes + +videoRecorder: + enabled: true + +ingress-nginx: + controller: + hostPort: + enabled: true + kind: DaemonSet + service: + type: LoadBalancer diff --git a/tests/charts/refValues/simplex-minikube.yaml b/tests/charts/refValues/simplex-minikube.yaml index 2213e8bd9b..b36417ce05 100644 --- a/tests/charts/refValues/simplex-minikube.yaml +++ b/tests/charts/refValues/simplex-minikube.yaml @@ -6,18 +6,12 @@ # Components serviceType is set to NodePort to allow access from outside the cluster via K8S_PUBLIC_IP and NodePort http://:30444/selenium # Use this reference values to deploy e.g. `helm upgrade --install test --values tests/charts/refValues/simplex-minikube.yaml docker-selenium/selenium-grid --version <0.26.3_onwards>` global: - K8S_PUBLIC_IP: "10.10.10.10" # Replace with your public IP seleniumGrid: logLevel: INFO -# imageRegistry: selenium -# imageTag: latest -# nodesImageTag: latest -# videoImageTag: latest tls: -# enabled: true ingress: - generateTLS: true + enabled: true ingress: enableWithController: true @@ -42,11 +36,8 @@ basicAuth: isolateComponents: true autoscaling: -# enabled: true - enableWithExistingKEDA: true + enabled: true scalingType: job - annotations: - helm.sh/hook: post-install,post-upgrade,post-rollback scaledOptions: minReplicaCount: 0 maxReplicaCount: 8 @@ -68,12 +59,8 @@ components: chromeNode: extraEnvironmentVariables: &extraEnvironmentVariablesNodes - - name: SE_NODE_SESSION_TIMEOUT - value: "300" - name: SE_VNC_NO_PASSWORD value: "true" - - name: SE_NODE_ENABLE_MANAGED_DOWNLOADS - value: "true" firefoxNode: extraEnvironmentVariables: *extraEnvironmentVariablesNodes @@ -90,4 +77,4 @@ ingress-nginx: enabled: true kind: DaemonSet service: - type: ClusterIP + type: LoadBalancer diff --git a/tests/charts/templates/test.py b/tests/charts/templates/test.py index a176cf3af4..c61448ec58 100644 --- a/tests/charts/templates/test.py +++ b/tests/charts/templates/test.py @@ -269,10 +269,11 @@ def test_update_strategy_in_all_components(self): f'{RELEASE_NAME}selenium-event-bus', f'{RELEASE_NAME}selenium-router', f'{RELEASE_NAME}selenium-session-map', - f'{RELEASE_NAME}selenium-session-queue',] - rolling = [f'{RELEASE_NAME}selenium-node-chrome', + f'{RELEASE_NAME}selenium-session-queue', + f'{RELEASE_NAME}selenium-node-chrome', f'{RELEASE_NAME}selenium-node-edge', f'{RELEASE_NAME}selenium-node-firefox',] + rolling = [] count_recreate = 0 count_rolling = 0 for doc in LIST_OF_DOCUMENTS: @@ -282,7 +283,7 @@ def test_update_strategy_in_all_components(self): count_rolling += 1 if doc['metadata']['name'] in recreate and doc['kind'] == 'Deployment': logger.info(f"Assert updateStrategy is set in resource {doc['metadata']['name']}") - self.assertTrue(doc['spec']['strategy']['type'] == 'RollingUpdate', f"Resource {doc['metadata']['name']} doesn't have strategy RollingUpdate") + self.assertTrue(doc['spec']['strategy']['type'] == 'Recreate', f"Resource {doc['metadata']['name']} doesn't have strategy Recreate") count_recreate += 1 self.assertEqual(count_rolling, len(rolling), "No deployment resources found with strategy RollingUpdate") self.assertEqual(count_recreate, len(recreate), "No deployment resources found with strategy Recreate")