From 31352032ee80a839ecea4fe65e4d1618713d9df1 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Tue, 11 Mar 2025 13:27:33 +0000 Subject: [PATCH 1/3] K8s: Node Relay to extend autoscaling Grid with test cloud resources Signed-off-by: Viet Nguyen Duc --- .github/workflows/helm-chart-test.yml | 38 ++++++++-- .github/workflows/k8s-scaling-test.yml | 8 +-- Makefile | 10 ++- NodeBase/Dockerfile | 1 + NodeBase/generate_config | 7 +- NodeBase/generate_relay_config | 16 ++++- Standalone/generate_config | 5 +- charts/selenium-grid/CONFIGURATION.md | 6 +- .../multiple-nodes-platform-relay.yaml | 72 +++++++++++++++++++ .../multiple-nodes-platform.yaml | 6 +- charts/selenium-grid/templates/_helpers.tpl | 4 ++ .../patch-keda/patch-keda-objects-cm.yaml | 49 +++++++++++++ .../patch-keda/patch-keda-objects-job.yaml | 17 +++-- .../templates/relay-node-scaledjobs.yaml | 2 +- charts/selenium-grid/values.yaml | 8 ++- tests/SeleniumTests/__init__.py | 7 +- .../ci/DeploymentAutoscaling-values.yaml | 6 ++ tests/charts/ci/JobAutoscaling-values.yaml | 5 ++ tests/charts/ci/base-resources-values.yaml | 9 +++ tests/charts/make/chart_test.sh | 27 +++++-- 20 files changed, 264 insertions(+), 39 deletions(-) create mode 100644 charts/selenium-grid/multiple-nodes-platform-relay.yaml create mode 100644 charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml diff --git a/.github/workflows/helm-chart-test.yml b/.github/workflows/helm-chart-test.yml index 18dcba271c..69eaa36fc9 100644 --- a/.github/workflows/helm-chart-test.yml +++ b/.github/workflows/helm-chart-test.yml @@ -47,6 +47,7 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: disabled - k8s-version: 'v1.27.16' cluster: 'minikube' @@ -56,6 +57,7 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: job - k8s-version: 'v1.28.15' cluster: 'minikube' @@ -65,8 +67,9 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: deployment - - k8s-version: 'v1.29.14' + - k8s-version: 'v1.29.15' cluster: 'minikube' helm-version: 'v3.14.3' docker-version: '27.5.1' @@ -74,8 +77,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: job_https - - k8s-version: 'v1.30.10' + - k8s-version: 'v1.30.11' cluster: 'minikube' helm-version: 'v3.15.4' docker-version: '27.5.1' @@ -83,8 +87,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: job_hostname - - k8s-version: 'v1.31.6' + - k8s-version: 'v1.31.7' cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' @@ -92,8 +97,9 @@ jobs: test-upgrade: true service-mesh: false os: ubuntu-22.04 + check-records-output: true test-strategy: deployment_https - - k8s-version: 'v1.32.2' + - k8s-version: 'v1.32.3' cluster: 'minikube' helm-version: 'v3.17.0' docker-version: '26.1.4' @@ -101,7 +107,18 @@ jobs: test-upgrade: true service-mesh: true os: ubuntu-22.04 + check-records-output: true test-strategy: playwright_connect_grid + - k8s-version: 'v1.32.3' + cluster: 'minikube' + helm-version: 'v3.17.0' + docker-version: '26.1.4' + python-version: '3.10' + test-upgrade: true + service-mesh: true + os: ubuntu-22.04 + check-records-output: false + test-strategy: job_relay env: CLUSTER: ${{ matrix.cluster }} KUBERNETES_VERSION: ${{ matrix.k8s-version }} @@ -110,6 +127,10 @@ jobs: DOCKER_VERSION: ${{ matrix.docker-version }} TEST_UPGRADE_CHART: ${{ matrix.test-upgrade }} SERVICE_MESH: ${{ matrix.service-mesh }} + CHECK_RECORD_OUTPUT: ${{ matrix.check-records-output }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_REGION: ${{ secrets.SAUCE_REGION }} steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -200,8 +221,13 @@ jobs: timeout_minutes: 30 max_attempts: 3 command: | - NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} \ - && NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video_integrity + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${{ matrix.test-strategy }} + exit_code=$? + if [[ "${CHECK_RECORD_OUTPUT}" = "true" ]] && [[ "${exit_code}" -eq 0 ]]; then + NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make test_video_integrity + exit_code=$? + fi + exit ${exit_code} - name: Upload Helm chart package if: always() uses: actions/upload-artifact@main diff --git a/.github/workflows/k8s-scaling-test.yml b/.github/workflows/k8s-scaling-test.yml index 9d6b2683da..768f92e0ff 100644 --- a/.github/workflows/k8s-scaling-test.yml +++ b/.github/workflows/k8s-scaling-test.yml @@ -75,28 +75,28 @@ jobs: python-version: '3.10' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions - - k8s-version: 'v1.29.14' + - k8s-version: 'v1.29.15' cluster: 'minikube' helm-version: 'v3.14.3' docker-version: '27.5.1' python-version: '3.11' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_job_count_strategy_default - - k8s-version: 'v1.30.10' + - k8s-version: 'v1.30.11' cluster: 'minikube' helm-version: 'v3.15.4' docker-version: '27.5.1' python-version: '3.12' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_deployment_count_in_chaos - - k8s-version: 'v1.31.6' + - k8s-version: 'v1.31.7' cluster: 'minikube' helm-version: 'v3.16.4' docker-version: '27.4.1' python-version: '3.13' os: ubuntu-22.04 test-strategy: test_k8s_autoscaling_deployment_count_with_node_max_sessions - - k8s-version: 'v1.32.2' + - k8s-version: 'v1.32.3' cluster: 'minikube' helm-version: 'v3.17.0' docker-version: '26.1.4' diff --git a/Makefile b/Makefile index e7d715e92b..4a51bf0a64 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ KEDA_TAG_PREV_VERSION := $(or $(KEDA_TAG_PREV_VERSION),$(KEDA_TAG_PREV_VERSION), KEDA_CORE_VERSION := $(or $(KEDA_CORE_VERSION),$(KEDA_CORE_VERSION),2.16.1) KEDA_TAG_VERSION := $(or $(KEDA_TAG_VERSION),$(KEDA_TAG_VERSION),2.16.1-selenium-grid) KEDA_BASED_NAME := $(or $(KEDA_BASED_NAME),$(KEDA_BASED_NAME),ndviet) -KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250225) +KEDA_BASED_TAG := $(or $(KEDA_BASED_TAG),$(KEDA_BASED_TAG),2.16.1-selenium-grid-20250310) TEST_PATCHED_KEDA := $(or $(TEST_PATCHED_KEDA),$(TEST_PATCHED_KEDA),true) all: hub \ @@ -990,6 +990,14 @@ chart_test_autoscaling_job_hostname: TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \ ./tests/charts/make/chart_test.sh JobAutoscaling +chart_test_autoscaling_job_relay: + PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \ + TEST_MULTIPLE_PLATFORMS=true TEST_MULTIPLE_PLATFORMS_RELAY=true CLEAR_POD_HISTORY=true \ + SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \ + VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \ + TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_relay_node_scaledJob_subPath.yaml" \ + ./tests/charts/make/chart_test.sh JobAutoscaling + chart_test_autoscaling_job_multiple_versions_without_explicit: TEST_MULTIPLE_VERSIONS=true TEST_MULTIPLE_VERSIONS_EXPLICIT=false make chart_test_autoscaling_job diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index 18ad6dccb3..898d78088c 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -50,6 +50,7 @@ ENV LANG_WHICH=${LANG_WHICH} \ SE_NODE_REGISTER_CYCLE="10" \ SE_NODE_REGISTER_SHUTDOWN_ON_FAILURE="true" \ SE_OTEL_SERVICE_NAME="selenium-node" \ + SE_NODE_RELAY_ONLY="true" \ # Setting Selenium Manager to work offline SE_OFFLINE="true" \ SE_NODE_BROWSER_VERSION="stable" \ diff --git a/NodeBase/generate_config b/NodeBase/generate_config index aaa5c44997..4f7c38f4c4 100755 --- a/NodeBase/generate_config +++ b/NodeBase/generate_config @@ -65,8 +65,11 @@ fi SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}" # 'browserName' is mandatory for default stereotype -if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]]; then - SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" +if [[ -z "${SE_NODE_STEREOTYPE}" ]] && [[ -n "${SE_NODE_BROWSER_NAME}" ]] && ([[ -z "${SE_NODE_RELAY_URL}" ]] || [[ "${SE_NODE_RELAY_ONLY}" = "false" ]]); then + SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then + SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")" + fi else SE_NODE_STEREOTYPE="${SE_NODE_STEREOTYPE}" fi diff --git a/NodeBase/generate_relay_config b/NodeBase/generate_relay_config index 6bc8607623..7ae230ddb2 100755 --- a/NodeBase/generate_relay_config +++ b/NodeBase/generate_relay_config @@ -8,7 +8,7 @@ fi if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "[relay]" >>"$FILENAME" - echo "url = \"${SE_NODE_RELAY_URL}\"" >>"$FILENAME" + echo "url = \"$(envsubst < <(echo ${SE_NODE_RELAY_URL}))\"" >>"$FILENAME" if [[ -z "${SE_NODE_RELAY_STATUS_ENDPOINT}" ]]; then echo "status-endpoint = \"/status\"" >>"$FILENAME" else @@ -18,7 +18,17 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "protocol-version = \"${SE_NODE_RELAY_PROTOCOL_VERSION}\"" >>"$FILENAME" fi if [[ -z "${SE_NODE_RELAY_STEREOTYPE}" ]]; then - SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_RELAY_BROWSER_NAME}\", \"platformName\": \"${SE_NODE_RELAY_PLATFORM_NAME}\", \"appium:platformVersion\": \"${SE_NODE_RELAY_PLATFORM_VERSION}\"}" + SE_NODE_RELAY_STEREOTYPE="{\"browserName\": \"${SE_NODE_RELAY_BROWSER_NAME:-${SE_NODE_BROWSER_NAME}}\", \"platformName\": \"${SE_NODE_RELAY_PLATFORM_NAME:-${SE_NODE_PLATFORM_NAME}}\"}" + if [[ -n "${SE_NODE_RELAY_PLATFORM_VERSION}" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"appium:platformVersion\":\"${SE_NODE_RELAY_PLATFORM_VERSION}\"}")" + fi + BROWSER_VERSION=${SE_NODE_RELAY_BROWSER_VERSION:-${SE_NODE_BROWSER_VERSION}} + if [[ -n "${BROWSER_VERSION}" ]] && [[ "${BROWSER_VERSION}" != "stable" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"browserVersion\":\"${BROWSER_VERSION}\"}")" + fi + if [[ "${SE_NODE_ENABLE_MANAGED_DOWNLOADS}" = "true" ]]; then + SE_NODE_RELAY_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_RELAY_STEREOTYPE}" "{\"se:downloadsEnabled\": true}")" + fi else SE_NODE_RELAY_STEREOTYPE="${SE_NODE_RELAY_STEREOTYPE}" fi @@ -31,5 +41,5 @@ if [[ -n "${SE_NODE_RELAY_URL}" ]]; then echo "Merged relay stereotype: ${SE_NODE_RELAY_STEREOTYPE}" fi fi - echo "configs = ['${SE_NODE_RELAY_MAX_SESSIONS}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME" + echo "configs = ['${SE_NODE_MAX_SESSIONS:-${SE_NODE_RELAY_MAX_SESSIONS}}', '${SE_NODE_RELAY_STEREOTYPE}']" >>"$FILENAME" fi diff --git a/Standalone/generate_config b/Standalone/generate_config index 5ca98d017c..67f98c6061 100755 --- a/Standalone/generate_config +++ b/Standalone/generate_config @@ -44,7 +44,10 @@ fi SE_NODE_CONTAINER_NAME="${SE_NODE_CONTAINER_NAME:-$(hostname)}" if [[ -z "$SE_NODE_STEREOTYPE" ]]; then - SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", ${SE_BROWSER_BINARY_LOCATION}, \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + SE_NODE_STEREOTYPE="{\"browserName\": \"${SE_NODE_BROWSER_NAME}\", \"browserVersion\": \"${SE_NODE_BROWSER_VERSION}\", \"platformName\": \"${SE_NODE_PLATFORM_NAME}\", \"se:containerName\": \"${SE_NODE_CONTAINER_NAME}\", \"container:hostname\": \"$(hostname)\"}" + if [[ -n "${SE_BROWSER_BINARY_LOCATION}" ]]; then + SE_NODE_STEREOTYPE="$(python3 /opt/bin/json_merge.py "${SE_NODE_STEREOTYPE}" "{${SE_BROWSER_BINARY_LOCATION}}")" + fi else SE_NODE_STEREOTYPE="$SE_NODE_STEREOTYPE" fi diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index bbe4bca908..abbfc40745 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -396,6 +396,8 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists | | autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete | | autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job | +| autoscaling.patchObjectFinalizers.cleanUpScript | string | `""` | Define your custom script to replace the default script | +| autoscaling.patchObjectFinalizers.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file | | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources | | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job | | autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"200m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"200Mi"}}` | Define resources for container in patch job | @@ -639,10 +641,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes | | relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes | | relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes | -| relayNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler | +| relayNode.hpa.browserName | string | `""` | browserName should match with Node stereotype and request capability is scaled by this scaler | | relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName | | relayNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler | -| relayNode.hpa.platformName | string | `"Android"` | platformName should match with Node stereotype and request capability is scaled by this scaler | +| relayNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler | | relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint | | relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option | | relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | diff --git a/charts/selenium-grid/multiple-nodes-platform-relay.yaml b/charts/selenium-grid/multiple-nodes-platform-relay.yaml new file mode 100644 index 0000000000..362d4efae2 --- /dev/null +++ b/charts/selenium-grid/multiple-nodes-platform-relay.yaml @@ -0,0 +1,72 @@ +# Utilize Relay Node to set up hybrid Autoscaling Grid with using on-premise and test cloud provider (e.g. SauceLabs, BrowserStack, etc.) +# +# For example: below incoming requests will be served by Node container on-premise +# options = ChromeOptions() +# options.set_capability('platformName', 'Linux') +# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL) +# +# Below incoming requests will be served by Relay Node where commands are forwarded to test cloud provider +# options = ChromeOptions() +# options.set_capability('platformName', 'macOS 13.0') +# driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL) +crossBrowsers: + chromeNode: + - nameOverride: '{{ $.Release.Name }}-node-chrome-linux' + hpa: + platformName: 'Linux' + firefoxNode: + - nameOverride: '{{ $.Release.Name }}-node-firefox-linux' + hpa: + platformName: 'Linux' + edgeNode: + - nameOverride: '{{ $.Release.Name }}-node-edge-linux' + hpa: + platformName: 'Linux' + relayNode: + - nameOverride: '{{ $.Release.Name }}-node-relay-chrome-macos' + hpa: + browserName: 'chrome' + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-chrome-windows' + hpa: + browserName: 'chrome' + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-firefox-macos' + hpa: + browserName: 'firefox' + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-firefox-windows' + hpa: + browserName: 'firefox' + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-edge-macos' + hpa: + browserName: "MicrosoftEdge" + sessionBrowserName: "msedge" + platformName: 'macOS' + - nameOverride: '{{ $.Release.Name }}-node-relay-edge-windows' + hpa: + browserName: "MicrosoftEdge" + sessionBrowserName: "msedge" + platformName: 'Windows 11' + - nameOverride: '{{ $.Release.Name }}-node-relay-safari-macos' + hpa: + browserName: 'safari' + platformName: 'macOS' + +relayNode: + enabled: true + videoRecorder: + enabled: false + extraEnvironmentVariables: +# - name: SAUCE_USERNAME +# value: "" +# - name: SAUCE_ACCESS_KEY +# value: "" +# - name: SAUCE_REGION +# value: "" +# - name: SE_NODE_RELAY_URL +# value: "https://$SAUCE_USERNAME:$SAUCE_ACCESS_KEY@ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" + extraEnvFrom: +# - secretRef: +# name: your-secret-with-all-env-vars diff --git a/charts/selenium-grid/multiple-nodes-platform.yaml b/charts/selenium-grid/multiple-nodes-platform.yaml index 1819d8b428..a15e9f112b 100644 --- a/charts/selenium-grid/multiple-nodes-platform.yaml +++ b/charts/selenium-grid/multiple-nodes-platform.yaml @@ -21,7 +21,7 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-chrome-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' firefoxNode: - nameOverride: '{{ $.Release.Name }}-node-firefox-platform-any' @@ -34,7 +34,7 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-firefox-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' edgeNode: - nameOverride: '{{ $.Release.Name }}-node-edge-platform-any' @@ -47,5 +47,5 @@ crossBrowsers: browserVersion: '' - nameOverride: '{{ $.Release.Name }}-node-edge-platform-windows' hpa: - platformName: 'windows' + platformName: 'Windows 11' browserVersion: '' diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index fbc286d112..560bc73f00 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -369,6 +369,10 @@ template: value: {{ $nodeCustomCapabilities | quote }} - name: SE_DRAIN_AFTER_SESSION_COUNT value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} + {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }} + - name: SE_NODE_BROWSER_NAME + value: {{ if hasKey .node.hpa "browserName" }}{{ .node.hpa.browserName | quote }}{{ else }}""{{ end }} + {{- end }} {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }} - name: SE_NODE_BROWSER_VERSION value: {{ if hasKey .node.hpa "browserVersion" }}{{ .node.hpa.browserVersion | quote }}{{ else }}""{{ end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml new file mode 100644 index 0000000000..3ea0658e03 --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml @@ -0,0 +1,49 @@ +{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + namespace: {{ .Release.Namespace }} + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- if $.Values.autoscaling.patchObjectFinalizers.cleanUpScript }} + cleanUpScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.cleanUpScript | nindent 4 }} +{{- else }} + cleanUpScript.sh: | + #!/bin/bash + set -e + set -x + echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." + for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch ScaledObjects $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done + for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch ScaledJobs $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done + for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch TriggerAuthentication $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done +{{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml index 0f9d06b7a9..08bf9fd0e0 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -30,16 +30,19 @@ spec: containers: - name: kubectl image: {{ $.Values.global.seleniumGrid.kubectlImage }} - command: ["/bin/sh", "-c"] - args: - - | - echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." - kubectl get ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -o=json | jq '.metadata.finalizers = null' | kubectl apply -f - || true ; - kubectl delete ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ; - kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ; + command: ["/bin/bash", "-c", "/cleanUpScript.sh"] + volumeMounts: + - name: cleanup-script + mountPath: /cleanUpScript.sh + subPath: cleanUpScript.sh {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} + volumes: + - name: cleanup-script + configMap: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }} {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} imagePullSecrets: - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} diff --git a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml index eec44d8eff..c5096e5fb8 100644 --- a/charts/selenium-grid/templates/relay-node-scaledjobs.yaml +++ b/charts/selenium-grid/templates/relay-node-scaledjobs.yaml @@ -28,7 +28,7 @@ spec: {{- $_ := set $podScope "name" (include "seleniumGrid.relayNode.fullname" (list $nodeConfig $)) -}} {{- $_ = set $podScope "node" $nodeConfig -}} {{- $_ = set $podScope "recorder" (mergeOverwrite $.Values.videoRecorder $nodeConfig.videoRecorder) -}} - {{- $_ = set $podScope "uploader" (get $.Values.videoRecorder (.Values.videoRecorder.uploader.name | toString)) -}} + {{- $_ = set $podScope "uploader" (get $.Values.videoRecorder ($podScope.recorder.uploader.name | toString)) -}} {{- $_ = set $podScope "podTemplate" (include "seleniumGrid.podTemplate" $podScope | fromYaml) }} {{- include "seleniumGrid.autoscalingTemplate" $podScope | nindent 2 }} --- diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 09509a2ebd..beb4b51d19 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1039,6 +1039,10 @@ autoscaling: "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation # This should be run before all other hooks (since delete action is called), so use a negative weight "helm.sh/hook-weight": "-1" + # -- Define your custom script to replace the default script + cleanUpScript: "" + # -- Default mode for ConfigMap is mounted as file + defaultMode: 0755 # -- Define an external service account name contains permissions to patch KEDA scaled resources serviceAccount: "" # -- Custom pull secret for container in patch job @@ -1871,13 +1875,13 @@ relayNode: scaledObjectOptions: hpa: # -- browserName should match with Node stereotype and request capability is scaled by this scaler - browserName: "chrome" + browserName: "" # -- sessionBrowserName if the browserName is different from the sessionBrowserName sessionBrowserName: "" # -- browserVersion should match with Node stereotype and request capability is scaled by this scaler browserVersion: "" # -- platformName should match with Node stereotype and request capability is scaled by this scaler - platformName: "Android" + platformName: "" # -- Skip check SSL when connecting to the Graphql endpoint unsafeSsl: '{{ template "seleniumGrid.graphqlURL.unsafeSsl" . }}' # Optional diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index b5632ea56b..c4bb97497e 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -33,15 +33,20 @@ TEST_CUSTOM_SPECIFIC_NAME = os.environ.get('TEST_CUSTOM_SPECIFIC_NAME', 'false').lower() == 'true' TEST_MULTIPLE_VERSIONS = os.environ.get('TEST_MULTIPLE_VERSIONS', 'false').lower() == 'true' TEST_MULTIPLE_PLATFORMS = os.environ.get('TEST_MULTIPLE_PLATFORMS', 'false').lower() == 'true' +TEST_MULTIPLE_PLATFORMS_RELAY = os.environ.get('TEST_MULTIPLE_PLATFORMS_RELAY', 'false').lower() == 'true' TEST_MULTIPLE_VERSIONS_EXPLICIT = os.environ.get('TEST_MULTIPLE_VERSIONS_EXPLICIT', 'true').lower() == 'true' LIST_CHROMIUM_VERSIONS = ['130.0', '129.0', '128.0'] LIST_FIREFOX_VERSIONS = ['132.0', '131.0', '130.0', '129.0', '128.0'] -LIST_PLATFORMS = ['Linux', None, 'Windows'] +LIST_PLATFORMS = ['Linux', None, 'Windows 11'] if not TEST_MULTIPLE_VERSIONS_EXPLICIT: LIST_CHROMIUM_VERSIONS.append(None) LIST_FIREFOX_VERSIONS.append(None) +if TEST_MULTIPLE_PLATFORMS_RELAY: + # Replace index with None to macOS + LIST_PLATFORMS[1] = 'macOS' + SELENIUM_GRID_URL = f"{SELENIUM_GRID_PROTOCOL}://{SELENIUM_GRID_HOST}:{SELENIUM_GRID_PORT}" CLIENT_CONFIG = ClientConfig( remote_server_addr=SELENIUM_GRID_URL, diff --git a/tests/charts/ci/DeploymentAutoscaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml index cb0b103c5f..d072fedeef 100644 --- a/tests/charts/ci/DeploymentAutoscaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -100,3 +100,9 @@ firefoxNode: enabled: *livenessProbe extraVolumeMounts: *extraVolumeMounts extraVolumes: *extraVolumes + +# Configuration for relay nodes +relayNode: + extraEnvFrom: + - secretRef: + name: test-cloud-credentials diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index d19412cc17..b7be78eafc 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -46,3 +46,8 @@ firefoxNode: enabled: *readinessProbe livenessProbe: enabled: *livenessProbe +# Configuration for relay nodes +relayNode: + extraEnvFrom: + - secretRef: + name: test-cloud-credentials diff --git a/tests/charts/ci/base-resources-values.yaml b/tests/charts/ci/base-resources-values.yaml index fbeb383ff8..9269e4329a 100644 --- a/tests/charts/ci/base-resources-values.yaml +++ b/tests/charts/ci/base-resources-values.yaml @@ -76,6 +76,15 @@ edgeNode: cpu: 250m memory: 2500Mi +relayNode: + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 250m + memory: 2500Mi + videoRecorder: resources: requests: diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index fa9fed7f59..1c3626460a 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -66,6 +66,7 @@ TEST_EXISTING_PTS=${TEST_EXISTING_PTS:-"false"} TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS:-"false"} TEST_MULTIPLE_VERSIONS_EXPLICIT=${TEST_MULTIPLE_VERSIONS_EXPLICIT:-"true"} TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS:-"false"} +TEST_MULTIPLE_PLATFORMS_RELAY=${TEST_MULTIPLE_PLATFORMS_RELAY:-"false"} TEST_CUSTOM_SPECIFIC_NAME=${TEST_CUSTOM_SPECIFIC_NAME:-"false"} wait_for_terminated() { @@ -91,6 +92,7 @@ cleanup() { kubectl logs -n ${SELENIUM_NAMESPACE} $pod --all-containers --tail=10000 > tests/tests/pod_logs_${pod}.txt done if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then + helm ls -A echo "Clean up chart release and namespace" helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --no-hooks || true wait_for_terminated @@ -112,12 +114,12 @@ on_failure() { kubectl describe pod -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt echo "There is step failed with exit status $exit_status" sudo chmod -R 777 ${HOST_PATH}/logs + cleanup exit $exit_status } # Trap ERR signal and call on_failure function -trap 'on_failure' ERR EXIT -trap 'cleanup' ERR +trap 'on_failure' ERR if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then rm -rf tests/tests/* @@ -347,11 +349,21 @@ if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ] && [ "${RENDER_HELM_TEMPLATE_ONLY} --from-file=tls.crt=${cert_dir}/tls.crt \ --from-file=tls.key=${cert_dir}/tls.key \ --from-file=server.jks=${cert_dir}/server.jks \ - --from-file=server.pass=${cert_dir}/server.pass + --from-file=server.pass=${cert_dir}/server.pass \ + --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - fi CHART_CERT_PATH="./tests/tests/tls.crt" fi +if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then + kubectl create secret generic -n ${SELENIUM_NAMESPACE} test-cloud-credentials \ + --from-literal=SAUCE_USERNAME=${SAUCE_USERNAME} \ + --from-literal=SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY} \ + --from-literal=SAUCE_REGION=${SAUCE_REGION} \ + --from-literal=SE_NODE_RELAY_URL="https://\$SAUCE_USERNAME:\$SAUCE_ACCESS_KEY@ondemand.\$SAUCE_REGION.saucelabs.com:443/wd/hub" \ + --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - +fi + if [ "${SECURE_INGRESS_ONLY_CONFIG_INLINE}" = "true" ]; then if [ "${SECURE_USE_EXTERNAL_CERT}" = "true" ]; then HELM_COMMAND_SET_IMAGES="${HELM_COMMAND_SET_IMAGES} \ @@ -421,10 +433,14 @@ if [ "${TEST_MULTIPLE_VERSIONS}" = "true" ]; then HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ --values ${CHART_PATH}/multiple-nodes-platform-version.yaml \ " -elif [ "${TEST_MULTIPLE_PLATFORMS}" = "true" ]; then +elif [ "${TEST_MULTIPLE_PLATFORMS}" = "true" ] && [ "${TEST_MULTIPLE_PLATFORMS_RELAY}" != "true" ]; then HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ --values ${CHART_PATH}/multiple-nodes-platform.yaml \ " +elif [ "${TEST_MULTIPLE_PLATFORMS_RELAY}" = "true" ]; then + HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ + --values ${CHART_PATH}/multiple-nodes-platform-relay.yaml \ + " fi HELM_COMMAND_SET_BASE_VALUES="${HELM_COMMAND_SET_BASE_VALUES} \ @@ -504,6 +520,7 @@ export TEST_AUTOSCALING_ITERATIONS=${TEST_AUTOSCALING_ITERATIONS:-"20"} export TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS} export TEST_MULTIPLE_VERSIONS_EXPLICIT=${TEST_MULTIPLE_VERSIONS_EXPLICIT} export TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS} +export TEST_MULTIPLE_PLATFORMS_RELAY=${TEST_MULTIPLE_PLATFORMS_RELAY} export TEST_CUSTOM_SPECIFIC_NAME=${TEST_CUSTOM_SPECIFIC_NAME} if [ "${MATRIX_BROWSER}" = "NoAutoscaling" ]; then ./tests/bootstrap.sh NodeFirefox @@ -527,5 +544,3 @@ else fi wait_for_terminated - -cleanup From 163e8f0607196f8679c915d7cd7ea8c1b564a408 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 17 Mar 2025 09:00:58 +0000 Subject: [PATCH 2/3] [ci] Cloud vendor login pass via request capabilities Signed-off-by: Viet Nguyen Duc --- charts/selenium-grid/CONFIGURATION.md | 7 ++- .../multiple-nodes-platform-relay.yaml | 6 +- .../selenium-grid/templates/_nameHelpers.tpl | 9 ++- .../patch-keda/delete-keda-objects-job.yaml | 55 +++++++++++++++++++ .../patch-keda/patch-keda-objects-cm.yaml | 41 ++++++++++++-- .../patch-keda/patch-keda-objects-job.yaml | 9 +-- charts/selenium-grid/values.yaml | 8 +-- tests/SeleniumTests/__init__.py | 21 +++++++ tests/charts/ci/nameOverride-values.yaml | 2 +- tests/charts/make/chart_test.sh | 4 +- 10 files changed, 136 insertions(+), 26 deletions(-) create mode 100644 charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index abbfc40745..856e204bde 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -394,9 +394,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob | | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job | | autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists | -| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete | -| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job | -| autoscaling.patchObjectFinalizers.cleanUpScript | string | `""` | Define your custom script to replace the default script | +| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `300` | Deadline (in seconds) for patch job to complete | +| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation"}` | Annotations for patch job | +| autoscaling.patchObjectFinalizers.deleteObjectsScript | string | `""` | Define your custom script to replace the default script | +| autoscaling.patchObjectFinalizers.patchFinalizersScript | string | `""` | Define your custom script to replace the default script | | autoscaling.patchObjectFinalizers.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file | | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources | | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job | diff --git a/charts/selenium-grid/multiple-nodes-platform-relay.yaml b/charts/selenium-grid/multiple-nodes-platform-relay.yaml index 362d4efae2..ff89ed139f 100644 --- a/charts/selenium-grid/multiple-nodes-platform-relay.yaml +++ b/charts/selenium-grid/multiple-nodes-platform-relay.yaml @@ -59,14 +59,10 @@ relayNode: videoRecorder: enabled: false extraEnvironmentVariables: -# - name: SAUCE_USERNAME -# value: "" -# - name: SAUCE_ACCESS_KEY -# value: "" # - name: SAUCE_REGION # value: "" # - name: SE_NODE_RELAY_URL -# value: "https://$SAUCE_USERNAME:$SAUCE_ACCESS_KEY@ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" +# value: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" extraEnvFrom: # - secretRef: # name: your-secret-with-all-env-vars diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 059e15cdb8..cef0a0159c 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -260,11 +260,18 @@ Server ConfigMap fullname {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Delete scaledObjects leafover job fullname +*/}} +{{- define "seleniumGrid.keda.deleteObjectsJob.fullname" -}} +{{- printf "%s-scaledobjects-deletion" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Patch scaledObjects finalizers job fullname */}} {{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}} -{{- tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch-scaledobjects-finalizers" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-scaledobjects-finalizers" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml new file mode 100644 index 0000000000..3f965fdc0a --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml @@ -0,0 +1,55 @@ +{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + namespace: {{ .Release.Namespace }} + annotations: &patch_objects_job_annotations + "helm.sh/hook-weight": "-10" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: &patch_objects_job_labels + app: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + metadata: + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + annotations: *patch_objects_job_annotations + {{- end }} + labels: *patch_objects_job_labels + name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + spec: + activeDeadlineSeconds: {{ $.Values.autoscaling.patchObjectFinalizers.activeDeadlineSeconds }} + serviceAccountName: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }} + serviceAccount: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }} + containers: + - name: kubectl + image: {{ $.Values.global.seleniumGrid.kubectlImage }} + command: ["/bin/bash", "-c", "/deleteObjectsScript.sh"] + volumeMounts: + - name: cleanup-script + mountPath: /deleteObjectsScript.sh + subPath: deleteObjectsScript.sh + {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: cleanup-script + configMap: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }} + {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} + imagePullSecrets: + - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} + {{- end }} + restartPolicy: Never + {{- with .Values.autoscaling.patchObjectFinalizers.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml index 3ea0658e03..7cbc212b4f 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml @@ -4,8 +4,9 @@ kind: ConfigMap metadata: name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} namespace: {{ .Release.Namespace }} - {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} annotations: + "helm.sh/hook-weight": "-20" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: @@ -16,14 +17,44 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: -{{- if $.Values.autoscaling.patchObjectFinalizers.cleanUpScript }} - cleanUpScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.cleanUpScript | nindent 4 }} +{{- if $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript }} + deleteObjectsScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript | nindent 4 }} +{{- else }} + deleteObjectsScript.sh: | + #!/bin/bash + set -e + set -x + echo "Delete ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." + for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete ScaledObjects $i -n {{ .Release.Namespace }} + fi + done + for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete ScaledJobs $i -n {{ .Release.Namespace }} + fi + done + for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete TriggerAuthentication $i -n {{ .Release.Namespace }} + fi + done +{{- end }} +{{- if $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript }} + patchFinalizersScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript | nindent 4 }} {{- else }} - cleanUpScript.sh: | + patchFinalizersScript.sh: | #!/bin/bash set -e set -x - echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." + echo "Patch finalizers of ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ -o jsonpath='{.items[*].metadata.name}{"\n"}'); do diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml index 08bf9fd0e0..f4e962260b 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -4,8 +4,9 @@ kind: Job metadata: name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} namespace: {{ .Release.Namespace }} - {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} annotations: &patch_objects_job_annotations + "helm.sh/hook-weight": "-1" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: &patch_objects_job_labels @@ -30,11 +31,11 @@ spec: containers: - name: kubectl image: {{ $.Values.global.seleniumGrid.kubectlImage }} - command: ["/bin/bash", "-c", "/cleanUpScript.sh"] + command: ["/bin/bash", "-c", "/patchFinalizersScript.sh"] volumeMounts: - name: cleanup-script - mountPath: /cleanUpScript.sh - subPath: cleanUpScript.sh + mountPath: /patchFinalizersScript.sh + subPath: patchFinalizersScript.sh {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index beb4b51d19..975ade994f 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1032,15 +1032,15 @@ autoscaling: # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists enabled: true # -- Deadline (in seconds) for patch job to complete - activeDeadlineSeconds: 120 + activeDeadlineSeconds: 300 # -- Annotations for patch job annotations: "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation - # This should be run before all other hooks (since delete action is called), so use a negative weight - "helm.sh/hook-weight": "-1" # -- Define your custom script to replace the default script - cleanUpScript: "" + deleteObjectsScript: "" + # -- Define your custom script to replace the default script + patchFinalizersScript: "" # -- Default mode for ConfigMap is mounted as file defaultMode: 0755 # -- Define an external service account name contains permissions to patch KEDA scaled resources diff --git a/tests/SeleniumTests/__init__.py b/tests/SeleniumTests/__init__.py index c4bb97497e..2802d1fee9 100644 --- a/tests/SeleniumTests/__init__.py +++ b/tests/SeleniumTests/__init__.py @@ -190,6 +190,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, @@ -228,6 +235,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, @@ -271,6 +285,13 @@ def setUp(self): platform_name = random.choice(LIST_PLATFORMS) if platform_name: options.set_capability('platformName', platform_name) + if TEST_MULTIPLE_PLATFORMS_RELAY: + options.set_capability('sauce:options', { + 'username': os.environ.get('SAUCE_USERNAME'), + 'accessKey': os.environ.get('SAUCE_ACCESS_KEY'), + 'name': f"{self._testMethodName} ({self.__class__.__name__})", + 'seleniumVersion': '4.29.0', + }) start_time = time.time() self.driver = webdriver.Remote( options=options, diff --git a/tests/charts/ci/nameOverride-values.yaml b/tests/charts/ci/nameOverride-values.yaml index 0d156cb080..a9a2e57b52 100644 --- a/tests/charts/ci/nameOverride-values.yaml +++ b/tests/charts/ci/nameOverride-values.yaml @@ -28,4 +28,4 @@ secrets: nameOverride: "selenium-grid-common-secrets" autoscaling: patchObjectFinalizers: - nameOverride: "selenium-grid-job-patch-objects" + nameOverride: "selenium-grid-patch" diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index 1c3626460a..5bbca5c2c3 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -357,10 +357,8 @@ fi if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then kubectl create secret generic -n ${SELENIUM_NAMESPACE} test-cloud-credentials \ - --from-literal=SAUCE_USERNAME=${SAUCE_USERNAME} \ - --from-literal=SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY} \ --from-literal=SAUCE_REGION=${SAUCE_REGION} \ - --from-literal=SE_NODE_RELAY_URL="https://\$SAUCE_USERNAME:\$SAUCE_ACCESS_KEY@ondemand.\$SAUCE_REGION.saucelabs.com:443/wd/hub" \ + --from-literal=SE_NODE_RELAY_URL="https://ondemand.\$SAUCE_REGION.saucelabs.com:443/wd/hub" \ --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - fi From 8b8bc3af10eda3fd0de9db8e5a9a95a7f5f09290 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Tue, 18 Mar 2025 07:10:07 +0000 Subject: [PATCH 3/3] Update config key for relayUrl Signed-off-by: Viet Nguyen Duc --- charts/selenium-grid/CONFIGURATION.md | 1 + charts/selenium-grid/README.md | 5 +++++ charts/selenium-grid/multiple-nodes-platform-relay.yaml | 2 ++ charts/selenium-grid/templates/_helpers.tpl | 4 ++++ charts/selenium-grid/values.yaml | 3 ++- tests/charts/ci/DeploymentAutoscaling-values.yaml | 1 + tests/charts/ci/JobAutoscaling-values.yaml | 1 + tests/charts/make/chart_test.sh | 1 - 8 files changed, 16 insertions(+), 2 deletions(-) diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index 0d2e000c66..3b24c595cd 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -593,6 +593,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | edgeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option | | edgeNode.videoRecorder | object | `{}` | Override specific video recording settings for edge node | | relayNode.enabled | bool | `false` | Enable relay nodes | +| relayNode.relayUrl | string | `""` | Specify another Grid, another network, or a cloud vendor that you wish to connect to (e.g. https://ondemand.us-west-1.saucelabs.com/wd/hub) | | relayNode.deploymentEnabled | bool | `true` | NOTE: Only used when autoscaling.enabled is false Enable creation of Deployment true (default) - if you want long-living pods false - for provisioning your own custom type such as Jobs | | relayNode.updateStrategy | object | `{"type":"RollingUpdate"}` | Global update strategy will be overwritten by individual component | | relayNode.replicas | int | `1` | Number of relay nodes | diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index ea1b84bec9..7946d095b2 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -360,6 +360,11 @@ For example [multiple-nodes-platform.yaml](./multiple-nodes-platform.yaml) file, For example [multiple-nodes-platform-version.yaml](./multiple-nodes-platform-version.yaml) file, it defines multiple scalers with `platformName: 'Linux'` and last few previous stable versions per browser node to scale against requests with `browserVersion` and `platformName` capabilities. +To extend your Grid, you can use Relay Node (which allows you to route Grid tests to another Grid, another network, or a cloud vendor). +Besides on-prem browser Nodes with Linux-based, you also can serve test requests with other platforms, browsers or even mobile devices which provided by cloud vendors. +Your teams will not worry about the underlying infrastructure, they just request to the single Grid endpoint hosted in your organization. +Check out values file [multiple-nodes-platform-relay.yaml](./multiple-nodes-platform-relay.yaml) for more details. + While deploying the chart, you can quickly use these extra values files by passing the file via `--values` flag to apply. ### Settings fixed-sized thread pool for the Distributor to create new sessions diff --git a/charts/selenium-grid/multiple-nodes-platform-relay.yaml b/charts/selenium-grid/multiple-nodes-platform-relay.yaml index ff89ed139f..3139325f3a 100644 --- a/charts/selenium-grid/multiple-nodes-platform-relay.yaml +++ b/charts/selenium-grid/multiple-nodes-platform-relay.yaml @@ -56,11 +56,13 @@ crossBrowsers: relayNode: enabled: true +# relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" videoRecorder: enabled: false extraEnvironmentVariables: # - name: SAUCE_REGION # value: "" +# Or can give relay url directly to environment variable SE_NODE_RELAY_URL. Value can be referring to value of existing env vars. # - name: SE_NODE_RELAY_URL # value: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" extraEnvFrom: diff --git a/charts/selenium-grid/templates/_helpers.tpl b/charts/selenium-grid/templates/_helpers.tpl index 560bc73f00..9e609f7a97 100644 --- a/charts/selenium-grid/templates/_helpers.tpl +++ b/charts/selenium-grid/templates/_helpers.tpl @@ -369,6 +369,10 @@ template: value: {{ $nodeCustomCapabilities | quote }} - name: SE_DRAIN_AFTER_SESSION_COUNT value: {{ and (eq (include "seleniumGrid.useKEDA" $) "true") (eq .Values.autoscaling.scalingType "job") | ternary $nodeMaxSessions 0 | quote }} + {{- with .node.relayUrl }} + - name: SE_NODE_RELAY_URL + value: {{ . | quote }} + {{- end }} {{- if and (eq (include "seleniumGrid.useKEDA" $) "true") }} - name: SE_NODE_BROWSER_NAME value: {{ if hasKey .node.hpa "browserName" }}{{ .node.hpa.browserName | quote }}{{ else }}""{{ end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 975ade994f..f752b8d8f1 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1704,7 +1704,8 @@ edgeNode: relayNode: # -- Enable relay nodes enabled: false - + # -- Specify another Grid, another network, or a cloud vendor that you wish to connect to (e.g. https://ondemand.us-west-1.saucelabs.com/wd/hub) + relayUrl: "" # -- NOTE: Only used when autoscaling.enabled is false # Enable creation of Deployment # true (default) - if you want long-living pods diff --git a/tests/charts/ci/DeploymentAutoscaling-values.yaml b/tests/charts/ci/DeploymentAutoscaling-values.yaml index d072fedeef..d9f1949f50 100644 --- a/tests/charts/ci/DeploymentAutoscaling-values.yaml +++ b/tests/charts/ci/DeploymentAutoscaling-values.yaml @@ -103,6 +103,7 @@ firefoxNode: # Configuration for relay nodes relayNode: + relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" extraEnvFrom: - secretRef: name: test-cloud-credentials diff --git a/tests/charts/ci/JobAutoscaling-values.yaml b/tests/charts/ci/JobAutoscaling-values.yaml index b7be78eafc..6310351970 100644 --- a/tests/charts/ci/JobAutoscaling-values.yaml +++ b/tests/charts/ci/JobAutoscaling-values.yaml @@ -48,6 +48,7 @@ firefoxNode: enabled: *livenessProbe # Configuration for relay nodes relayNode: + relayUrl: "https://ondemand.$SAUCE_REGION.saucelabs.com:443/wd/hub" extraEnvFrom: - secretRef: name: test-cloud-credentials diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index d94e531b14..790abdd3bb 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -358,7 +358,6 @@ fi if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then kubectl create secret generic -n ${SELENIUM_NAMESPACE} test-cloud-credentials \ --from-literal=SAUCE_REGION=${SAUCE_REGION} \ - --from-literal=SE_NODE_RELAY_URL="https://ondemand.\$SAUCE_REGION.saucelabs.com:443/wd/hub" \ --dry-run=client -o yaml | kubectl apply -n ${SELENIUM_NAMESPACE} -f - fi