Revise all e2e tests to use the kind cluster with local registry

Signed-off-by: Ye Cao <[email protected]>

Author: dashanji

.github/workflows/vineyard-operator.yaml

@@ -122,7 +122,13 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
+      
+      - name: Build a kind cluster with local registry
+        run: |
+          touch /tmp/e2e-k8s.config
+          export KUBECONFIG=/tmp/e2e-k8s.config
+          bash k8s/test/hack/build-kind-cluster-with-local-registry.sh
+      
       - name: Build docker image
         run: |
           make -C k8s docker-build
@@ -179,7 +185,7 @@ jobs:
           e2e-file: k8s/test/e2e/failover/e2e.yaml
 
       - name: e2e-tests-repartition-dask
-        if: false  ## "No space left on device" error on CI.  ${{ matrix.job == 'e2e-tests-repartition-dask' }}
+        if: ${{ matrix.job == 'e2e-tests-repartition-dask' }}
         uses: apache/skywalking-infra-e2e@main
         with:
           e2e-file: k8s/test/e2e/repartition/dask-repartition-e2e.yaml

k8s/Makefile

@@ -46,7 +46,7 @@ KUSTOMIZE_VERSION ?= [email protected]
 CONTROLLER_TOOLS_VERSION ?= v0.9.2
 GOLANGCI_LINT_VERSION ?= v1.49.0
 HELMIFY_VERSION ?= v0.3.18
-E2E_VERSION ?= afdf1cca0519d65bc480d8680b7a27f9b41fc421
+E2E_VERSION ?= 2631e76926604c4e30ca170bed916804c86980b6
 
 all: manager
 
@@ -56,68 +56,116 @@ check: lint e2e-test
 
 # Build a 4-nodes(1 master and 3 workers) kind cluster with local registry
 build-local-cluster:
-	bash test/hack/build-kind-cluster-with-local-registry.sh $(name)
+	@touch /tmp/e2e-k8s.config
+	@echo "the kubeconfig path is /tmp/e2e-k8s.config"
+	@echo "Creating the kind cluster with local registry"
+	@bash test/hack/build-kind-cluster-with-local-registry.sh
 .PHONY: build-local-cluster
 
-# Run e2e tests
-e2e-test: e2e prepare-e2e-test vineyardd-e2e-test serialize-e2e-test spill-e2e-test autogenerated-helm-chart-e2e-test local-assembly-e2e-test distributed-assembly-e2e-test repartition-e2e-test sidecar-e2e-test
-	@echo "All e2e test2 passed!"
-.PHONY: e2e-test
+delete-local-cluster:
+	@rm /tmp/e2e-k8s.config
+	@echo "Deleting the kind cluster with local registry"
+	@bash test/hack/delete-kind-cluster-with-local-registry.sh
+.PHONY: delete-local-cluster
 
-prepare-e2e-test:
+prepare-e2e-test: e2e
 	@echo "Install e2e test dependencies..."
-	sudo bash test/hack/prepare-e2e.sh
+	@sudo bash test/hack/prepare-e2e.sh
 	@echo "Build vineyard operator image..."
-	make docker-build
+	@make docker-build
 .PHONY: prepare-e2e-test
 
-vineyardd-e2e-test:
-	@echo "Running vineyardd e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/vineyardd/e2e.yaml
-	@echo "vineyardd e2e test passed."
-.PHONY: vineyardd-e2e-test
-
-serialize-e2e-test:
-	@echo "Running serialize e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/serialize/e2e.yaml
-	@echo "serialize e2e test passed."
-.PHONY: serialize-e2e-test
-
-spill-e2e-test:
-	@echo "Running spill e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/spill/e2e.yaml
-	@echo "spill e2e test passed."
-.PHONY: spill-e2e-test
-
-autogenerated-helm-chart-e2e-test:
-	@echo "Running autogenerated helm chart e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
-	@echo "autogenerated helm chart e2e test passed."
-.PHONY: autogenerated-helm-chart-e2e-test
-
-local-assembly-e2e-test:
+e2e-tests:
+	
+e2e-tests-assembly-local: prepare-e2e-test build-local-cluster kustomize
 	@echo "Running local assembly e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/local-assembly-e2e.yaml
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/local-assembly-e2e.yaml
 	@echo "local assembly e2e test passed."
-.PHONY: local-assembly-e2e-test
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-assembly-local
 
-distributed-assembly-e2e-test:
+e2e-tests-assembly-distributed: prepare-e2e-test build-local-cluster
 	@echo "Running distributed assembly e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/distributed-assembly-e2e.yaml
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/assembly/distributed-assembly-e2e.yaml
 	@echo "distributed assembly e2e test passed."
-.PHONY: distributed-assembly-e2e-test
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-assembly-distributed
 
-repartition-e2e-test:
-	@echo "Running repartition e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/repartition/e2e.yaml
-	@echo "repartition e2e test passed."
-.PHONY: repartition-e2e-test
+e2e-tests-autogenerated-helm-chart: prepare-e2e-test build-local-cluster
+	@echo "Running autogenerated helm chart e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/autogenerated-helm-chart/e2e.yaml
+	@echo "autogenerated helm chart e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-autogenerated-helm-chart
+
+e2e-tests-failover: prepare-e2e-test build-local-cluster
+	@echo "Running failover e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/failover/e2e.yaml
+	@echo "failover e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-failover
+
+e2e-tests-repartition-dask: prepare-e2e-test build-local-cluster
+	@echo "Running repartition dask e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/repartition/dask-repartition-e2e.yaml
+	@echo "repartition dask e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-repartition-dask
+
+e2e-tests-scheduler-outside-cluster: prepare-e2e-test build-local-cluster
+	@echo "Running scheduler outside cluster e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/scheduling-outside-cluster/e2e.yaml
+	@echo "scheduler outside cluster e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-scheduler-outside-cluster
+
+e2e-tests-serialize: prepare-e2e-test build-local-cluster
+	@echo "Running serialize e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/serialize/e2e.yaml
+	@echo "serialize e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-serialize
 
-sidecar-e2e-test:
+e2e-tests-sidecar: prepare-e2e-test build-local-cluster
 	@echo "Running sidecar e2e test..."
-	cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/sidecar/e2e.yaml
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/sidecar/e2e.yaml
 	@echo "sidecar e2e test passed."
-.PHONY: sidecar-e2e-test
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-sidecar
+
+e2e-tests-spill: prepare-e2e-test build-local-cluster
+	@echo "Running spill e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/spill/e2e.yaml
+	@echo "spill e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-spill
+
+e2e-tests-workflow: prepare-e2e-test build-local-cluster
+	@echo "Running workflow e2e test..."
+	@cd .. && ${GOBIN}/e2e run --config=k8s/test/e2e/workflow/e2e.yaml
+	@echo "workflow e2e test passed."
+	@# the next step is recover the default vineyard operator image(vineyardcloudnative/vineyard-operator)
+	@cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	@make delete-local-cluster
+.PHONY: e2e-tests-workflow
 
 # Build manager binary
 manager: generate fmt
@@ -196,7 +244,7 @@ helmify:
 # Install e2e if not exist
 e2e:
     ifeq (${E2E},)
-		@echo "helmify not exist, installing it..."
+		@echo "e2e not exist, installing it..."
 		go install github.com/apache/skywalking-infra-e2e/cmd/e2e@$(E2E_VERSION)
     endif
 	@echo "e2e installed"

k8s/README.md

@@ -34,3 +34,11 @@ To deploy the vineyard operator, run
 ```bash
 make deploy
 ```
+
+### Test
+
+You could create a default kind cluster and run a specific e2e test locally as follows.
+
+```bash
+make -C k8s e2e-tests-assembly-local
+```

k8s/pkg/templates/operation/dask-repartition.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: dask-repartition-operation
-        image: ghcr.io/v6d-io/v6d/dask-repartition
+        image: localhost:5001/dask-repartition
         imagePullPolicy: IfNotPresent
         env:
         - name: GLOBALOBJECT_ID

k8s/test/e2e/Makefile

@@ -1,5 +1,7 @@
+# setup the registry
+DEFAULT_REGISTRY := ghcr.io/v6d-io/v6d
 ifeq ($(REGISTRY),)
-    REGISTRY := ghcr.io/v6d-io/v6d
+    REGISTRY := $(DEFAULT_REGISTRY)
 endif
 
 TAG ?= latest
@@ -9,15 +11,24 @@ push-%: %
 	docker push $(REGISTRY)/$<
 .PHONY: push-%
 
+# the phony is to change the image's registry to the default registry(ghcr.io/v6d-io/v6d)
+# as it's hard-coded in the vineyard-operator, such as assembly、repartition、backup、recover etc.
+tag-%: %
+	docker tag $(REGISTRY)/$< $(DEFAULT_REGISTRY)/$<
+.PHONY: tag-%
+
 load-%: %
-	kind load docker-image --name $(KIND) $(REGISTRY)/$<
+	kind load docker-image --name $(KIND) $(DEFAULT_REGISTRY)/$<
 .PHONY: load-%
 
 ############# assembly testing ################################################
 
+# build and tag local assembly image, as it's hard-coded in the vineyard operator templates
+build-assembly-job: local-assembly tag-local-assembly load-local-assembly
+
 local-assembly-images: local-assembly assembly-job1 assembly-job2
 publish-local-assembly-images: push-local-assembly push-assembly-job1 push-assembly-job2
-kind-load-local-assembly-images: load-local-assembly load-assembly-job1 load-assembly-job2
+e2e-test-assembly: build-assembly-job push-assembly-job1 push-assembly-job2
 
 local-assembly:
 	docker build assembly-demo/ -f Dockerfile \
@@ -39,9 +50,12 @@ assembly-job2:
 
 ############# distributed assembly testing ####################################
 
+# build and tag distributed assembly image, as it's hard-coded in the vineyard operator templates
+build-distributed-assembly-job: distributed-assembly tag-distributed-assembly load-distributed-assembly
+
 distributed-assembly-images: distributed-assembly distributed-assembly-job1 distributed-assembly-job2 distributed-assembly-job3
 publish-distributed-assembly-images: push-distributed-assembly push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3
-kind-load-distributed-assembly-images: load-distributed-assembly load-distributed-assembly-job1 load-distributed-assembly-job2 load-distributed-assembly-job3
+e2e-test-distributed-assembly: build-distributed-assembly-job push-distributed-assembly-job1 push-distributed-assembly-job2 push-distributed-assembly-job3
 
 distributed-assembly:
 	docker build assembly-demo/ -f Dockerfile \
@@ -73,7 +87,7 @@ distributed-assembly-job3:
 
 serialize-images: serialize-job
 publish-serialize-images: push-serialize-job
-kind-load-serialize-images: load-serialize-job
+e2e-test-serialize: push-serialize-job
 
 serialize-job:
 	docker build . -f Dockerfile \
@@ -83,9 +97,14 @@ serialize-job:
 
 ############# failover testing ################################################
 
+# build and tag backup image, as it's hard-coded in the vineyard operator templates
+build-backup-job: backup-job tag-backup-job load-backup-job
+# build and tag recover image, as it's hard-coded in the vineyard operator templates
+build-recover-job: recover-job tag-recover-job load-recover-job
+
 failover-images: backup-job recover-job build-local-object build-distributed-object-step1 build-distributed-object-step2 get-local-object get-distributed-object
 publish-failover-images: push-backup-job push-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object
-kind-load-failover-images: load-backup-job load-recover-job load-build-local-object load-build-distributed-object-step1 load-build-distributed-object-step2 load-get-local-object load-get-distributed-object
+e2e-test-failover: build-backup-job build-recover-job push-build-local-object push-build-distributed-object-step1 push-build-distributed-object-step2 push-get-local-object push-get-distributed-object
 
 backup-job:
 	docker build failover-demo/ -f Dockerfile \
@@ -131,9 +150,12 @@ get-distributed-object:
 
 ############# repartition testing #############################################
 
+# build and tag dask repartition image, as it's hard-coded in the vineyard operator templates
+build-repartition-job: dask-repartition tag-dask-repartition load-dask-repartition
+
 repartition-images: dask-repartition dask-repartition-job1 dask-repartition-job2 dask-worker-with-vineyard
 publish-repartition-images: push-dask-repartition push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard
-kind-load-repartition-images: load-dask-repartition load-dask-repartition-job1 load-dask-repartition-job2 load-dask-worker-with-vineyard
+e2e-test-repartition: build-repartition-job push-dask-repartition-job1 push-dask-repartition-job2 push-dask-worker-with-vineyard
 
 dask-repartition:
 	docker build repartition-demo/ -f Dockerfile \
@@ -165,7 +187,7 @@ dask-worker-with-vineyard:
 
 sidecar-images: sidecar-job
 publish-sidecar-images: push-sidecar-job
-kind-load-sidecar-images: load-sidecar-job
+e2e-test-sidecar: push-sidecar-job
 
 sidecar-job:
 	docker build sidecar-demo/ -f Dockerfile \
@@ -177,7 +199,7 @@ sidecar-job:
 
 workflow-images: workflow-job1 workflow-job2
 publish-workflow-images: push-workflow-job1 push-workflow-job2
-kind-load-workflow-images: load-workflow-job1 load-workflow-job2
+e2e-test-workflow: push-workflow-job1 push-workflow-job2
 
 workflow-job1:
 	docker build workflow-demo/ -f Dockerfile \
@@ -190,3 +212,16 @@ workflow-job2:
 		--build-arg APP=job2.py \
 		-t $(REGISTRY)/workflow-job2:$(TAG)
 .PHONY: workflow-job2
+
+# publish all images to ghcr registry
+ALL-IMAGES ?= \
+publish-local-assembly-images \
+publish-distributed-assembly-images \
+publish-serialize-images \
+publish-failover-images \
+publish-repartition-images \
+publish-sidecar-images \
+publish-workflow-images
+
+publish-all-images: $(ALL-IMAGES)
+.PHONY: publish-all-images

k8s/test/e2e/assembly-demo/assembly-job.yaml

@@ -49,7 +49,7 @@ spec:
       {{- end }}
       containers:
         - name: {{ (datasource "config").$job.jobName }}
-          image: ghcr.io/v6d-io/v6d/{{ (datasource "config").$job.jobName }}
+          image: localhost:5001/{{ (datasource "config").$job.jobName }}
           env:
             - name: JOB_NAME
               value: {{ (datasource "config").$job.jobName }}

k8s/test/e2e/assembly/distributed-assembly-e2e.yaml

@@ -16,7 +16,7 @@
 # Test assembly operation in different node
 setup:
   env: kind
-  file: ../kind.yaml
+  kubeconfig: /tmp/e2e-k8s.config
   steps:
     - name: install cert-manager
       command: |
@@ -25,16 +25,19 @@ setup:
         - namespace: cert-manager
           resource: pod
           for: condition=Ready
-    - name: install scheduler-plugin and operator
+    - name: install vineyard operator
       command: |
-        kind load docker-image vineyardcloudnative/vineyard-operator:latest
+        docker tag vineyardcloudnative/vineyard-operator:latest localhost:5001/vineyard-operator:latest
+        docker push localhost:5001/vineyard-operator:latest
+        export IMG=localhost:5001/vineyard-operator:latest
         make -C k8s deploy
-
-        make -C k8s/test/e2e kind-load-distributed-assembly-images
       wait:
         - namespace: vineyard-system
           resource: deployment/vineyard-controller-manager
           for: condition=Available
+    - name: download all distributed assembly images into kind cluster
+      command: |
+        make -C k8s/test/e2e e2e-test-distributed-assembly REGISTRY=localhost:5001
     - name: install vineyardd
       command: |
         kubectl apply -f k8s/test/e2e/vineyardd.yaml