Export coverage, merge unit-inttests, calicovpp/ctl image

This patch adds coverage exporting for boht unit and integration (with VPP)
tests.
- It merges 'unit' and 'integration' tests into a single pipeline, always
launching VPP if passed the VPP_BINARY envvar
- It adds a signal handler for USR2 to export coverage if built as such
- It removes the WITH_DGB option and default to always packaging gdb
with VPP.
- It adds a calicovpp/ctl image to the build pipeline

Signed-off-by: Nathan Skrzypczak <[email protected]>

Author: sknat

.ci/buildspec.yml

@@ -29,18 +29,16 @@ phases:
 
   build:
     commands:
-      - make go-check
-      - make mdlint
-
-      - make -C vpp-manager clean-vpp
+      - make clean-vpp
       - make images
-      - docker images
 
-      - make run-integration-tests
+      - make mdlint
+      - make ci-lint
+      - make ci-test
+      - make ci-cov
+
       - make push
 
   post_build:
     commands:
-      - echo "Build complete"
-
       - echo Build completed on `date`

Dockerfile.depend

@@ -9,7 +9,7 @@ ENV UNATTENDED=y
 
 RUN apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
-    apt-utils wget cmake curl git
+    apt-utils wget cmake curl git docker.io
 
 ENV GOVERSION=1.24.0
 ENV GOROOT="/root/.go"

Makefile

@@ -8,15 +8,15 @@ build:
 	$(MAKE) -C calico-vpp-agent $@
 	$(MAKE) -C vpp-manager $@
 	$(MAKE) -C multinet-monitor $@
-	@mkdir -p cmd/bin
-	go build -o cmd/bin/calicovppctl ./cmd/calicovppctl/main.go
+	$(MAKE) -C cmd/calicovppctl $@
 
 .PHONY: image images
 images: image
 image:
 	$(MAKE) -C calico-vpp-agent $@
 	$(MAKE) -C vpp-manager $@
 	$(MAKE) -C multinet-monitor $@
+	$(MAKE) -C cmd/calicovppctl $@
 
 .PHONY: image-kind
 image-kind: image
@@ -49,13 +49,18 @@ push:
 	$(MAKE) -C calico-vpp-agent $@
 	$(MAKE) -C vpp-manager $@
 	$(MAKE) -C multinet-monitor $@
+	$(MAKE) -C cmd/calicovppctl $@
 
 .PHONY: dev
 dev:
 	$(MAKE) -C calico-vpp-agent ALSO_LATEST=y $@
 	$(MAKE) -C vpp-manager ALSO_LATEST=y $@
 	$(MAKE) -C multinet-monitor ALSO_LATEST=y $@
 
+.PHONY: clean-vpp
+clean-vpp:
+	$(MAKE) -C vpp-manager clean-vpp
+
 .PHONY: proto
 proto:
 	$(MAKE) -C calico-vpp-agent $@
@@ -284,22 +289,14 @@ lint:
 	test -d vpp-manager/vpp_build && touch vpp-manager/vpp_build/go.mod || true
 	golangci-lint run --color=never
 
-.PHONY: test
-test: go-lint
-	rm -rf .coverage/unit
-	mkdir -p .coverage/unit
-	go test -cover -covermode=atomic ./...  \
-		-args \
-		-test.gocoverdir=$(shell pwd)/.coverage/unit
-
 .PHONY: cov-html
 cov-html:
 	go tool covdata percent -i=.coverage/unit
 	go tool covdata textfmt -i=.coverage/unit -o .coverage/profile
 	go tool cover -html=.coverage/profile
 
 .PHONY: cov
-cov: test
+cov:
 	go tool covdata percent -i=.coverage/unit
 	@go tool covdata textfmt -i=.coverage/unit -o .coverage/profile
 	@echo "TOTAL:"
@@ -347,16 +344,55 @@ builder-image: ## Make dependencies image. (Not required normally; is implied in
 		   && ${PUSH_IMAGE} )
 	docker tag ${DEPEND_IMAGE} ${DEPEND_BASE}:latest
 
-.PHONY: go-check
-go-check: builder-image
-	@echo Checking go code
+# make test - runs the unit & VPP-integration tests
+# requiring sudo, this is useful in dev as this caches go deps.
+# Altought this requires go, sudo installed
+.PHONY: test
+test: builder-image
+	@rm -rf $(shell pwd)/.coverage/unit
+	@mkdir -p $(shell pwd)/.coverage/unit
+	$(MAKE) -C vpp-manager image
+	$(MAKE) -C vpp-manager mock-pod-image
+	sudo -E env "PATH=$$PATH" VPP_BINARY=/usr/bin/vpp \
+		VPP_IMAGE=calicovpp/vpp:$(TAG) \
+		go test ./... \
+		-cover \
+		-covermode=atomic \
+		-test.v \
+		-test.gocoverdir=$(shell pwd)/.coverage/unit \
+
+# make ci-test - runs the unit & VPP-integration tests
+# within a container, with mounted /var/run/docker.sock
+# this is portable, but lack go cache
+.PHONY: ci-test
+ci-test: builder-image
+	@rm -rf $(shell pwd)/.coverage/unit
+	@mkdir -p $(shell pwd)/.coverage/unit
+	$(MAKE) -C vpp-manager image
+	$(MAKE) -C vpp-manager mock-pod-image
 	docker run -t --rm \
+		--privileged \
+		--pid=host \
+		-v /proc:/proc \
 		-v $(CURDIR):/vpp-dataplane \
+		-v /tmp/cni-tests-vpp:/tmp/cni-tests-vpp \
+		-v /var/run/docker.sock:/var/run/docker.sock \
+		--env VPP_BINARY=/usr/bin/vpp \
+		--env VPP_IMAGE=calicovpp/vpp:$(TAG) \
+		-w /vpp-dataplane \
 		${DEPEND_IMAGE} \
-		make -C /vpp-dataplane cov
-
-.PHONY: go-lint
-go-lint: lint
+		go test ./... \
+			-cover \
+			-covermode=atomic \
+			-test.v \
+			-test.gocoverdir=/vpp-dataplane/.coverage/unit
+
+.PHONY: ci-%
+ci-%: builder-image
+	docker run -t --rm \
+		-v $(CURDIR):/vpp-dataplane \
+		${DEPEND_IMAGE} \
+		make -C /vpp-dataplane $*
 
 .PHONY: depend-image-hash
 depend-image-hash:

calico-vpp-agent/Makefile

@@ -1,28 +1,24 @@
 include ../common.mk
 
-.PHONY: all build gobgp image push proto
-
 TAG ?= latest # Tag images with :$(TAG)
 ALSO_LATEST ?= n # If 'y' also tag images with :latest
 GENERATE_LOG_FILE=../vpplink/generated/generate.log
 VERSION_FILE=version
 
-all: build gobgp image
+.PHONY: all
+all: image
 
 bin:
 	mkdir -p bin
 
-.PHONY: felix-api-proxy
-felix-api-proxy: bin
-	${DOCKER_RUN} go build -o ./bin/felix-api-proxy ./cmd/api-proxy
-
-build: felix-api-proxy bin
-	${DOCKER_RUN} go build -o ./bin/calico-vpp-agent ./cmd
-
-gobgp: bin
+.PHONY: build
+build: bin
+	${DOCKER_RUN} go build $(COVER_OPTS) -o ./bin/felix-api-proxy ./cmd/api-proxy
+	${DOCKER_RUN} go build $(COVER_OPTS) -o ./bin/calico-vpp-agent ./cmd
 	${DOCKER_RUN} go build -o ./bin/gobgp github.com/osrg/gobgp/v3/cmd/gobgp/
 
-image: build gobgp
+.PHONY: image
+image: build
 	@echo "Image tag                   : $(TAG)"                         > $(VERSION_FILE)
 	@echo "VPP-dataplane version       : $(shell git log -1 --oneline)" >> $(VERSION_FILE)
 	@cat $(GENERATE_LOG_FILE)                                           >> $(VERSION_FILE)
@@ -32,6 +28,14 @@ image: build gobgp
 		docker tag calicovpp/agent:$(TAG) calicovpp/agent:prerelease; \
 	fi
 
+.PHONY: image-cov
+image-cov: build-cov
+	@echo "Image tag                   : $(TAG)"                         > $(VERSION_FILE)
+	@echo "VPP-dataplane version       : $(shell git log -1 --oneline)" >> $(VERSION_FILE)
+	@cat $(GENERATE_LOG_FILE)                                           >> $(VERSION_FILE)
+	docker build --pull -t calicovpp/agent:$(TAG) .
+
+.PHONY: push
 push: ${PUSH_DEP}
 	set -e; for registry in ${REGISTRIES}; do \
 		docker tag calicovpp/agent:$(TAG) $${registry}calicovpp/agent:$(TAG); \
@@ -42,7 +46,9 @@ push: ${PUSH_DEP}
 		docker push --all-tags $${registry}calicovpp/agent; \
 	done
 
+.PHONY: dev
 dev: image
 
+.PHONY: proto
 proto:
 	$(MAKE) -C proto $@

calico-vpp-agent/cmd/calico_vpp_dataplane.go

@@ -228,6 +228,10 @@ func main() {
 		syscall.SIGUSR2,
 	)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	config.HandleUsr2Signal(ctx, log.WithFields(logrus.Fields{"component": "sighdlr"}))
+
 	select {
 	case sig := <-sigChan:
 		switch sig {

calico-vpp-agent/cni/cni_node_test.go

@@ -47,16 +47,15 @@ import (
 
 // Names of integration tests arguments
 const (
-	IntegrationTestEnableArgName = "INTEGRATION_TEST"
-	VppImageArgName              = "VPP_IMAGE"
-	VppBinaryArgName             = "VPP_BINARY"
-	VppContainerExtraArgsName    = "VPP_CONTAINER_EXTRA_ARGS"
+	VppImageArgName           = "VPP_IMAGE"
+	VppBinaryArgName          = "VPP_BINARY"
+	VppContainerExtraArgsName = "VPP_CONTAINER_EXTRA_ARGS"
 )
 
 // TestCniIntegration runs all the ginkgo integration test inside CNI package
 func TestCniIntegration(t *testing.T) {
 	// skip test if test run is not integration test run (prevent accidental run of integration tests using go test ./...)
-	_, isIntegrationTestRun := os.LookupEnv(IntegrationTestEnableArgName)
+	_, isIntegrationTestRun := os.LookupEnv(VppImageArgName)
 	if !isIntegrationTestRun {
 		t.Skip("skipping CNI integration tests (set INTEGRATION_TEST env variable to run these tests)")
 	}

cmd/calicovppctl/Dockerfile

@@ -0,0 +1,20 @@
+FROM --platform=${BUILDPLATFORM} golang:1.24 AS builder
+
+WORKDIR /src
+COPY go.mod go.sum /src
+RUN go mod download
+
+COPY *.go /src
+
+ARG TARGETOS
+ARG TARGETARCH
+
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+	go build \
+    -ldflags="-s -w" -o /cmd/calicovppctl
+
+FROM scratch
+WORKDIR /
+COPY --from=builder /cmd/calicovppctl /calicovppctl
+
+CMD ["./calicovppctl"]

cmd/calicovppctl/Makefile

@@ -0,0 +1,35 @@
+include ../../common.mk
+
+BIN_DIR = $(VPP_DATAPLANE_DIR)/bin
+
+.PHONY: bin
+bin:
+	@mkdir -p $(BIN_DIR)
+
+.PHONY: build
+build: bin
+	${DOCKER_RUN} go build -o $(BIN_DIR)/calicovppctl .
+
+.PHONY: image
+image:
+ifndef CI_BUILD
+	docker build --network=host --build-arg http_proxy=${DOCKER_BUILD_PROXY} \
+		--platform linux/arm64,linux/amd64 \
+		-t calicovpp/ctl:$(TAG) .
+endif
+
+.PHONY: push
+push: ${PUSH_DEP}
+	docker buildx create \
+		--name vppctlbuilder \
+		--driver docker-container \
+		--use
+	docker buildx build \
+		--network=host \
+		--build-arg http_proxy=${DOCKER_BUILD_PROXY} \
+		--platform linux/arm64,linux/amd64 \
+		--push \
+		$(foreach registry,$(REGISTRIES),-t $(registry)calicovpp/ctl:$(TAG)) \
+		$(if $(ALSO_LATEST),$(foreach registry,$(REGISTRIES),-t $(registry)calicovpp/ctl:latest)) \
+		.
+	docker buildx rm vppctlbuilder

cmd/calicovppctl/go.mod

@@ -0,0 +1,55 @@
+module github.com/projectcalico/vpp-dataplane/cmd/calicovppctl/v3
+
+go 1.24.4
+
+require (
+	github.com/gookit/color v1.5.4
+	github.com/onsi/gomega v1.36.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	google.golang.org/protobuf v1.36.5 // indirect
+	k8s.io/api v0.32.5
+	k8s.io/apimachinery v0.32.5
+	k8s.io/client-go v0.32.5
+	sigs.k8s.io/yaml v1.4.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/ginkgo/v2 v2.22.2 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
+	golang.org/x/oauth2 v0.28.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
+)