Merge pull request #2489 from iwankgb/perf_integration_tests

Integration test for Docker and perf stats

Author: dashpole

.github/workflows/test.yml

@@ -26,6 +26,7 @@ jobs:
       matrix:
         go-versions: [1.13.x, 1.14.x]
         platform: [ubuntu-latest]
+        environment-variables: [build/config/plain.sh, build/config/libpfm4.sh]
     runs-on: ${{ matrix.platform }}
     timeout-minutes: 30
     steps:
@@ -40,4 +41,11 @@ jobs:
           path: go/src/github.com/google/cadvisor
       - name: Run integration tests
         run: |
-          cd $GITHUB_WORKSPACE/go/src/github.com/google/cadvisor && make docker-test-integration
+          cd $GITHUB_WORKSPACE/go/src/github.com/google/cadvisor && source ${{ matrix.environment-variables }} && make docker-test-integration
+      - name: Upload cAdvisor log file
+        uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: cadvisor.log
+          path: ${{ github.workspace }}/go/src/github.com/google/cadvisor/cadvisor.log
+

build/config/libpfm4.sh

@@ -0,0 +1,18 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export GO_FLAGS="-tags=libpfm,netgo -race"
+export PACKAGES="sudo libpfm4"
+export BUILD_PACKAGES="libpfm4 libpfm4-dev"
+export CADVISOR_ARGS="-perf_events_config=perf/testing/perf-non-hardware.json"

build/config/plain.sh

@@ -0,0 +1,15 @@
+# Copyright 2020 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Nothing happens here, this is just default config.

build/integration-in-docker.sh

@@ -19,35 +19,50 @@ set -e
 ROOT="$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)"
 TMPDIR=$(mktemp -d)
 function delete() {
-    echo "Deleting ${TMPDIR}..."
-    if [[ $EUID -ne 0 ]]; then
-      sudo rm -rf "${TMPDIR}"
-    else
-      rm -rf "${TMPDIR}"
-    fi
+  echo "Deleting ${TMPDIR}..."
+  if [[ $EUID -ne 0 ]]; then
+    sudo rm -rf "${TMPDIR}"
+  else
+    rm -rf "${TMPDIR}"
+  fi
 }
 trap delete EXIT INT
 
-docker run --rm \
-  -w /go/src/github.com/google/cadvisor \
-  -v ${PWD}:/go/src/github.com/google/cadvisor \
-  golang:1.13 \
-  bash -c "env GOOS=linux GO_FLAGS='-race' ./build/build.sh amd64 && \
-    env GOOS=linux go test -c github.com/google/cadvisor/integration/tests/api &&  \
-    env GOOS=linux go test -c github.com/google/cadvisor/integration/tests/healthz"
-
-EXTRA_DOCKER_OPTS="-e DOCKER_IN_DOCKER_ENABLED=true"
-if [[ "${OSTYPE}" == "linux"* ]]; then
-  EXTRA_DOCKER_OPTS+=" -v ${TMPDIR}/docker-graph:/docker-graph"
-fi
-
-mkdir ${TMPDIR}/docker-graph
-docker run --rm \
-  -w /go/src/github.com/google/cadvisor \
-  -v ${ROOT}:/go/src/github.com/google/cadvisor \
-  ${EXTRA_DOCKER_OPTS} \
-  --privileged \
-  --entrypoint="" \
-  gcr.io/k8s-testimages/bootstrap \
-  bash -c "apt update && apt install sudo && \
-/usr/local/bin/runner.sh build/integration.sh"
+function run_tests() {
+  BUILD_CMD="env GOOS=linux GO_FLAGS='$GO_FLAGS' ./build/build.sh amd64 && \
+    env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/api && \
+    env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/healthz"
+
+  if [ "$BUILD_PACKAGES" != "" ]; then
+    BUILD_CMD="apt-get update && apt-get install $BUILD_PACKAGES && \
+    $BUILD_CMD"
+  fi
+  docker run --rm \
+    -w /go/src/github.com/google/cadvisor \
+    -v ${PWD}:/go/src/github.com/google/cadvisor \
+    golang:1.13 \
+    bash -c "$BUILD_CMD"
+
+  EXTRA_DOCKER_OPTS="-e DOCKER_IN_DOCKER_ENABLED=true"
+  if [[ "${OSTYPE}" == "linux"* ]]; then
+    EXTRA_DOCKER_OPTS+=" -v ${TMPDIR}/docker-graph:/docker-graph"
+  fi
+
+  mkdir ${TMPDIR}/docker-graph
+  docker run --rm \
+    -w /go/src/github.com/google/cadvisor \
+    -v ${ROOT}:/go/src/github.com/google/cadvisor \
+    ${EXTRA_DOCKER_OPTS} \
+    --privileged \
+    --cap-add="sys_admin" \
+    --entrypoint="" \
+    gcr.io/k8s-testimages/bootstrap \
+    bash -c "apt update && apt install $PACKAGES && \
+CADVISOR_ARGS="$CADVISOR_ARGS" /usr/local/bin/runner.sh build/integration.sh"
+}
+
+GO_FLAGS=${GO_FLAGS:-"-tags=netgo -race"}
+PACKAGES=${PACKAGES:-"sudo"}
+BUILD_PACKAGES=${BUILD_PACKAGES:-}
+CADVISOR_ARGS=${CADVISOR_ARGS:-}
+run_tests "$GO_FLAGS" "$PACKAGES" "$BUILD_PACKAGES" "$CADVISOR_ARGS"

build/integration.sh

@@ -26,7 +26,7 @@ printf "" # Refresh sudo credentials if necessary.
 function start {
   set +e  # We want to handle errors if cAdvisor crashes.
   echo ">> starting cAdvisor locally"
-  GORACE="halt_on_error=1" ./cadvisor --docker_env_metadata_whitelist=TEST_VAR --v=4 --logtostderr &> "$log_file"
+  GORACE="halt_on_error=1" ./cadvisor --docker_env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &> "$log_file"
   if [ $? != 0 ]; then
     echo "!! cAdvisor exited unexpectedly with Exit $?"
     kill $TEST_PID # cAdvisor crashed: abort testing.
@@ -56,5 +56,5 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do
 done
 
 echo ">> running integration tests against local cAdvisor"
-./api.test --vmodule=*=2
-./healthz.test --vmodule=*=2
+./api.test --vmodule=*=2 -test.v
+./healthz.test --vmodule=*=2 -test.v

docs/development/integration_testing.md

@@ -1,4 +1,26 @@
-# Integration Testing cAdvisor
+# Integration Testing cAdvisor 
+
+## Docker-based tests
+
+The cAdvisor integration tests are run per-pr using [Github Actions](https://help.github.com/en/actions). Workflow configuration can be found at [.github/workflows/test.yml](.github/workflows/test.yml). Tests are executed in Docker containers run on MS Azure virtual machines.
+
+To run them locally Docker must be installed on your machine. Following command allows you to execute default suite of integration tests:
+
+```
+make docker-test-integration
+```
+
+Build scripts take care of building cAdvisor and integration tests, and executing them against running cAdvisor process.
+
+In order to run non-default tests suites (e.g. such that rely on third-party C libraries) you must source one of the files available at [build/config](build/config), e.g.:
+
+```
+source build/config/libpfm4.sh && make docker-test-integration
+```
+
+All the necessary packages will be installed, build flags will be applied and additional parameters will be passed to cAdvisor automatically. Configuration is performed using shell environment variables.
+
+## VM-base tests (legacy)
 
 The cAdvisor integration tests are run per-pr using the [kubernetes node-e2e testing framework](https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-node-tests.md) on GCE instances.  To make use of this framework, complete the setup of GCP described in the node-e2e testing framework, clone `k8s.io/kubernetes`, and from that repository run:
 ```

integration/tests/api/perf_test.go

@@ -0,0 +1,57 @@
+// +build libpfm,cgo
+
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"testing"
+	"time"
+
+	v1 "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/integration/framework"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPerfEvents(t *testing.T) {
+	fm := framework.New(t)
+	defer fm.Cleanup()
+
+	containerID := fm.Docker().RunPause()
+	waitForContainerInfo(fm, containerID)
+
+	info, err := fm.Cadvisor().Client().DockerContainer(containerID, &v1.ContainerInfoRequest{
+		NumStats: 1,
+	})
+
+	assert.Nil(t, err)
+	assert.Len(t, info.Stats, 1)
+	assert.Greater(t, len(info.Stats[0].PerfStats), 0, "Length of info.Stats[0].PerfStats is not greater than zero")
+	for k, stat := range info.Stats[0].PerfStats {
+		//Everything beyond name is non-deterministic unfortunately.
+		assert.Contains(t, []string{"context-switches", "cpu-migrations-custom"}, stat.Name, "Wrong metric name for key %d: %#v", k, stat)
+	}
+}
+
+func waitForContainerInfo(fm framework.Framework, containerID string) {
+	framework.RetryForDuration(func() error {
+		_, err := fm.Cadvisor().Client().DockerContainer(containerID, &v1.ContainerInfoRequest{NumStats: 1})
+		if err != nil {
+			return err
+		}
+		return nil
+	}, 5*time.Second)
+}

perf/testing/perf-non-hardware.json

@@ -0,0 +1,15 @@
+{
+  "events": [
+    ["context-switches"],
+    ["cpu-migrations-custom"]
+  ],
+  "custom_events": [
+    {
+      "type": 1,
+      "config": [
+        "0x4"
+      ],
+      "name": "cpu-migrations-custom"
+    }
+  ]
+}