chore: improve test workflow and coverage reports

.github/workflows/unit-test.yml → .github/workflows/test.yml

@@ -1,14 +1,15 @@
-name: Unit Tests
+name: Test Packages
 on:
   push:
     branches: [main]
   pull_request:
+    branches: [main]
 
 permissions:
   contents: read
 
 jobs:
-  build-and-cache:
+  compile:
     strategy:
       fail-fast: false
     runs-on: ubuntu-latest
@@ -26,19 +27,33 @@ jobs:
           node-version: 18
       - name: Install
         run: npm ci
-      - name: Build
-        run: npm run compile
+      # Note: when pushing to main we want to test only what changed in
+      # last commit. Otherwise we want to test all changes from origin/main.
+      # So we set the right values for base and head commits depending
+      # on the type of event (PR or push)
+      # ref: https://nx.dev/ci/features/affected#configure-affected-on-ci
+      - name: Set base and head commits
+        run: |
+          if [ "${{github.event_name}}" == "push" ]; then
+            echo "NX_BASE=origin/main~1" >> "$GITHUB_ENV"
+            echo "NX_HEAD=origin/main" >> "$GITHUB_ENV"
+          else
+            echo "NX_BASE=origin/main" >> "$GITHUB_ENV"
+            echo "NX_HEAD=HEAD" >> "$GITHUB_ENV"
+          fi
+      - name: Compile (Delta)
+        run: npm run compile:ci:affected
       - name: Upload Build Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: tests-build-cache-${{ github.run_number }}
+          name: compile-cache-${{ github.run_number }}
           path: .nx
           include-hidden-files: true
           if-no-files-found: error
           retention-days: 1
 
   unit-test:
-    needs: build-and-cache
+    needs: compile
     strategy:
       fail-fast: false
       matrix:
@@ -172,6 +187,7 @@ jobs:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       NPM_CONFIG_UNSAFE_PERM: true
+      NODE_OPTIONS: --max-old-space-size=4096
     steps:
       - name: Checkout
         uses: actions/checkout@v5
@@ -185,56 +201,90 @@ jobs:
       - name: Download Build Artifacts
         uses: actions/download-artifact@v5
         with:
-          name: tests-build-cache-${{ github.run_number }}
+          name: compile-cache-${{ github.run_number }}
           path: .nx
-      - name: Build
-        run: npm run compile
-      - name: Unit tests (Full)
-        if: matrix.code-coverage
-        run: npm run test
-      - name: Unit tests (Delta)
-        if: ${{ !matrix.code-coverage }}
-        run: npm run test:ci:changed
-      - name: Report Coverage
-        if: ${{ matrix.code-coverage && !cancelled()}}
-        uses: codecov/codecov-action@v5
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      # Note: see comment in the compile job
+      - name: Set base and head commits
+        run: |
+          if [ "${{github.event_name}}" == "push" ]; then
+            echo "NX_BASE=origin/main~1" >> "$GITHUB_ENV"
+            echo "NX_HEAD=origin/main" >> "$GITHUB_ENV"
+          else
+            echo "NX_BASE=origin/main" >> "$GITHUB_ENV"
+            echo "NX_HEAD=HEAD" >> "$GITHUB_ENV"
+          fi
+      - name: Compile (Delta)
+        run: npm run compile:ci:affected
+      - name: Unit tests (Browser - Delta)
+        if: ${{ matrix.node == '22' }}
+        run: npm run test:browser:ci:affected
+      - name: Unit tests (Nodejs - Delta)
+        run: npm run test:ci:affected
+      - name: Test All Versions (Delta)
+        run: npm run test-all-versions:ci:affected
+      - name: Upload Test Artifacts
+        uses: actions/upload-artifact@v4
         with:
-          verbose: true
+          name: tests-coverage-cache-${{ github.run_number }}-${{ matrix.node }}
+          include-hidden-files: true
+          if-no-files-found: error
+          retention-days: 1
+          path: |
+            !node_modules
+            packages/*/.nyc_output/**
+            LICENSE
 
-  browser-test:
-    needs: build-and-cache
-    strategy:
-      fail-fast: false
-      matrix:
-        node: ["22"]
+  test-coverage-report:
     runs-on: ubuntu-latest
-    env:
-      NPM_CONFIG_UNSAFE_PERM: true
-      NODE_OPTIONS: --max-old-space-size=4096
+    needs: unit-test
     steps:
       - name: Checkout
         uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: ${{ matrix.node }}
+          node-version: 18
       - name: Install
         run: npm ci
-      - name: Download Build Artifacts
+      # NOTE: keep this in sync with the node versions from `unit-test` job
+      - name: Download Test Artifacts (18)
         uses: actions/download-artifact@v5
         with:
-          name: tests-build-cache-${{ github.run_number }}
-          path: .nx
-      - name: Build
-        run: npm run compile
-      - name: Unit tests
-        run: npm run test:browser
-      - name: Report Coverage
-        uses: codecov/codecov-action@v5
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          name: tests-coverage-cache-${{ github.run_number }}-18
+          path: .
+      - name: Download Test Artifacts (18.19.0)
+        uses: actions/download-artifact@v5
         with:
-          verbose: true
+          name: tests-coverage-cache-${{ github.run_number }}-18.19.0
+          path: .
+      - name: Download Test Artifacts (20)
+        uses: actions/download-artifact@v5
+        with:
+          name: tests-coverage-cache-${{ github.run_number }}-20
+          path: .
+      - name: Download Test Artifacts (20.6.0)
+        uses: actions/download-artifact@v5
+        with:
+          name: tests-coverage-cache-${{ github.run_number }}-20.6.0
+          path: .
+      - name: Download Test Artifacts (22)
+        uses: actions/download-artifact@v5
+        with:
+          name: tests-coverage-cache-${{ github.run_number }}-22
+          path: .
+      # Note: see comment in the compile job
+      - name: Set base and head commits
+        run: |
+          if [ "${{github.event_name}}" == "push" ]; then
+            echo "NX_BASE=origin/main~1" >> "$GITHUB_ENV"
+            echo "NX_HEAD=origin/main" >> "$GITHUB_ENV"
+          else
+            echo "NX_BASE=origin/main" >> "$GITHUB_ENV"
+            echo "NX_HEAD=HEAD" >> "$GITHUB_ENV"
+          fi
+      - name: Merge coverage (Delta)
+        run: npm run test-merge-coverage:ci:affected
+      - name: Report Coverage with Flags
+        run: node ./scripts/codecov-upload-flags.mjs ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.head.label }}
+

codecov.yml

@@ -1,8 +1,9 @@
 codecov:
   notify:
-    require_ci_to_pass: no
+    after_n_builds: 1
+  require_ci_to_pass: no
 comment:
-  layout: "header, changes, diff, files"
+  layout: "header, changes, diff, flags, files"
   behavior: default
 coverage:
   status:
@@ -13,3 +14,11 @@ coverage:
       default:
         target: auto
         threshold: 1%
+# Codecov action does not read this file to do the split of coverage
+# by flag. So there is no reason to have explicit flag definition.
+# We use automatinc flag management.
+# ref: https://about.codecov.io/blog/introducing-improved-flag-management-with-automatic-flags/
+# note: Uploads are made per package(flag) in ./scripts/codecov-upload-flags.mjs
+flag_management:
+  default_rules:
+    carryforward: true

package.json

@@ -13,13 +13,18 @@
     "clean": "nx run-many -t clean",
     "version:update": "nx run-many -t version:update",
     "compile": "nx run-many -t compile",
+    "compile:ci:affected": "nx affected -t compile",
     "test": "nx run-many -t test",
+    "test:ci:affected": "nx affected -t test",
     "test:browser": "nx run-many -t test:browser",
-    "test:ci:changed": "nx affected -t test --base=origin/main --head=HEAD",
-    "test-all-versions": "nx run-many -t test-all-versions",
+    "test:browser:ci:affected": "nx affected -t test:browser",
+    "test-all-versions": "nx run-many --parallel=false -t test-all-versions",
+    "test-all-versions:ci:affected": "nx affected --parallel=false -t test-all-versions",
+    "test-merge-coverage:ci:affected": "nx affected -t test-merge-coverage",
     "test-services:start": "docker compose -f ./test/docker-compose.yaml up -d --wait",
     "test-services:stop": "docker compose -f ./test/docker-compose.yaml down",
     "test:with-services-env": "cross-env NODE_OPTIONS='-r dotenv/config' DOTENV_CONFIG_PATH=./test/test-services.env npm test",
+    "test-all-versions:with-services-env": "cross-env NODE_OPTIONS='-r dotenv/config' DOTENV_CONFIG_PATH=./test/test-services.env npm run test-all-versions",
     "changelog": "lerna-changelog",
     "lint": "nx run-many -t lint && npm run lint:deps && npm run lint:readme && npm run lint:markdown && npm run lint:semconv-deps && npm run lint:examples",
     "lint:fix": "nx run-many -t lint:fix && npm run lint:markdown:fix",

packages/instrumentation-amqplib/package.json

@@ -38,10 +38,11 @@
     "lint:readme": "node ../../scripts/lint-readme.js",
     "prepublishOnly": "npm run compile",
     "tdd": "npm run test -- --watch-extensions ts --watch",
-    "test": "nyc mocha --require '@opentelemetry/contrib-test-utils' 'test/**/*.test.ts'",
+    "test": "nyc --no-clean  mocha --require '@opentelemetry/contrib-test-utils' 'test/**/*.test.ts'",
     "test:with-services-env": "cross-env NODE_OPTIONS='-r dotenv/config' DOTENV_CONFIG_PATH=../../test/test-services.env npm test",
     "test-all-versions": "tav",
     "test-all-versions:with-services-env": "cross-env NODE_OPTIONS='-r dotenv/config' DOTENV_CONFIG_PATH=../../test/test-services.env npm run test-all-versions",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
     "test-services:start": "cd ../.. && npm run test-services:start rabbitmq",
     "test-services:stop": "cd ../.. && npm run test-services:stop rabbitmq",
     "version:update": "node ../../scripts/version-update.js",

packages/instrumentation-aws-lambda/package.json

@@ -10,16 +10,17 @@
     "directory": "packages/instrumentation-aws-lambda"
   },
   "scripts": {
-    "test": "nyc mocha 'test/**/*.test.ts'",
-    "tdd": "npm run test -- --watch-extensions ts --watch",
     "clean": "rimraf build/*",
+    "compile": "tsc -p .",
+    "compile:with-dependencies": "nx run-many -t compile -p @opentelemetry/instrumentation-aws-lambda",
     "lint": "eslint . --ext=ts,js,mjs",
     "lint:fix": "eslint . --ext=ts,js,mjs --fix",
     "lint:readme": "node ../../scripts/lint-readme.js",
     "prepublishOnly": "npm run compile",
-    "version:update": "node ../../scripts/version-update.js",
-    "compile:with-dependencies": "nx run-many -t compile -p @opentelemetry/instrumentation-aws-lambda",
-    "compile": "tsc -p ."
+    "tdd": "npm run test -- --watch-extensions ts --watch",
+    "test": "nyc --no-clean mocha 'test/**/*.test.ts'",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
+    "version:update": "node ../../scripts/version-update.js"
   },
   "keywords": [
     "aws-lambda",

packages/instrumentation-bunyan/package.json

@@ -11,14 +11,15 @@
   },
   "scripts": {
     "clean": "rimraf build/*",
-    "compile:with-dependencies": "nx run-many -t compile -p @opentelemetry/instrumentation-bunyan",
     "compile": "tsc -p .",
+    "compile:with-dependencies": "nx run-many -t compile -p @opentelemetry/instrumentation-bunyan",
     "lint": "eslint . --ext=ts,js,mjs",
     "lint:fix": "eslint . --ext=ts,js,mjs --fix",
     "prepublishOnly": "npm run compile",
     "tdd": "npm run test -- --watch-extensions ts --watch",
-    "test": "nyc mocha 'test/**/*.test.ts'",
+    "test": "nyc --no-clean mocha 'test/**/*.test.ts'",
     "test-all-versions": "tav",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
     "version:update": "node ../../scripts/version-update.js"
   },
   "keywords": [

packages/instrumentation-cassandra-driver/package.json

@@ -17,10 +17,12 @@
     "lint:fix": "eslint . --ext=ts,js,mjs --fix",
     "lint:readme": "node ../../scripts/lint-readme.js",
     "prepublishOnly": "npm run compile",
+    "setup:dev": "nx run-many -t compile -p @opentelemetry/instrumentation-cassandra-driver",
     "tdd": "npm run test -- --watch-extensions ts --watch",
-    "test": "nyc mocha 'test/**/*.test.ts'",
+    "test": "nyc --no-clean mocha 'test/**/*.test.ts'",
     "test:with-services-env": "cross-env NODE_OPTIONS='-r dotenv/config' DOTENV_CONFIG_PATH=../../test/test-services.env npm test",
     "//todo": "echo \"add test-all-versions\"",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
     "test-services:start": "cd ../.. && npm run test-services:start cassandra",
     "test-services:stop": "cd ../.. && npm run test-services:stop cassandra",
     "version:update": "node ../../scripts/version-update.js"

packages/instrumentation-connect/package.json

@@ -17,7 +17,8 @@
     "lint:fix": "eslint . --ext=ts,js,mjs --fix",
     "lint:readme": "node ../../scripts/lint-readme.js",
     "prepublishOnly": "npm run compile",
-    "test": "nyc mocha 'test/**/*.test.ts'",
+    "test": "nyc --no-clean mocha 'test/**/*.test.ts'",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
     "version:update": "node ../../scripts/version-update.js",
     "watch": "tsc -w"
   },

packages/instrumentation-dataloader/package.json

@@ -17,8 +17,9 @@
     "lint:fix": "eslint . --ext=ts,js,mjs --fix",
     "prepublishOnly": "npm run compile",
     "tdd": "npm run test -- --watch-extensions ts --watch",
-    "test": "nyc mocha 'test/**/*.test.ts'",
+    "test": "nyc --no-clean mocha 'test/**/*.test.ts'",
     "test-all-versions": "tav",
+    "test-merge-coverage": "nyc merge .nyc_output coverage/coverage-final.json",
     "version:update": "node ../../scripts/version-update.js"
   },
   "keywords": [