Rollup merge of #124327 - Kobzol:ci-skip-jobs, r=pietroalbini

matthiaskrgr · web-flow · commit 036bf570adac · 2024-04-25T06:31:04.000+02:00
CI: implement job skipping in Python matrix calculation

This removes the `step` YAML anchor and the corresponding bash script.

Best reviewed commit-by-commit.

r? ```@pietroalbini```
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -88,96 +88,70 @@ jobs:
         run: "echo \"[CI_PR_NUMBER=$num]\""
         env:
           num: "${{ github.event.number }}"
-        if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
+        if: "success() && github.event_name == 'pull_request'"
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
         env:
           EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
-        if: success() && !env.SKIP_JOB
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
-        if: success() && !env.SKIP_JOB
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
-        if: success() && !env.SKIP_JOB
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
-        if: success() && !env.SKIP_JOB
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
-        if: success() && !env.SKIP_JOB
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
-        if: success() && !env.SKIP_JOB
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
-        if: success() && !env.SKIP_JOB
       - name: install clang
         run: src/ci/scripts/install-clang.sh
-        if: success() && !env.SKIP_JOB
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
-        if: success() && !env.SKIP_JOB
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
-        if: success() && !env.SKIP_JOB
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
-        if: success() && !env.SKIP_JOB
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
-        if: success() && !env.SKIP_JOB
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
-        if: success() && !env.SKIP_JOB
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
-        if: success() && !env.SKIP_JOB
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
-        if: success() && !env.SKIP_JOB
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
-        if: success() && !env.SKIP_JOB
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
-        if: success() && !env.SKIP_JOB
       - name: run the build
         run: src/ci/scripts/run-build-from-ci.sh 2>&1
         env:
           AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
-        if: success() && !env.SKIP_JOB
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
         with:
           name: "${{ env.DOC_ARTIFACT_NAME }}"
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
-        if: success() && !env.SKIP_JOB
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
           AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
-        if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
+        if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
   master:
     name: master
     runs-on: ubuntu-latest
@@ -202,7 +176,6 @@ jobs:
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
   try-success:
     needs:
       - job
diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
@@ -17,10 +17,13 @@
 
 import yaml
 
+CI_DIR = Path(__file__).absolute().parent.parent
 JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
+Job = Dict[str, Any]
 
-def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]:
     """
     Add a `name` attribute to each job, based on its image and the given `prefix`.
     """
@@ -29,7 +32,7 @@ def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
     return jobs
 
 
-def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
+def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]:
     """
     Prepends `environment` to the `env` attribute of each job.
     The `env` of each job has higher precedence than `environment`.
@@ -77,7 +80,7 @@ def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
     return None
 
 
-def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Job]:
     if job_type == JobType.PR:
         return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
     elif job_type == JobType.Try:
@@ -88,6 +91,13 @@ def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str
     return []
 
 
+def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]:
+    """
+    Skip CI jobs that are not supposed to be executed on the given `channel`.
+    """
+    return [j for j in jobs if j.get("only_on_channel", channel) == channel]
+
+
 def get_github_ctx() -> GitHubCtx:
     return GitHubCtx(
         event_name=os.environ["GITHUB_EVENT_NAME"],
@@ -107,9 +117,13 @@ def get_github_ctx() -> GitHubCtx:
     job_type = find_job_type(github_ctx)
     logging.info(f"Job type: {job_type}")
 
+    with open(CI_DIR / "channel") as f:
+        channel = f.read().strip()
+
     jobs = []
     if job_type is not None:
         jobs = calculate_jobs(job_type, data)
+    jobs = skip_jobs(jobs, channel)
 
     logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
     print(f"jobs={json.dumps(jobs)}")
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
@@ -107,9 +107,6 @@ x--expand-yaml-anchors--remove:
   - &job-aarch64-linux
     os: [self-hosted, ARM64, linux]
 
-  - &step
-    if: success() && !env.SKIP_JOB
-
   - &base-ci-job
     defaults:
       run:
@@ -151,7 +148,7 @@ x--expand-yaml-anchors--remove:
         run: echo "[CI_PR_NUMBER=$num]"
         env:
           num: ${{ github.event.number }}
-        if: success() && !env.SKIP_JOB && github.event_name == 'pull_request'
+        if: success() && github.event_name == 'pull_request'
 
       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
@@ -161,71 +158,51 @@ x--expand-yaml-anchors--remove:
           # are passed to the `setup-environment.sh` script encoded in JSON,
           # which then uses log commands to actually set them.
           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
-        <<: *step
-
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        <<: *step
 
       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh
-        <<: *step
 
       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh
-        <<: *step
 
       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh
-        <<: *step
 
       - name: install awscli
         run: src/ci/scripts/install-awscli.sh
-        <<: *step
 
       - name: install sccache
         run: src/ci/scripts/install-sccache.sh
-        <<: *step
 
       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh
-        <<: *step
 
       - name: install clang
         run: src/ci/scripts/install-clang.sh
-        <<: *step
 
       - name: install tidy
         run: src/ci/scripts/install-tidy.sh
-        <<: *step
 
       - name: install WIX
         run: src/ci/scripts/install-wix.sh
-        <<: *step
 
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh
-        <<: *step
 
       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh
-        <<: *step
 
       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh
-        <<: *step
 
       - name: install ninja
         run: src/ci/scripts/install-ninja.sh
-        <<: *step
 
       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh
-        <<: *step
 
       # Disable automatic line ending conversion (again). On Windows, when we're
       # installing dependencies, something switches the git configuration directory or
@@ -234,19 +211,15 @@ x--expand-yaml-anchors--remove:
       # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh
-        <<: *step
 
       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh
-        <<: *step
 
       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh
-        <<: *step
 
       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh
-        <<: *step
 
       - name: run the build
         # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
@@ -255,11 +228,9 @@ x--expand-yaml-anchors--remove:
           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh
-        <<: *step
 
       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
@@ -269,7 +240,6 @@ x--expand-yaml-anchors--remove:
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5
-        <<: *step
 
       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
@@ -281,8 +251,7 @@ x--expand-yaml-anchors--remove:
         # adding the condition is helpful as this way CI will not silently skip
         # deploying artifacts from a dist builder if the variables are misconfigured,
         # erroring about invalid credentials instead.
-        if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
-        <<: *step
+        if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
 
   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
   # Check out their documentation for more information on why they're needed.
@@ -399,7 +368,6 @@ jobs:
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
-        <<: *step
 
   # These jobs don't actually test anything, but they're used to tell bors the
   # build completed, as there is no practical way to detect when a workflow is
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
@@ -79,7 +79,7 @@ pr:
     <<: *job-linux-16c
 
 # Jobs that run when you perform a try build (@bors try)
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.try, to avoid repeating
 # it in each job definition.
 try:
   - image: dist-x86_64-linux
@@ -88,7 +88,7 @@ try:
     <<: *job-linux-16c
 
 # Main CI jobs that have to be green to merge a commit into master
-# These jobs automatically inherit envs.production, to avoid repeating
+# These jobs automatically inherit envs.auto, to avoid repeating
 # it in each job definition.
 auto:
   #############################
@@ -200,24 +200,23 @@ auto:
   # channel name on the output), and this builder prevents landing
   # changes that would result in broken builds after a promotion.
   - image: x86_64-gnu-stable
+    # Only run this job on the nightly channel. Running this on beta
+    # could cause failures when `dev: 1` in `stage0.txt`, and running
+    # this on stable is useless.
+    only_on_channel: nightly
     env:
       IMAGE: x86_64-gnu
       RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-      # Only run this job on the nightly channel. Running this on beta
-      # could cause failures when `dev: 1` in `stage0.txt`, and running
-      # this on stable is useless.
-      CI_ONLY_WHEN_CHANNEL: nightly
     <<: *job-linux-4c
 
   - image: x86_64-gnu-aux
     <<: *job-linux-4c
 
   - image: x86_64-gnu-integration
-    env:
-      # Only run this job on the nightly channel. Fuchsia requires
-      # nightly features to compile, and this job would fail if
-      # executed on beta and stable.
-      CI_ONLY_WHEN_CHANNEL: nightly
+    # Only run this job on the nightly channel. Fuchsia requires
+    # nightly features to compile, and this job would fail if
+    # executed on beta and stable.
+    only_on_channel: nightly
     <<: *job-linux-8c
 
   - image: x86_64-gnu-debug
diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh
