Merge pull request #1307 from o1-labs/dw/improve-ci-slightly

CI: consolidate workflows and improve caching strategy

Author: dannywillems

.github/actions/setup-build-deps/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: 'Include SQLite3 in the installation'
     required: false
     default: 'false'
+  install-nextest:
+    description: 'Install cargo-nextest for faster test execution'
+    required: false
+    default: 'false'
 runs:
   using: 'composite'
   steps:
@@ -23,4 +27,10 @@ runs:
       if: runner.os == 'macOS'
       shell: bash
       run: |
-        brew install protobuf
+        brew install protobuf
+
+    - name: Install cargo-nextest
+      if: inputs.install-nextest == 'true'
+      uses: taiki-e/install-action@v2
+      with:
+        tool: nextest

.github/actions/setup-rust/action.yml

@@ -30,4 +30,6 @@ runs:
       if: inputs.enable-cache == 'true'
       uses: Swatinem/rust-cache@v2
       with:
-        prefix-key: ${{ inputs.cache-prefix }}
+        prefix-key: ${{ inputs.cache-prefix }}-${{ inputs.toolchain }}
+        shared-key: shared-${{ inputs.toolchain }}
+        save-if: ${{ github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' }}

.github/workflows/ci.yaml

@@ -5,17 +5,38 @@ on:
   pull_request:
     paths-ignore: [ "frontend" ]
   workflow_dispatch:
+    inputs:
+      refresh_cache:
+        description: 'Refresh cargo cache'
+        required: false
+        type: boolean
+        default: false
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
   OPENMINA_PANIC_ON_BUG: true
+  CARGO_INCREMENTAL: 1
+  RUSTFLAGS: "-C overflow-checks=off -C debug-assertions=off"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
+  refresh-cache:
+    if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.refresh_cache == 'true' }}
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v5
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          toolchain: 1.84
+          enable-cache: false
+      - name: Clean cargo cache
+        run: cargo clean
+
   ledger-tests:
     timeout-minutes: 20
     runs-on: ubuntu-24.04
@@ -194,13 +215,7 @@ jobs:
           cache-prefix: build-tests-${{ matrix.os }}-v0
 
       - name: Build tests
-        run: |
-          mkdir -p target/release/tests
-
-          cargo build --release --tests --package=openmina-node-testing --package=cli
-          cargo build --release --tests --package=openmina-node-testing --package=cli --message-format=json > cargo-build-test.json
-          jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' cargo-build-test.json > tests.tsv
-          while read NAME FILE; do cp -a $FILE target/release/tests/$NAME; done < tests.tsv
+        run: make build-tests
 
       - name: Upload tests
         if: matrix.os == 'ubuntu-22.04'

.github/workflows/doc-commands.yml renamed to .github/workflows/doc-commands.yaml

@@ -8,13 +8,15 @@ on:
 jobs:
   format:
     name: Format using nightly
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@stable
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
         with:
           toolchain: nightly
           components: rustfmt
+          cache-prefix: format-nightly-v0
       - uses: taiki-e/install-action@v2
         with:
           tool: taplo

.github/workflows/fmt.yaml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   frontend-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Git checkout
         uses: actions/checkout@v5

.github/workflows/frontend.yaml

@@ -12,21 +12,23 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-24.04]
         toolchain: [1.84]
     steps:
       - uses: actions/checkout@v5
       - name: Setup build dependencies
-        run: |
-          sudo apt update || true
-          sudo apt install -y protobuf-compiler sqlite3 || true
+        uses: ./.github/actions/setup-build-deps
+        with:
+          include-sqlite: true
       - name: Setup SQLite database for SQLx
         run: |
           sqlite3 /tmp/heartbeats.db < tools/heartbeats-processor/schema.sql
-      - uses: dtolnay/rust-toolchain@stable
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
         with:
           toolchain: ${{ matrix.toolchain }}
           components: clippy, rustfmt
+          cache-prefix: lint-${{ matrix.toolchain }}-v0
       - name: Run make check
         run: make check
         env:
@@ -42,18 +44,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-24.04]
         toolchain: [nightly]
     steps:
       - uses: actions/checkout@v5
       - name: Setup build dependencies
-        run: |
-          sudo apt update || true
-          sudo apt install -y protobuf-compiler || true
-      - uses: dtolnay/rust-toolchain@stable
+        uses: ./.github/actions/setup-build-deps
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
         with:
           toolchain: ${{ matrix.toolchain }}
           components: clippy, rustfmt
+          cache-prefix: lint-tx-fuzzing-${{ matrix.toolchain }}-v0
       - name: Run transaction Fuzzing check
         run: make check-tx-fuzzing
 
@@ -63,7 +65,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-24.04]
     steps:
       - uses: actions/checkout@v5
       - name: Install shellcheck
@@ -79,7 +81,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-24.04]
     steps:
       - uses: actions/checkout@v5
       - name: Install hadolint

.github/workflows/lint.yml renamed to .github/workflows/lint.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   markdown-format:
     name: Check Markdown Formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v5

.github/workflows/markdown-format.yml renamed to .github/workflows/markdown-format.yaml

@@ -45,6 +45,22 @@ build-release: ## Build the project in release mode
 build-testing: ## Build the testing binary with scenario generators
 	cargo build --release --features scenario-generators --bin openmina-node-testing
 
+.PHONY: build-tests
+build-tests: ## Build tests for scenario testing
+	@mkdir -p target/release/tests
+	@cargo build --release --tests \
+		--package=openmina-node-testing \
+		--package=cli
+	@cargo build --release --tests \
+		--package=openmina-node-testing \
+		--package=cli \
+		--message-format=json > cargo-build-test.json
+	@jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' \
+		cargo-build-test.json > tests.tsv
+	@while read NAME FILE; do \
+		cp -a $$FILE target/release/tests/$$NAME; \
+	done < tests.tsv
+
 .PHONY: build-tests-webrtc
 build-tests-webrtc: ## Build tests for WebRTC
 	@mkdir -p target/release/tests
@@ -224,7 +240,7 @@ test-ledger: build-ledger ## Run ledger tests in release mode, requires nightly
 
 .PHONY: test-p2p
 test-p2p: ## Run P2P tests
-	cargo test -p p2p --tests
+	cargo test -p p2p --tests --release
 
 .PHONY: test-release
 test-release: ## Run tests in release mode
@@ -234,6 +250,26 @@ test-release: ## Run tests in release mode
 test-vrf: ## Run VRF tests, requires nightly Rust
 	@cd vrf && cargo +nightly test --release -- -Z unstable-options --report-time
 
+.PHONY: nextest
+nextest: ## Run tests with cargo-nextest for faster execution
+	@cargo nextest run
+
+.PHONY: nextest-release
+nextest-release: ## Run tests in release mode with cargo-nextest
+	@cargo nextest run --release
+
+.PHONY: nextest-p2p
+nextest-p2p: ## Run P2P tests with cargo-nextest
+	@cargo nextest run -p p2p --tests
+
+.PHONY: nextest-ledger
+nextest-ledger: build-ledger ## Run ledger tests with cargo-nextest, requires nightly Rust
+	@cd ledger && cargo +nightly nextest run --release
+
+.PHONY: nextest-vrf
+nextest-vrf: ## Run VRF tests with cargo-nextest, requires nightly Rust
+	@cd vrf && cargo +nightly nextest run --release
+
 # Docker build targets
 
 .PHONY: docker-build-all