ci: change s3 cache provider to optimize costs

.github/actions/aws_credentials/action.yaml

@@ -0,0 +1,43 @@
+---
+# This file contains configuration of aws credentials file.
+# Its primary use is to prepare a credentials file that will be used as a secrets mount when building Docker images.
+name: "aws_credentials"
+description: "Configure .aws/credentials"
+inputs:
+  access_key_id:
+    description: Access key ID
+    required: true
+  secret_access_key:
+    description: Secret access key
+    required: true
+  profile:
+    description: AWS profile to use
+    default: "default"
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      shell: bash
+      run: |
+        mkdir -p "${HOME}/.aws"
+        cat >> ${HOME}/.aws/credentials << EOF
+        [${{ inputs.profile }}]
+        aws_access_key_id=${{ inputs.access_key_id }}
+        aws_secret_access_key=${{ inputs.secret_access_key }}
+        EOF
+        chmod -R go-rwx ${HOME}/.aws
+
+    - name: Set env variables
+      shell: bash
+      run: |
+        # Exit on any error  
+        set -euo pipefail  
+        # Validate AWS_PROFILE is not empty  
+        if [ -z "${{ inputs.profile }}" ]; then  
+          echo "Error: AWS_PROFILE cannot be empty"
+          exit 1
+        fi
+        # Export variables        
+        echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV
+        echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV

.github/actions/aws_ecr_login/action.yaml

@@ -0,0 +1,42 @@
+---
+# Login to AWS ECR
+name: "aws_ecr_login"
+description: "Login to AWS ECR to store Docker containers"
+inputs:
+  aws_account_id:
+    description: AWS account ID (AWS_ACCOUNT_ID)
+    required: true
+  aws_access_key_id:
+    description: Access key ID (AWS_ACCESS_KEY_ID)
+    required: true
+  aws_secret_access_key:
+    description: Secret access key (AWS_SECRET_ACCESS_KEY)
+    required: true
+  aws_region:
+    description: AWS region to use (AWS_REGION)
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials and bucket region
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws_access_key_id }}
+        aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - name: Login to ECR
+      run: |
+        aws ecr get-login-password \
+          --region ${{ inputs.aws_region }} | docker login --username AWS --password-stdin ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_region  }}.amazonaws.com
+      shell: bash
+
+    # Unset AWS credentials to avoid conflicts, as we prefer credentials from ~/.aws/credentials to authenticate
+    - name: Unset AWS credentials to avoid conflicts
+      shell: bash
+      run: |
+        echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV
+        echo AWS_REGION='' >> $GITHUB_ENV
+        echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV
+        echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV

.github/actions/docker/action.yaml

@@ -25,21 +25,24 @@ inputs:
   cargo_profile:
     description: Cargo build profile, i.e release or dev
     default: dev
-  bucket:
-    description: S3 bucket to use for caching, must match runner define in `runs-on`
-    default: multi-runner-cache-x1xibo9c
-  region:
+  cache_bucket:
+    description: S3 bucket to use for caching (both sccache and layer cache)
+    required: true
+  cache_region:
     description: S3 bucket region
     required: true
-  aws_access_key_id:
-    description: AWS access key ID
+  cache_endpoint:
+    description: S3 endpoint to use for caching
+    required: true
+  cache_access_key_id:
+    description: Access key ID for s3 cache
     required: true
-  aws_secret_access_key:
-    description: AWS secret access key
+  cache_secret_access_key:
+    description: Secret access key for s3 cache
     required: true
   cache_to_name:
-    description: 'Save cache to name manifest (should be used only on default branch)'
-    default: 'false'
+    description: "Save cache to name manifest (should be used only on default branch)"
+    default: "false"
 outputs:
   digest:
     value: ${{ steps.docker_build.outputs.digest }}
@@ -80,8 +83,9 @@ runs:
       id: layer_cache_settings
       with:
         name: ${{ inputs.image_name }}
-        region: ${{ inputs.region }}
-        bucket: ${{ inputs.bucket }}
+        region: ${{ inputs.cache_region }}
+        bucket: ${{ inputs.cache_bucket }}
+        endpoint: ${{ inputs.cache_endpoint }}
         cache_to_name: ${{ inputs.cache_to_name }}
 
     - name: Set HOME variable to github context
@@ -133,7 +137,7 @@ runs:
       id: arch
       uses: actions/github-script@v6
       with:
-        result-encoding: 'string'
+        result-encoding: "string"
         script: return '${{ inputs.platform }}'.replace('linux/', '');
 
     - name: Inject cargo cache into docker
@@ -148,6 +152,16 @@ runs:
           }
         skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
 
+    - name: Setup sccache vars
+      uses: ./.github/actions/sccache
+      with:
+        bucket: ${{ inputs.cache_bucket }}
+        region: ${{ inputs.cache_region }}
+        endpoint: ${{ inputs.cache_endpoint }}
+        access_key_id: ${{ inputs.cache_access_key_id }}
+        secret_access_key: ${{ inputs.cache_secret_access_key }}
+        install: false
+
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
       uses: docker/build-push-action@v6
@@ -159,14 +173,16 @@ runs:
         push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
+        secret-files: |
+          AWS=${{ env.HOME }}/.aws/credentials
         build-args: |
           CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }}
-          RUSTC_WRAPPER=sccache
-          SCCACHE_BUCKET=${{ inputs.bucket }}
-          SCCACHE_REGION=${{ inputs.region }}
-          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
-          AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
+          SCCACHE_BUCKET=${{ inputs.cache_bucket }}
+          SCCACHE_REGION=${{ inputs.cache_region }}
+          SCCACHE_ENDPOINT=${{ inputs.cache_endpoint }}
+          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          SCCACHE_VERSION=${{ env.SCCACHE_VERSION }}
+          AWS_PROFILE=sccache
         cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
         cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true

.github/actions/librocksdb/action.yaml

@@ -9,10 +9,6 @@ inputs:
     description: RocksDB version, eg. "8.10.2"
     required: false
     default: "8.10.2"
-  bucket:
-    description: S3 bucket to use for caching
-    required: false
-    default: multi-runner-cache-x1xibo9c
   force:
     description: Force rebuild
     required: false

.github/actions/rust/action.yaml

@@ -31,7 +31,7 @@ runs:
         fi
 
         echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV
-        echo "::set-output name=version::$TOOLCHAIN_VERSION"
+        echo "version=$TOOLCHAIN_VERSION"  >> $GITHUB_OUTPUT
 
     - uses: dtolnay/rust-toolchain@master
       name: Install Rust toolchain

.github/actions/s3-layer-cache-settings/action.yaml

@@ -1,4 +1,4 @@
-name: 'Get S3 Docker Layer Cache settings'
+name: "Get S3 Docker Layer Cache settings"
 description: |
   This action generates string with s3-based cache configuration for docker buildx.
   It defines three manifests:
@@ -8,33 +8,36 @@ description: |
 
 inputs:
   name:
-    description: 'Cache key name will be used as a prefix for all docker image manifests'
+    description: "Cache key name will be used as a prefix for all docker image manifests"
     required: true
   head_ref:
-    description: 'Head ref for an additional manifest to hit all builds for this head'
+    description: "Head ref for an additional manifest to hit all builds for this head"
     default: ${{ github.ref }}
   region:
     description: S3 region
     required: true
   bucket:
     description: S3 bucket name
     required: true
+  endpoint:
+    description: S3 endpoint to use for caching
+    required: false
   prefix:
     description: S3 key prefix
-    default: 'cache-layers/'
+    default: "cache-layers/"
   mode:
     description: Cache mode
     default: max
   cache_to_name:
-    description: 'Save cache to name manifest (should be used only on default branch)'
-    default: 'false'
+    description: "Save cache to name manifest (should be used only on default branch)"
+    default: "false"
 
 outputs:
   cache_to:
-    description: 'String with s3-based cache configuration for docker buildx cache-to option'
+    description: "String with s3-based cache configuration for docker buildx cache-to option"
     value: ${{ steps.script.outputs.cache_to }}
   cache_from:
-    description: 'String with s3-based cache configuration for docker buildx cache-from option'
+    description: "String with s3-based cache configuration for docker buildx cache-from option"
     value: ${{ steps.script.outputs.cache_from }}
 
 runs:
@@ -49,6 +52,7 @@ runs:
             region: '${{ inputs.region }}',
             bucket: '${{ inputs.bucket }}',
             prefix: '${{ inputs.prefix }}',
+            endpoint_url: '${{ inputs.endpoint }}',
           };
 
           const settingsString = Object.entries(settings)
@@ -59,7 +63,7 @@ runs:
           const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-');
 
           const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}';
-          const headRefManifestName = '${{ inputs.name }}_tag_${ sanitizedHeadRef }';
+          const headRefManifestName = '${{ inputs.name }}_tag_' + sanitizedHeadRef;
 
           const cacheFromManifestNames = [
             shaManifestName,

.github/actions/sccache/action.yaml

@@ -0,0 +1,55 @@
+---
+name: "sccache"
+description: "Configure sccache caching"
+inputs:
+  bucket:
+    description: S3 bucket to use for caching
+    required: true
+  region:
+    description: S3 bucket region
+    required: true
+  endpoint:
+    description: S3 endpoint to use for caching
+    required: true
+  access_key_id:
+    description: S3 endpoint access key ID
+    required: true
+  secret_access_key:
+    description: S3 endpoint secret access key
+    required: true
+  install:
+    description: "Install sccache"
+    default: "true"
+  version:
+    description: "sccache version"
+    default: "0.8.2"
+    required: false
+
+# TODO: Cache deps here to save 1 minute
+runs:
+  using: composite
+  steps:
+    - name: Install sccache binary
+      if: ${{ inputs.install == 'true' }}
+      uses: mozilla-actions/[email protected]
+      with:
+        version: "v${{ inputs.version }}"
+
+    - name: Configure AWS credentials
+      uses: ./.github/actions/aws_credentials
+      with:
+        access_key_id: ${{ inputs.access_key_id }}
+        secret_access_key: ${{ inputs.secret_access_key }}
+        profile: "sccache"
+
+    - name: Configure sccache
+      shell: bash
+      run: |
+        echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        echo "SCCACHE_BUCKET=${{ inputs.bucket }}" >> $GITHUB_ENV
+        echo "SCCACHE_REGION=${{ inputs.region }}" >> $GITHUB_ENV
+        echo "SCCACHE_ENDPOINT=${{ inputs.endpoint }}" >> $GITHUB_ENV
+        echo "SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu" >> $GITHUB_ENV
+        # "SCCACHE_VERSION" is used inside Docker to install the same version of sccache
+        echo "SCCACHE_VERSION=${{ inputs.version }}" >> $GITHUB_ENV

.github/workflows/release-docker-image.yml

@@ -47,13 +47,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Configure AWS credentials and bucket region
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         if: ${{ matrix.platform == 'linux/arm64' }}
@@ -70,9 +63,11 @@ jobs:
           cargo_profile: ${{ inputs.cargo_profile }}
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-          region: ${{ secrets.AWS_REGION }}
-          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          cache_region: ${{ vars.CACHE_REGION }}
+          cache_bucket: ${{ vars.CACHE_S3_BUCKET }}
+          cache_endpoint: ${{ vars.CACHE_S3_ENDPOINT }}
+          cache_access_key_id: ${{ secrets.CACHE_KEY_ID }}
+          cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }}
 
       - name: Export digest
         run: |