Merge Compatible to develop oct22

buildkite/scripts/debian/start_local_repo.sh

@@ -5,6 +5,22 @@ if [ -z $MINA_DEB_CODENAME ]; then
     exit 1
 fi
 
+# Parse command line arguments
+ARCH="amd64"  # default architecture
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --arch)
+            ARCH="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
 LOCAL_DEB_FOLDER=_build
 
 set -eou pipefail
@@ -16,4 +32,4 @@ mkdir -p $LOCAL_DEB_FOLDER
 source ./buildkite/scripts/export-git-env-vars.sh
 ./buildkite/scripts/cache/manager.sh read --root legacy/debians "$MINA_DEB_CODENAME/*" _build
 ./buildkite/scripts/cache/manager.sh read "debians/$MINA_DEB_CODENAME/*" _build
-./scripts/debian/aptly.sh start --codename $MINA_DEB_CODENAME --debians $LOCAL_DEB_FOLDER --component unstable --clean --background --wait
+./scripts/debian/aptly.sh start --codename $MINA_DEB_CODENAME --debians $LOCAL_DEB_FOLDER --component unstable --clean --background --wait --archs $ARCH

buildkite/scripts/release/manager.sh

@@ -1915,6 +1915,10 @@ function pull(){
                 __networks=${2:?$error_message}
                 shift 2;
             ;;
+            --archs )
+                __archs=${2:?$error_message}
+                shift 2;
+            ;;
             * )
                 echo -e "${RED} !! Unknown option: $1${CLEAR}\n";
                 echo "";
@@ -1936,6 +1940,8 @@ function pull(){
     echo " - Target: $__target"
     echo " - Codenames: $__codenames"
     echo " - Networks: $__networks"
+    echo " - Architectures: $__archs"
+
     if [[ -n ${__from_special_folder+x} ]]; then
         echo " - From special folder: $__from_special_folder"
     fi
@@ -1947,26 +1953,28 @@ function pull(){
     read -r -a __artifacts_arr <<< "$__artifacts"
     read -r -a __codenames_arr <<< "$__codenames"
     read -r -a __networks_arr <<< "$__networks"
+    read -r -a __archs_arr <<< "$__archs"
 
-    for __artifact in "${__artifacts_arr[@]}"; do
-        for __codename in "${__codenames_arr[@]}"; do
-            for network in "${__networks_arr[@]}"; do
-                echo "  📥  Pulling $__artifact for $__codename codename and $network network"
-                local __artifact_full_name
-                local __source_path
-                __artifact_full_name=$(get_artifact_with_suffix $__artifact $network)
-
-                if [[ -n ${__from_special_folder+x} ]]; then
-                    __source_path="$(storage_root "$__backend")/$__from_special_folder/${__artifact_full_name}_*"
-                else
-                    __source_path="$(storage_root "$__backend")/$__buildkite_build_id/debians/$__codename/${__artifact_full_name}_*"
-                fi
-
-                storage_download "$__backend" "$__source_path" "$__target"
+    for __arch in "${__archs_arr[@]}"; do
+        for __artifact in "${__artifacts_arr[@]}"; do
+            for __codename in "${__codenames_arr[@]}"; do
+                for network in "${__networks_arr[@]}"; do
+                    echo "  📥  Pulling $__artifact for $__codename codename and $network network"
+                    local __artifact_full_name
+                    local __source_path
+                    __artifact_full_name=$(get_artifact_with_suffix $__artifact $network)
+
+                    if [[ -n ${__from_special_folder+x} ]]; then
+                        __source_path="$(storage_root "$__backend")/$__from_special_folder/${__artifact_full_name}_*_${__arch}.deb"
+                    else
+                        __source_path="$(storage_root "$__backend")/$__buildkite_build_id/debians/$__codename/${__artifact_full_name}_*_${__arch}.deb"
+                    fi
+
+                    storage_download "$__backend" "$__source_path" "$__target"
+                done
             done
         done
     done
-
     echo " ✅  Done."
     echo ""
 }

buildkite/src/Command/DockerImage.dhall

@@ -109,7 +109,8 @@ let generateStep =
                 then  " && echo Skipping local debian repo setup "
 
                 else      " && ./buildkite/scripts/debian/update.sh --verbose"
-                      ++  " && apt-get install aptly -y && ./buildkite/scripts/debian/start_local_repo.sh"
+                      ++  " && apt-get install aptly -y && ./buildkite/scripts/debian/start_local_repo.sh --arch ${Arch.lowerName
+                                                                                                                     spec.arch}"
 
           let maybeStopDebianRepo =
                       if spec.no_debian

buildkite/src/Command/MinaArtifact.dhall

@@ -139,7 +139,7 @@ let build_artifacts
                   ]
             , label = "Debian: Build ${labelSuffix spec}"
             , key = "build-deb-pkg${Optional/default Text "" spec.suffix}"
-            , target = Size.XLarge
+            , target = Size.Multi
             , if_ = spec.if_
             , retries =
               [ Command.Retry::{

buildkite/src/Constants/ContainerImages.dhall

@@ -5,22 +5,22 @@
 { toolchainBase = "codaprotocol/ci-toolchain-base:v3"
 , minaToolchainBookworm =
     { amd64 =
-        "gcr.io/o1labs-192920/mina-toolchain@sha256:dfa8a0eb32742900d890590875a7f7436545cd46d8c4ff147fc6a29997e5d4f3"
+        "gcr.io/o1labs-192920/mina-toolchain@sha256:a3517722b1573ac19f9361d42aa0b5f15a108d62fa73f97f0a74e195af1a2e90"
     , arm64 =
-        "gcr.io/o1labs-192920/mina-toolchain@sha256:8e92e0b4c9202e0e5f31afd48713d28bde903959ff7e55cbc1c080b0a8df5e3d"
+        "gcr.io/o1labs-192920/mina-toolchain@sha256:b4a7040b64473f89e51372ab6e11f9332730e085766aa157c25e28c6b9c0c6d4"
     }
 , minaToolchainBullseye.amd64 =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:bcc6b9899d5d99c83287c2735fb686a6169268d48b11262d9dfa03c1dfd0cece"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:524a3fb77d6702f38ad63ec737e398478e082387753b8d01e1ccf607d2917343"
 , minaToolchainNoble =
     { amd64 =
-        "gcr.io/o1labs-192920/mina-toolchain@sha256:0c6f1c0921c7f76be7b86948e1f9e82d8270002fc19f0b48647bb1604489268f"
+        "gcr.io/o1labs-192920/mina-toolchain@sha256:3fd6ebfedb905ba358903639c2ebcdbd2277316b854d80f12c74df8dcd83734b"
     , arm64 =
-        "gcr.io/o1labs-192920/mina-toolchain@sha256:182f0aa05988c5a00cc1ca5a5b651904282f3a0f7cd75faabe0a52e7d332cecb"
+        "gcr.io/o1labs-192920/mina-toolchain@sha256:adfff17bf21b79efc3b2d54d0648049a36fa7b54b732773bd7f2f651869c1f54"
     }
 , minaToolchainJammy.amd64 =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:a3e5c7dc30c67d8a9769deee282f0b5b2a513629bf272aa9b9ec7d6aee68a4e4"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:6db30a6faf94f7dcc23a14bcbc01d558b2164955067fd21eb92a981fe231dfe1"
 , minaToolchain =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:bcc6b9899d5d99c83287c2735fb686a6169268d48b11262d9dfa03c1dfd0cece"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:524a3fb77d6702f38ad63ec737e398478e082387753b8d01e1ccf607d2917343"
 , postgres = "postgres:12.4-alpine"
 , xrefcheck =
     "dkhamsing/awesome_bot@sha256:a8adaeb3b3bd5745304743e4d8a6d512127646e420544a6d22d9f58a07f35884"

buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest.dhall

@@ -0,0 +1,53 @@
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+
+let Size = ../../Command/Size.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+
+let Docker = ../../Command/Docker/Type.dhall
+
+let commands =
+      [ Cmd.run
+          "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --quiet --yes python3 python3-pip build-essential sudo curl"
+      , Cmd.run "./scripts/rocksdb-compatibility/install-rocksdb.sh"
+      , Cmd.run
+          "pip3 install --break-system-packages -r ./scripts/rocksdb-compatibility/requirements.txt"
+      , Cmd.run "python3 ./scripts/rocksdb-compatibility/test.py"
+      ]
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "scripts/rocksdb-compatibility")
+          , S.exactly
+              "buildkite/src/Jobs/Test/RocksDBLedgerTarCompatibilityTest"
+              "dhall"
+          ]
+        , path = "Test"
+        , name = "RocksDBLedgerTarCompatibilityTest"
+        , tags =
+          [ PipelineTag.Type.Fast
+          , PipelineTag.Type.Test
+          , PipelineTag.Type.Stable
+          ]
+        }
+      , steps =
+        [ Command.build
+            Command.Config::{
+            , commands = commands
+            , label = "Check RocksDB Ledger Tar Compatibility"
+            , key = "test"
+            , target = Size.Multi
+            , docker = Some Docker::{ image = "ubuntu:noble" }
+            }
+        ]
+      }

buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall

@@ -38,5 +38,5 @@ in  Pipeline.build
           , PipelineTag.Type.Stable
           ]
         }
-      , steps = [ TestExecutive.executeLocal "hard-fork" dependsOn ]
+      , steps = [ TestExecutive.executeLocal "post-hard-fork" dependsOn ]
       }

changes/17819.md

@@ -0,0 +1,2 @@
+Fix crash in ledger sync check that could occur when trying to load ledger
+databases that were only partially synced to a network ledger

changes/17954.md

@@ -0,0 +1 @@
+Mina Daemon now use RocksDB 10.5.1 under the hood to parse genesis/epoch ledger tars. It's tested that RocksDB preserved backward compatibility on the ledger tars we serve with old RocksDB version.

changes/17962.md

@@ -0,0 +1,3 @@
+Altering mina-missing-block-auditor app.
+
+Correctly handling network derived from devnet/mainnet, but without pre-hardfork blocks

dockerfiles/stages/1-build-deps

@@ -45,7 +45,7 @@ ARG OPAM_VERSION=2.3.0
 #   `nix flake update opam-repository`).
 # - scripts/update_opam_switch.sh
 ARG OPAM_REPOSITORY_COMMIT=08d8c16c16dc6b23a5278b06dff0ac6c7a217356
-ARG O1LABS_OPAM_REPOSITORY_COMMIT=cabde639f92d259d4c131b00200d7a53d854ee74
+ARG O1LABS_OPAM_REPOSITORY_COMMIT=dd90c5c72b7b7caeca3db3224b2503924deea08a
 
 # Golang version number used to detemine tarball name
 ARG GO_VERSION=1.19.11

flake.nix

@@ -31,14 +31,14 @@
   inputs.describe-dune.inputs.nixpkgs.follows = "nixpkgs";
   inputs.describe-dune.inputs.flake-utils.follows = "utils";
 
-  inputs.o1-opam-repository.url = "github:o1-labs/opam-repository/cabde639f92d259d4c131b00200d7a53d854ee74";
+  inputs.o1-opam-repository.url = "github:o1-labs/opam-repository/dd90c5c72b7b7caeca3db3224b2503924deea08a";
   inputs.o1-opam-repository.flake = false;
 
   # The version must be the same as the version used in:
   # - dockerfiles/1-build-deps
   # - flake.nix (and flake.lock after running
   #   `nix flake update opam-repository`).
-  # - scripts/update_opam_switch.sh
+  # - scripts/update-opam-switch.sh
   inputs.opam-repository.url =
     "github:ocaml/opam-repository/08d8c16c16dc6b23a5278b06dff0ac6c7a217356";
   inputs.opam-repository.flake = false;

nix/ocaml.nix

@@ -66,10 +66,9 @@ let
       };
 
       rocksdb_stubs =
-        # TODO uncomment after updating rocksdb_stubs to 10.5.1
-        # assert (super.rocksdb_stubs.version == pkgs.rocksdb-mina.version)
-        #   || builtins.throw
-        #   "rocksdb_stubs version (${super.rocksdb_stubs.version}) does not match rocksdb-mina version (${pkgs.rocksdb-mina.version})";
+        assert (super.rocksdb_stubs.version == pkgs.rocksdb-mina.version)
+          || builtins.throw
+          "rocksdb_stubs version (${super.rocksdb_stubs.version}) does not match rocksdb-mina version (${pkgs.rocksdb-mina.version})";
         super.rocksdb_stubs.overrideAttrs {
           MINA_ROCKSDB = "${pkgs.rocksdb-mina}/lib/librocksdb.a";
         };

opam.export

@@ -49,7 +49,7 @@ roots: [
   "qcheck-alcotest.0.20"
   "re2.v0.14.0"
   "rocks.0.2.0"
-  "rocksdb_stubs.10.2.1"
+  "rocksdb_stubs.10.5.1"
   "rpc_parallel.v0.14.0"
   "sexp_diff_kernel.v0.14.0"
   "sodium.dev"
@@ -250,7 +250,7 @@ installed: [
   "res.5.0.1"
   "result.1.5"
   "rocks.0.2.0"
-  "rocksdb_stubs.10.2.1"
+  "rocksdb_stubs.10.5.1"
   "rpc_parallel.v0.14.0"
   "rresult.0.6.0"
   "seq.base"