Merge remote-tracking branch 'origin/develop' into lyh/app-state-size32-dev

Author: glyh

.gitignore

@@ -34,17 +34,14 @@ target
 ./release
 .vscode
 
-### IntelliJ IDEA ###
-**/.idea/
-*.iws
-*.iml
-*.ipr
-
 ### Direnv ###
 .env
 .envrc
 .direnv/
 
+### Python ###
+**/.venv
+**/__pycache__
 # pipenv
 Pipfile
 Pipfile.lock

CODEOWNERS

@@ -1,6 +1,5 @@
 /CODEOWNERS @dannywillems @deepthiskumar @georgeee
 /buildkite @MinaProtocol/infra-eng-reviewers
-/frontend/ @MinaProtocol/product-eng-reviewers
 /dockerfiles/ @MinaProtocol/infra-eng-reviewers
 /scripts/ @MinaProtocol/infra-eng-reviewers @MinaProtocol/protocol-eng-reviewers
 /Makefile @MinaProtocol/infra-eng-reviewers

Makefile

@@ -164,6 +164,7 @@ build-daemon-utils: ocaml_checks reformat-diff libp2p_helper ## Build daemon uti
 		src/app/validate_keypair/validate_keypair.exe \
 		src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe \
 		src/lib/snark_worker/standalone/run_snark_worker.exe \
+		src/app/rocksdb-scanner/rocksdb_scanner.exe \
 		--profile=$(DUNE_PROFILE) \
 		&& echo "✅ Build complete"
 
@@ -221,6 +222,7 @@ build-archive-utils: ocaml_checks reformat-diff ## Build archive node and relate
 		src/app/archive_blocks/archive_blocks.exe \
 		src/app/extract_blocks/extract_blocks.exe \
 		src/app/missing_blocks_auditor/missing_blocks_auditor.exe \
+		src/app/archive_hardfork_toolbox/archive_hardfork_toolbox.exe \
 		--profile=$(DUNE_PROFILE)  \
 		&& echo "✅ Build complete"
 

README-dev.md

@@ -150,7 +150,7 @@ A number of C libraries are expected to be available in the system and are also
 If you use vim, add this snippet in your `.vimrc` file to use Merlin. (Note:Be sure to change the HOME directory to match yours.)
 
 ```bash
-let s:ocamlmerlin="/Users/USERNAME/.opam/4.14.0/share/merlin"
+let s:ocamlmerlin="/Users/USERNAME/.opam/4.14.2/share/merlin"
 execute "set rtp+=".s:ocamlmerlin."/vim"
 execute "set rtp+=".s:ocamlmerlin."/vimbufsync"
 let g:syntastic_ocaml_checkers=['merlin']
@@ -301,10 +301,10 @@ We use the [Dune](https://github.com/ocaml/dune/) build system for OCaml code.
 
 OCaml dependencies live in the [`opam.export`](./opam.export) file. This file is machine generated and must not be modified.
 
-To add a new dependency, you most likely will need to create a new fresh switch to avoid pushing in any local dependency (like `ocaml-lsp`). The following commands assume that the version of the OCaml compiler used in the codebase is 4.14.0:
+To add a new dependency, you most likely will need to create a new fresh switch to avoid pushing in any local dependency (like `ocaml-lsp`). The following commands assume that the version of the OCaml compiler used in the codebase is 4.14.2:
 
 ```shell
-opam switch create mina_fresh 4.14.0
+opam switch create mina_fresh 4.14.2
 opam switch import opam.export
 ```
 

README-spacetime.md

@@ -4,17 +4,26 @@ check_syntax:
 	# xargs will short-circuit if a command fails with code 255
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" &&  dhall --file {} > /dev/null || exit 255'
 
-check_lint:
+convert_backticks_to_ifs:
+	sed -i -E 's/`if` =/if =/g' src/Command/Base.dhall
+
+convert_ifs_to_backticks:
+	sed -i -E 's/if =/`if` =/g' src/Command/Base.dhall
+
+check_lint: convert_backticks_to_ifs
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii lint --check --inplace {} || exit 255'
 
-check_format:
+check_format: convert_backticks_to_ifs
+	@echo "Dhall version: $$(dhall --version)"
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{} format" && dhall --ascii format --check --inplace {} || exit 255'
 
 lint:
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii lint --inplace {} || exit 255'
+	$(MAKE) convert_ifs_to_backticks
 
 format:
 	find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255'
+	$(MAKE) convert_ifs_to_backticks
 
 check_filter:
 	# Ensure Jobs.dhall is generated before proceeding

buildkite/Makefile

@@ -137,8 +137,8 @@ main() {
     )
 
     local scripts=(
-        "src/app/archive/upgrade-to-mesa.sql"
-        "src/app/archive/downgrade-to-berkeley.sql"
+        "src/app/archive/upgrade_to_mesa.sql"
+        "src/app/archive/downgrade_to_berkeley.sql"
     )
 
     # Check if either monitored file has changes
@@ -202,4 +202,4 @@ main() {
     exit 0
 }
 
-main "$@"
+main "$@"

buildkite/scripts/archive/upgrade-script-check.sh

@@ -5,6 +5,22 @@ if [ -z $MINA_DEB_CODENAME ]; then
     exit 1
 fi
 
+# Parse command line arguments
+ARCH="amd64"  # default architecture
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --arch)
+            ARCH="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
 LOCAL_DEB_FOLDER=_build
 
 set -eou pipefail
@@ -16,4 +32,4 @@ mkdir -p $LOCAL_DEB_FOLDER
 source ./buildkite/scripts/export-git-env-vars.sh
 ./buildkite/scripts/cache/manager.sh read --root legacy/debians "$MINA_DEB_CODENAME/*" _build
 ./buildkite/scripts/cache/manager.sh read "debians/$MINA_DEB_CODENAME/*" _build
-./scripts/debian/aptly.sh start --codename $MINA_DEB_CODENAME --debians $LOCAL_DEB_FOLDER --component unstable --clean --background --wait
+./scripts/debian/aptly.sh start --codename $MINA_DEB_CODENAME --debians $LOCAL_DEB_FOLDER --component unstable --clean --background --wait --archs $ARCH

buildkite/scripts/debian/start_local_repo.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euox pipefail
+
 # Generate Hardfork Package Build Script
 #
 # This script generates Dhall configuration for creating hardfork packages in the Mina protocol
@@ -10,11 +12,12 @@
 #   ./run-hardfork-package-gen.sh
 #
 # REQUIRED ENVIRONMENT VARIABLES:
-#   CODENAMES           - Comma-separated list of Debian codenames (e.g., "Bullseye,Focal")
-#   NETWORK             - Target network name (e.g., "Devnet", "Mainnet")
-#   GENESIS_TIMESTAMP   - Genesis timestamp in ISO format (e.g., "2024-04-07T11:45:00Z")
-#   CONFIG_JSON_GZ_URL  - URL to the gzipped genesis config JSON file
-#   VERSION             - Version string for the hardfork package
+#   CODENAMES                     - Comma-separated list of Debian codenames (e.g., "Bullseye,Focal")
+#   NETWORK                       - Target network name (e.g., "Devnet", "Mainnet")
+#   GENESIS_TIMESTAMP             - Genesis timestamp in ISO format (e.g., "2024-04-07T11:45:00Z")
+#   CONFIG_JSON_GZ_URL            - URL to the gzipped genesis config JSON file
+#   VERSION                       - Version string for the hardfork package (optional, if not set, defaults to calculated from git)
+#   PRECOMPUTED_FORK_BLOCK_PREFIX - (Optional) Prefix for precomputed fork block URLs (e.g., "gs://mina_network_block_data/mainnet")
 #
 # EXAMPLE:
 #   export CODENAMES="Bullseye,Focal"
@@ -42,20 +45,21 @@ function usage() {
     echo -e "${RED}☞  $1${CLEAR}\n";
   fi
   cat << EOF
-  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)
-  NETWORK                     The Docker and Debian network (Devnet, Mainnet)
-  GENESIS_TIMESTAMP           The Genesis timestamp in ISO format (e.g. 2024-04-07T11:45:00Z)
-  CONFIG_JSON_GZ_URL          The URL to the gzipped genesis config JSON file
-  VERSION                     The version of the hardfork package to generate (e.g. 3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e)
-
+  CODENAMES                     The Debian codenames (Bullseye, Focal etc.)
+  NETWORK                       The Docker and Debian network (Devnet, Mainnet)
+  GENESIS_TIMESTAMP             The Genesis timestamp in ISO format (e.g. 2024-04-07T11:45:00Z)
+  CONFIG_JSON_GZ_URL            The URL to the gzipped genesis config JSON file
+  VERSION                       (Optional) The version of the hardfork package to generate (e.g. 3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e)
+  PRECOMPUTED_FORK_BLOCK_PREFIX (Optional) The prefix for precomputed fork block URLs (e.g. gs://mina_network_block_data/mainnet)
 EOF
   exit 1
 }
 
 function to_dhall_list() {
   local input_str="$1"
   local dhall_type="$2"
-  local arr=("${input_str//,/ }")
+  local arr
+  IFS=',' read -ra arr <<< "$input_str"
   local dhall_list=""
 
   if [[ ${#arr[@]} -eq 0 || -z "${arr[0]}" ]]; then
@@ -72,6 +76,52 @@ function to_dhall_list() {
   echo "$dhall_list"
 }
 
+if [[ -z "$CODENAMES" ]]; then
+  usage "CODENAMES environment variable is required"
+fi
+
+if [[ -z "$NETWORK" ]]; then
+  usage "NETWORK environment variable is required"
+fi
+
+if [[ -z "$CONFIG_JSON_GZ_URL" ]]; then
+  usage "CONFIG_JSON_GZ_URL environment variable is required"
+fi
+
+# Format GENESIS_TIMESTAMP as Optional Text for Dhall
+if [[ -z "$GENESIS_TIMESTAMP" ]]; then
+  GENESIS_TIMESTAMP="None Text"
+else
+  # shellcheck disable=SC2089
+  GENESIS_TIMESTAMP="(Some \"${GENESIS_TIMESTAMP}\")"
+fi
+
+# Format VERSION as Optional Text for Dhall
+if [[ -z "$VERSION" ]]; then
+  VERSION="(None Text)"
+else
+  # shellcheck disable=SC2089
+  VERSION="(Some \"${VERSION}\")"
+fi
+
+# Format PRECOMPUTED_FORK_BLOCK_PREFIX as Optional Text for Dhall
+if [[ -z "$PRECOMPUTED_FORK_BLOCK_PREFIX" ]]; then
+  PRECOMPUTED_FORK_BLOCK_PREFIX="(None Text)"
+else
+  # shellcheck disable=SC2089
+  PRECOMPUTED_FORK_BLOCK_PREFIX="(Some \"${PRECOMPUTED_FORK_BLOCK_PREFIX}\")"
+fi
+
 DHALL_CODENAMES=$(to_dhall_list "$CODENAMES" "$DEBIAN_VERSION_DHALL_DEF.DebVersion")
 
-echo $GENERATE_HARDFORK_PACKAGE_DHALL_DEF'.generate_hardfork_package '"$DHALL_CODENAMES"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' (None Text) "'"${CONFIG_JSON_GZ_URL}"'" "'""'" ' | dhall-to-yaml --quoted
+# shellcheck disable=SC2089
+printf '%s.generate_hardfork_package %s %s.Type.%s %s "%s" "%s" %s %s\n' \
+  "$GENERATE_HARDFORK_PACKAGE_DHALL_DEF" \
+  "$DHALL_CODENAMES" \
+  "$NETWORK_DHALL_DEF" \
+  "$NETWORK" \
+  "$GENESIS_TIMESTAMP" \
+  "$CONFIG_JSON_GZ_URL" \
+  "" \
+  "$VERSION" \
+  "$PRECOMPUTED_FORK_BLOCK_PREFIX" | dhall-to-yaml --quoted

buildkite/scripts/entrypoints/run-hardfork-package-gen.sh

@@ -21,6 +21,8 @@ fi
 # Fetch a fresh copy of the target branch
 source buildkite/scripts/refresh_code.sh
 
+git fetch origin "${BRANCH}"
+
 # Check mergeability. We use flags so that
 # * `--no-commit` stops us from updating the index with a merge commit,
 # * `--no-ff` stops us from updating the index to the HEAD, if the merge is a