Merge branch 'master' into idf-release/v5.5

Author: me-no-dev

.github/scripts/find_new_boards.sh

@@ -27,12 +27,25 @@ echo "$modified_lines"
 boards_array=()
 previous_board=""
 
+# Define excluded entries that are not actual boards
+excluded_entries=("" "+" "-" "esp32_family" "menu")
+
 # Extract board names from the modified lines, and add them to the boards_array
 while read -r line; do
     board_name=$(echo "$line" | cut -d '.' -f1 | cut -d '#' -f1)
     # remove + or - from the board name at the beginning
     board_name=${board_name#[-+]}
-    if [ "$board_name" != "" ] && [ "$board_name" != "+" ] && [ "$board_name" != "-" ] && [ "$board_name" != "esp32_family" ]; then
+    
+    # Check if board_name is in excluded entries
+    is_excluded=false
+    for excluded in "${excluded_entries[@]}"; do
+        if [ "$board_name" = "$excluded" ]; then
+            is_excluded=true
+            break
+        fi
+    done
+    
+    if [ "$is_excluded" = false ]; then
         if [ "$board_name" != "$previous_board" ]; then
             boards_array+=("espressif:esp32:$board_name")
             previous_board="$board_name"

.github/scripts/on-release.sh

@@ -54,6 +54,30 @@ if [ -n "${VENDOR}" ]; then
     echo "Setting packager: $VENDOR"
 fi
 
+function update_version {
+    set -e
+    set -o pipefail
+
+    local tag=$1
+    local major
+    local minor
+    local patch
+
+    # Extract major, minor, and patch from the tag
+    # We need to make sure to remove the "v" prefix from the tag and any characters after the patch version
+    tag=$(echo "$tag" | sed 's/^v//g' | sed 's/-.*//g')
+    major=$(echo "$tag" | cut -d. -f1)
+    minor=$(echo "$tag" | cut -d. -f2)
+    patch=$(echo "$tag" | cut -d. -f3 | sed 's/[^0-9].*//')  # Remove non-numeric suffixes like RC1, alpha, beta
+
+    echo "Major: $major, Minor: $minor, Patch: $patch"
+
+    "${SCRIPTS_DIR}/update-version.sh" "$major" "$minor" "$patch"
+
+    set +e
+    set +o pipefail
+}
+
 function get_file_size {
     local file="$1"
     if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -199,8 +223,31 @@ set -e
 ##
 
 mkdir -p "$OUTPUT_DIR"
-PKG_DIR="$OUTPUT_DIR/$PACKAGE_NAME"
+PKG_DIR="${OUTPUT_DIR:?}/$PACKAGE_NAME"
 PACKAGE_ZIP="$PACKAGE_NAME.zip"
+PACKAGE_XZ="$PACKAGE_NAME.tar.xz"
+LIBS_ZIP="$PACKAGE_NAME-libs.zip"
+LIBS_XZ="$PACKAGE_NAME-libs.tar.xz"
+
+echo "Updating version..."
+if ! update_version "$RELEASE_TAG"; then
+    echo "ERROR: update_version failed!"
+    exit 1
+fi
+git config --global github.user "github-actions[bot]"
+git config --global user.name "github-actions[bot]"
+git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+git add .
+
+# We should only commit if there are changes
+need_update_commit=true
+if git diff --cached --quiet; then
+    echo "Version already updated"
+    need_update_commit=false
+else
+    echo "Creating version update commit..."
+    git commit -m "change(version): Update core version to $RELEASE_TAG"
+fi
 
 echo "Updating submodules ..."
 git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
@@ -283,7 +330,7 @@ echo \#define ARDUINO_ESP32_GIT_DESC "$(git -C "$GITHUB_WORKSPACE" describe --ta
 echo \#define ARDUINO_ESP32_RELEASE_"$ver_define" >> "$PKG_DIR/cores/esp32/core_version.h"
 echo \#define ARDUINO_ESP32_RELEASE \""$ver_define"\" >> "$PKG_DIR/cores/esp32/core_version.h"
 
-# Compress package folder
+# Compress ZIP package folder
 echo "Creating ZIP ..."
 pushd "$OUTPUT_DIR" >/dev/null
 zip -qr "$PACKAGE_ZIP" "$PACKAGE_NAME"
@@ -293,22 +340,112 @@ if [ $? -ne 0 ]; then
 fi
 
 # Calculate SHA-256
-echo "Calculating SHA sum ..."
-PACKAGE_PATH="$OUTPUT_DIR/$PACKAGE_ZIP"
+echo "Calculating ZIP SHA sum ..."
+PACKAGE_PATH="${OUTPUT_DIR:?}/$PACKAGE_ZIP"
 PACKAGE_SHA=$(shasum -a 256 "$PACKAGE_ZIP" | cut -f 1 -d ' ')
 PACKAGE_SIZE=$(get_file_size "$PACKAGE_ZIP")
 popd >/dev/null
-rm -rf "$PKG_DIR"
 echo "'$PACKAGE_ZIP' Created! Size: $PACKAGE_SIZE, SHA-256: $PACKAGE_SHA"
 echo
 
-# Upload package to release page
-echo "Uploading package to release page ..."
+# Compress XZ package folder
+echo "Creating XZ ..."
+pushd "$OUTPUT_DIR" >/dev/null
+tar -cJf "$PACKAGE_XZ" "$PACKAGE_NAME"
+if [ $? -ne 0 ]; then
+    echo "ERROR: Failed to create $PACKAGE_XZ ($?)"
+    exit 1
+fi
+
+# Calculate SHA-256
+echo "Calculating XZ SHA sum ..."
+PACKAGE_XZ_PATH="${OUTPUT_DIR:?}/$PACKAGE_XZ"
+PACKAGE_XZ_SHA=$(shasum -a 256 "$PACKAGE_XZ" | cut -f 1 -d ' ')
+PACKAGE_XZ_SIZE=$(get_file_size "$PACKAGE_XZ")
+popd >/dev/null
+echo "'$PACKAGE_XZ' Created! Size: $PACKAGE_XZ_SIZE, SHA-256: $PACKAGE_XZ_SHA"
+echo
+
+# Upload ZIP package to release page
+echo "Uploading ZIP package to release page ..."
 PACKAGE_URL=$(git_safe_upload_asset "$PACKAGE_PATH")
 echo "Package Uploaded"
 echo "Download URL: $PACKAGE_URL"
 echo
 
+# Upload XZ package to release page
+echo "Uploading XZ package to release page ..."
+PACKAGE_XZ_URL=$(git_safe_upload_asset "$PACKAGE_XZ_PATH")
+echo "Package Uploaded"
+echo "Download URL: $PACKAGE_XZ_URL"
+echo
+
+# Remove package folder
+rm -rf "$PKG_DIR"
+
+# Copy Libs from lib-builder to release in ZIP and XZ
+
+libs_url=$(cat "$PACKAGE_JSON_TEMPLATE" | jq -r ".packages[0].tools[] | select(.name == \"esp32-arduino-libs\") | .systems[0].url")
+libs_sha=$(cat "$PACKAGE_JSON_TEMPLATE" | jq -r ".packages[0].tools[] | select(.name == \"esp32-arduino-libs\") | .systems[0].checksum" | sed 's/^SHA-256://')
+libs_size=$(cat "$PACKAGE_JSON_TEMPLATE" | jq -r ".packages[0].tools[] | select(.name == \"esp32-arduino-libs\") | .systems[0].size")
+echo "Downloading libs from lib-builder ..."
+echo "URL: $libs_url"
+echo "Expected SHA: $libs_sha"
+echo "Expected Size: $libs_size"
+echo
+
+echo "Downloading libs from lib-builder ..."
+curl -L -o "$OUTPUT_DIR/$LIBS_ZIP" "$libs_url"
+
+# Check SHA and Size
+zip_sha=$(sha256sum "$OUTPUT_DIR/$LIBS_ZIP" | awk '{print $1}')
+zip_size=$(stat -c%s "$OUTPUT_DIR/$LIBS_ZIP")
+echo "Downloaded SHA: $zip_sha"
+echo "Downloaded Size: $zip_size"
+if [ "$zip_sha" != "$libs_sha" ] || [ "$zip_size" != "$libs_size" ]; then
+    echo "ERROR: ZIP SHA and Size do not match"
+    exit 1
+fi
+
+# Extract ZIP
+
+echo "Repacking libs to XZ ..."
+unzip -q "$OUTPUT_DIR/$LIBS_ZIP" -d "$OUTPUT_DIR"
+pushd "$OUTPUT_DIR" >/dev/null
+tar -cJf "$LIBS_XZ" "esp32-arduino-libs"
+popd >/dev/null
+
+# Upload ZIP and XZ libs to release page
+
+echo "Uploading ZIP libs to release page ..."
+LIBS_ZIP_URL=$(git_safe_upload_asset "$OUTPUT_DIR/$LIBS_ZIP")
+echo "ZIP libs Uploaded"
+echo "Download URL: $LIBS_ZIP_URL"
+echo
+
+echo "Uploading XZ libs to release page ..."
+LIBS_XZ_URL=$(git_safe_upload_asset "$OUTPUT_DIR/$LIBS_XZ")
+echo "XZ libs Uploaded"
+echo "Download URL: $LIBS_XZ_URL"
+echo
+
+# Update libs URLs in JSON template
+echo "Updating libs URLs in JSON template ..."
+
+# Update all libs URLs in the JSON template
+libs_jq_arg="(.packages[0].tools[] | select(.name == \"esp32-arduino-libs\") | .systems[].url) = \"$LIBS_ZIP_URL\""
+
+cat "$PACKAGE_JSON_TEMPLATE" | jq "$libs_jq_arg" > "$OUTPUT_DIR/package-libs-updated.json"
+PACKAGE_JSON_TEMPLATE="$OUTPUT_DIR/package-libs-updated.json"
+
+echo "Libs URLs updated in JSON template"
+echo
+
+# Clean up
+rm -rf "${OUTPUT_DIR:?}/esp32-arduino-libs"
+rm -rf "${OUTPUT_DIR:?}/$LIBS_ZIP"
+rm -rf "${OUTPUT_DIR:?}/$LIBS_XZ"
+
 ##
 ## TEMP WORKAROUND FOR RV32 LONG PATH ON WINDOWS
 ##
@@ -469,6 +606,17 @@ if [ "$RELEASE_PRE" == "false" ]; then
     echo
 fi
 
+if [ "$need_update_commit" == "true" ]; then
+    echo "Pushing version update commit..."
+    git push
+    new_tag_commit=$(git rev-parse HEAD)
+    echo "New commit: $new_tag_commit"
+
+    echo "Moving tag $RELEASE_TAG to $new_tag_commit..."
+    git tag -f "$RELEASE_TAG" "$new_tag_commit"
+    git push --force origin "$RELEASE_TAG"
+fi
+
 set +e
 
 ##

.github/scripts/tests_matrix.sh

@@ -17,6 +17,7 @@ targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2','esp32p4'"
 mkdir -p info
 
 echo "[$wokwi_types]" > info/wokwi_types.txt
+echo "[$hw_types]" > info/hw_types.txt
 echo "[$targets]" > info/targets.txt
 
 {

.github/scripts/tests_run.sh

@@ -88,10 +88,7 @@ function run_test {
         fi
 
         if [ $platform == "wokwi" ]; then
-            extra_args=("--target" "$target" "--embedded-services" "arduino,wokwi" "--wokwi-timeout=$wokwi_timeout")
-            if [[ -f "$sketchdir/scenario.yaml" ]]; then
-                extra_args+=("--wokwi-scenario" "$sketchdir/scenario.yaml")
-            fi
+            extra_args=("--target" "$target" "--embedded-services" "arduino,wokwi")
             if [[ -f "$sketchdir/diagram.$target.json" ]]; then
                 extra_args+=("--wokwi-diagram" "$sketchdir/diagram.$target.json")
             fi
@@ -137,7 +134,6 @@ SCRIPTS_DIR="./.github/scripts"
 COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
 
 platform="hardware"
-wokwi_timeout=60000
 chunk_run=0
 options=0
 erase=0
@@ -156,7 +152,6 @@ while [ -n "$1" ]; do
         ;;
     -W )
         shift
-        wokwi_timeout=$1
         if [[ -z $WOKWI_CLI_TOKEN ]]; then
             echo "Wokwi CLI token is not set"
             exit 1

.github/scripts/update-version.sh

@@ -2,6 +2,10 @@
 # Disable shellcheck warning about using 'cat' to read a file.
 # shellcheck disable=SC2002
 
+# Exit on any error and make pipelines fail if any command fails
+set -e
+set -o pipefail
+
 # For reference: add tools for all boards by replacing one line in each board
 # "[board].upload.tool=esptool_py" to "[board].upload.tool=esptool_py\n[board].upload.tool.default=esptool_py\n[board].upload.tool.network=esp_ota"
 #cat boards.txt | sed "s/\([a-zA-Z0-9_\-]*\)\.upload\.tool\=esptool_py/\1\.upload\.tool\=esptool_py\\n\1\.upload\.tool\.default\=esptool_py\\n\1\.upload\.tool\.network\=esp_ota/"
@@ -38,6 +42,11 @@ echo "Updating issue template..."
 cat .github/ISSUE_TEMPLATE/Issue-report.yml | \
 sed "s/.*\- latest master .*/        - latest master \(checkout manually\)\\n        - v$ESP_ARDUINO_VERSION/g" > __issue-report.yml && mv __issue-report.yml .github/ISSUE_TEMPLATE/Issue-report.yml
 
+echo "Updating GitLab variables..."
+cat .gitlab/workflows/common.yml | \
+sed "s/ESP_IDF_VERSION:.*/ESP_IDF_VERSION: \"$ESP_IDF_VERSION\"/g" | \
+sed "s/ESP_ARDUINO_VERSION:.*/ESP_ARDUINO_VERSION: \"$ESP_ARDUINO_VERSION\"/g" > .gitlab/workflows/__common.yml && mv .gitlab/workflows/__common.yml .gitlab/workflows/common.yml
+
 echo "Updating platform.txt..."
 cat platform.txt | sed "s/version=.*/version=$ESP_ARDUINO_VERSION/g" > __platform.txt && mv __platform.txt platform.txt