Merge branch 'main' into reduce-NeighborArray-memory

* main: (27 commits)
  deps(java): bump com.github.luben:zstd-jni from 1.5.7-2 to 1.5.7-3 (apache#14621)
  Improve user-facing docs for geo package (apache#14534)
  Enabling histogram collection for PointRangeQuery (apache#14560)
  Move sloppySin into SloppyMath from GeoUtils (apache#14516)
  Rewrite APIJAR extractor to use Java 24 classfile API and kill ASM dependency also for build system (apache#14613)
  CHANGES entry for apache#14226 (optimistic KNN Query)
  OptimisticKnnVectorQuery (apache#14226)
  Fix for Windows (spaces in paths) apache#14608
  Update jdk requirements in README to OpenJDK 24 (apache#14610)
  Always check gradle wrapper sha checksum and download if necessary (apache#14608)
  Fix changelog verifier (apache#14606)
  MultiRange query for SortedNumeric DocValues (apache#14404)
  Remove RANDOM_PRELOAD read advice, which is not actually used (apache#14593)
  Remove duplicate test (apache#14602)
  Refactor the expressions compiler to use official ClassData BSM with indexed lookup (apache#14602)
  Disallow EA versions to run Gradle (apache#14601)
  Add back-compat indices for 10.2.1
  Add Lucene 10.2.1 version constant
  DOAP changes for release 10.2.1
  Revert "An attempt to make jenkins pass with the currently installed jdk24-ea. To be reverted later. apache#14600"
  ...

Author: weizijun

.github/workflows/verify-changelog-and-set-milestone.yml

@@ -6,6 +6,7 @@ on:
 env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   ISSUE: ${{ github.event.issue.html_url }}
+  BASE_REPO: ${{ github.repository }}
   SKIP_CHANGELOG_CHECK_LABEL: ${{ '"skip-changelog-check"' }}
   CHANGE_LOG_FILE: ${{ 'lucene/CHANGES.txt' }}
 
@@ -25,7 +26,8 @@ jobs:
         run: |
           echo "################## STEP 1 ##################"
           echo "Get all labels from PR #${{ github.event.number }}"
-          mapfile -t labels < <(gh pr view ${{ github.event.number }} --json labels -q '.labels[].name')
+          # Use the BASE_REPO variable to ensure it expands correctly
+          mapfile -t labels < <(gh pr view ${{ github.event.number }} --repo "$BASE_REPO" --json labels -q '.labels[].name')
           IFS=','; echo "${labels[*]}"
           for label in "${labels[@]}"; do
             if [[ $label == ${{ env.SKIP_CHANGELOG_CHECK_LABEL }} ]]; then
@@ -38,10 +40,25 @@ jobs:
           echo -e "\n"
           echo "################## STEP 2 ##################"
           echo "Checking for change log entry in ${{ env.CHANGE_LOG_FILE }}"
-          git fetch origin ${{ github.event.pull_request.base.ref }}
-          git log  --pretty=oneline | tail -n 2 | cat
-          echo "merge base sha: ${{ github.event.pull_request.base.sha }}, merge head sha: ${{ github.event.pull_request.head.sha }}"
-          if ! git diff ${{ github.event.pull_request.base.sha }} --name-only | grep -q "${{ env.CHANGE_LOG_FILE }}"; then
+          git remote add upstream https://github.com/${{ github.repository }}.git
+
+          # Add and fetch full history of the base branch
+          BASE_BRANCH=${{ github.event.pull_request.base.ref }}
+          # We need this to ensure we will be able to find the merge base
+          git fetch --unshallow || true
+          git fetch upstream $BASE_BRANCH
+
+          # Find merge base commit to diff against
+          BASE_COMMIT=$(git merge-base HEAD upstream/$BASE_BRANCH)
+          echo "Using merge base for comparison: $BASE_COMMIT"
+          echo "Last 3 commits up to merge base:"
+          git log --oneline -n 3 $BASE_COMMIT
+          echo "Last 3 commits up to PR head:"
+          git log --oneline -n 3 HEAD
+
+          echo "Diff:"
+          git diff --name-only $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}
+          if ! git diff --name-only $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}; then
             echo "Change log file:${{ env.CHANGE_LOG_FILE }} does not contains an entry corresponding to changes introduced in PR. Please add a changelog entry."
             exit 0
           else
@@ -54,7 +71,9 @@ jobs:
           echo "Extracting Lucene version from change log entry"
           # git diff header pattern -> "@@ -15,0 +16,4 @@" 
           # try to extract the line number at which new entry is added, here it's line number 16
-          diff=$(git diff ${{ github.event.pull_request.base.sha }} --unified=0 -- ${{ env.CHANGE_LOG_FILE }})
+          echo "Diff:"
+          git diff --unified=0 $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }}
+          diff=$(git diff --unified=0 $BASE_COMMIT HEAD -- ${{ env.CHANGE_LOG_FILE }})
           lucene_version=""
           diff_header_pattern="@@ -[0-9]+,?[0-9]* \+([0-9]*),?[0-9]* @@"
           if [[ $diff =~ $diff_header_pattern ]]; then

README.md

@@ -40,7 +40,7 @@ comprehensive documentation, visit:
 
 ### Basic steps:
 
-1. Install OpenJDK 23.
+1. Install [OpenJDK 24](https://jdk.java.net/archive/).
 2. Clone Lucene's git repository (or download the source distribution).
 3. Run gradle launcher script (`gradlew`).
 

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -35,7 +35,9 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
 
 /**
  * Standalone class that can be used to download a gradle-wrapper.jar
@@ -66,12 +68,47 @@ public static void checkVersion() {
   }
 
   public void run(Path destination) throws IOException, NoSuchAlgorithmException {
-    Path checksumPath =
-        destination.resolveSibling(destination.getFileName().toString() + ".sha256");
+    var expectedFileName = destination.getFileName().toString();
+    Path checksumPath = destination.resolveSibling(expectedFileName + ".sha256");
     if (!Files.exists(checksumPath)) {
       throw new IOException("Checksum file not found: " + checksumPath);
     }
-    String expectedChecksum = Files.readString(checksumPath, StandardCharsets.UTF_8).trim();
+
+    String expectedChecksum;
+    try (var lines = Files.lines(checksumPath, StandardCharsets.UTF_8)) {
+      expectedChecksum =
+          lines
+              .map(
+                  line -> {
+                    // "The default mode is to print a line with: checksum, a space,
+                    // a character indicating input mode  ('*' for binary, ' ' for text
+                    // or where binary is insignificant), and name for each FILE."
+                    var spaceIndex = line.indexOf(" ");
+                    if (spaceIndex != -1 && spaceIndex + 2 < line.length()) {
+                      var mode = line.charAt(spaceIndex + 1);
+                      String fileName = line.substring(spaceIndex + 2);
+                      if (mode == '*' && fileName.equals(expectedFileName)) {
+                        return line.substring(0, spaceIndex);
+                      }
+                    }
+
+                    Logger.getLogger(WrapperDownloader.class.getName())
+                        .warning(
+                            "Something is wrong with the checksum file. Regenerate with "
+                                + "'sha256sum -b gradle-wrapper.jar > gradle-wrapper.jar.sha256'");
+                    return null;
+                  })
+              .filter(Objects::nonNull)
+              .findFirst()
+              .orElse(null);
+
+      if (expectedChecksum == null) {
+        throw new IOException(
+            "The checksum file did not contain the expected checksum for '"
+                + expectedFileName
+                + "'?");
+      }
+    }
 
     Path versionPath =
         destination.resolveSibling(destination.getFileName().toString() + ".version");

dev-tools/doap/lucene.rdf

@@ -67,6 +67,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+       <Version>
+         <name>lucene-10.2.1</name>
+         <created>2025-05-01</created>
+         <revision>10.2.1</revision>
+       </Version>
+    </release>
     <release>
        <Version>
          <name>lucene-10.2.0</name>

dev-tools/scripts/requirements.txt

@@ -1,9 +1,18 @@
+# jinja template processing of releaseWizard.yaml
 Jinja2==3.1.6
+# parsing and processing of releaseWizard.yaml
 PyYAML==6.0.2
+# international holidays in releaseWizard 
 holidays==0.71
+# calendar processing in releaseWizard
 ics==0.7.2
+# terminal processing in releaseWizard
 console-menu==0.8.0
+# pull request processing in githubPRs
 PyGithub==2.6.1
+# JIRA processing in githubPRs
 jira==3.8.0
+# type-checking in "make lint"
 basedpyright==1.29.1
+# linting in "make lint"
 ruff==0.11.7

gradle/generation/extract-jdk-apis.gradle

@@ -23,18 +23,10 @@ configure(project(":lucene:core")) {
     mrjarJavaVersions = [ 24 ]
   }
 
-  configurations {
-    apiextractor
-  }
-
-  dependencies {
-    apiextractor deps.asm.core
-  }
-
   plugins.withType(JavaPlugin) {
     mrjarJavaVersions.each { jdkVersion ->
       def task = tasks.create(name: "generateJdkApiJar${jdkVersion}", type: JavaExec) {
-        description "Regenerate the API-only JAR file with public Panama Foreign & Vector API from JDK ${jdkVersion}"
+        description "Regenerate the API-only JAR file with public Panama Vector API from JDK ${jdkVersion}"
         group "generation"
 
         javaLauncher = javaToolchains.launcherFor {
@@ -46,20 +38,20 @@ configure(project(":lucene:core")) {
             javaLauncher.get()
             return true
           } catch (Exception e) {
-            logger.warn('Launcher for Java {} is not available; skipping regeneration of Panama Foreign & Vector API JAR.', jdkVersion)
+            logger.warn('Launcher for Java {} is not available; skipping regeneration of Panama Vector API JAR.', jdkVersion)
             logger.warn('Error: {}', e.cause?.message)
             logger.warn("Please make sure to point env 'JAVA{}_HOME' to exactly JDK version {} or enable Gradle toolchain auto-download.", jdkVersion, jdkVersion)
             return false
           }
         }
 
-        classpath = configurations.apiextractor
         mainClass = file("${resources}/ExtractJdkApis.java") as String
         systemProperties = [
                 'user.timezone': 'UTC',
                 'file.encoding': 'UTF-8',
         ]
         args = [
+                rootProject.minJavaVersion.toString(),
                 jdkVersion,
                 apijars.file("jdk${jdkVersion}.apijar"),
         ]