Merge branch 'main' into save-fake-tasks-to-create-task-index

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.22", "8.13.4", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.4
+      - label: "{{matrix.image}} / 8.13.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,7 +543,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.4
+          BWC_VERSION: 8.13.5
 
       - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0

.buildkite/pipelines/periodic.template.yml

@@ -88,6 +88,7 @@ steps:
               - openjdk17
               - openjdk21
               - openjdk22
+              - openjdk23
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -113,6 +114,7 @@ steps:
               - openjdk17
               - openjdk21
               - openjdk22
+              - openjdk23
             BWC_VERSION: $BWC_LIST
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -591,8 +591,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.13.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.4#bwcTest
+      - label: 8.13.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -601,7 +601,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.13.4
+          BWC_VERSION: 8.13.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -714,7 +714,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.22", "8.13.4", "8.14.0", "8.15.0"]
+            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -735,6 +735,7 @@ steps:
               - openjdk17
               - openjdk21
               - openjdk22
+              - openjdk23
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -760,7 +761,8 @@ steps:
               - openjdk17
               - openjdk21
               - openjdk22
-            BWC_VERSION: ["7.17.22", "8.13.4", "8.14.0", "8.15.0"]
+              - openjdk23
+            BWC_VERSION: ["7.17.22", "8.13.5", "8.14.0", "8.15.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -30,6 +30,6 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.4"
+  - "8.13.5"
   - "8.14.0"
   - "8.15.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.22"
-  - "8.13.4"
+  - "8.13.5"
   - "8.14.0"
   - "8.15.0"

TESTING.asciidoc

@@ -551,13 +551,19 @@ When running `./gradlew check`, minimal bwc checks are also run against compatib
 
 ==== BWC Testing against a specific remote/branch
 
-Sometimes a backward compatibility change spans two versions. A common case is a new functionality
-that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
-To test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of
-pulling the release branch from GitHub. You do so using the `bwc.remote` and `bwc.refspec.BRANCH` system properties:
+Sometimes a backward compatibility change spans two versions.
+A common case is a new functionality that needs a BWC bridge in an unreleased versioned of a release branch (for example, 5.x).
+Another use case, since the introduction of serverless, is to test BWC against main in addition to the other released branches.
+To do so, specify the `bwc.refspec` remote and branch to use for the BWC build as `origin/main`.
+To test against main, you will also need to create a new version in link:./server/src/main/java/org/elasticsearch/Version.java[Version.java], 
+increment `elasticsearch` in link:./build-tools-internal/version.properties[version.properties], and hard-code the `project.version` for ml-cpp 
+in link:./x-pack/plugin/ml/build.gradle[ml/build.gradle].
+
+In general, to test the changes, you can instruct Gradle to build the BWC version from another remote/branch combination instead of pulling the release branch from GitHub.
+You do so using the `bwc.refspec.{VERSION}` system property:
 
 -------------------------------------------------
-./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x
+./gradlew check -Dtests.bwc.refspec.8.15=origin/main
 -------------------------------------------------
 
 The branch needs to be available on the remote that the BWC makes of the

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.LowercaseNormalizer;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
@@ -71,7 +72,8 @@ public static MapperService create(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.MapperMetrics;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -154,7 +155,8 @@ protected SearchExecutionContext buildSearchExecutionContext() {
             null,
             () -> true,
             null,
-            Collections.emptyMap()
+            Collections.emptyMap(),
+            MapperMetrics.NOOP
         );
     }
 
@@ -186,7 +188,8 @@ protected final MapperService createMapperService(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            }
+            },
+            MapperMetrics.NOOP
         );
 
         try {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java

@@ -60,6 +60,7 @@
 import static org.gradle.api.JavaVersion.VERSION_20;
 import static org.gradle.api.JavaVersion.VERSION_21;
 import static org.gradle.api.JavaVersion.VERSION_22;
+import static org.gradle.api.JavaVersion.VERSION_23;
 
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
@@ -336,8 +337,8 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), classpath);
-            // Enable explicitly for each release as appropriate. Just JDK 20/21/22 for now, and just the vector module.
-            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22)) {
+            // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
+            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
             }
             spec.jvmArgs("-Xmx1g");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/AdoptiumJdkToolchainResolver.java

@@ -11,7 +11,6 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.commons.compress.utils.Lists;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainDownload;
 import org.gradle.jvm.toolchain.JavaToolchainRequest;
@@ -21,25 +20,25 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
-import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.StreamSupport;
 
 import static org.gradle.jvm.toolchain.JavaToolchainDownload.fromUri;
 
 public abstract class AdoptiumJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
     // package protected for better testing
-    final Map<AdoptiumVersionRequest, Optional<AdoptiumVersionInfo>> CACHED_SEMVERS = new ConcurrentHashMap<>();
+    final Map<AdoptiumVersionRequest, Optional<String>> CACHED_RELEASES = new ConcurrentHashMap<>();
 
     @Override
     public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         if (requestIsSupported(request) == false) {
             return Optional.empty();
         }
         AdoptiumVersionRequest versionRequestKey = toVersionRequest(request);
-        Optional<AdoptiumVersionInfo> versionInfo = CACHED_SEMVERS.computeIfAbsent(
+        Optional<String> versionInfo = CACHED_RELEASES.computeIfAbsent(
             versionRequestKey,
             (r) -> resolveAvailableVersion(versionRequestKey)
         );
@@ -54,12 +53,12 @@ private AdoptiumVersionRequest toVersionRequest(JavaToolchainRequest request) {
         return new AdoptiumVersionRequest(platform, arch, javaLanguageVersion);
     }
 
-    private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
+    private Optional<String> resolveAvailableVersion(AdoptiumVersionRequest requestKey) {
         ObjectMapper mapper = new ObjectMapper();
         try {
             int languageVersion = requestKey.languageVersion.asInt();
             URL source = new URL(
-                "https://api.adoptium.net/v3/info/release_versions?architecture="
+                "https://api.adoptium.net/v3/info/release_names?architecture="
                     + requestKey.arch
                     + "&image_type=jdk&os="
                     + requestKey.platform
@@ -71,14 +70,8 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
                     + ")"
             );
             JsonNode jsonNode = mapper.readTree(source);
-            JsonNode versionsNode = jsonNode.get("versions");
-            return Optional.of(
-                Lists.newArrayList(versionsNode.iterator())
-                    .stream()
-                    .map(this::toVersionInfo)
-                    .max(Comparator.comparing(AdoptiumVersionInfo::semver))
-                    .get()
-            );
+            JsonNode versionsNode = jsonNode.get("releases");
+            return StreamSupport.stream(versionsNode.spliterator(), false).map(JsonNode::textValue).findFirst();
         } catch (FileNotFoundException e) {
             // request combo not supported (e.g. aarch64 + windows
             return Optional.empty();
@@ -87,21 +80,10 @@ private Optional<AdoptiumVersionInfo> resolveAvailableVersion(AdoptiumVersionReq
         }
     }
 
-    private AdoptiumVersionInfo toVersionInfo(JsonNode node) {
-        return new AdoptiumVersionInfo(
-            node.get("build").asInt(),
-            node.get("major").asInt(),
-            node.get("minor").asInt(),
-            node.get("openjdk_version").asText(),
-            node.get("security").asInt(),
-            node.get("semver").asText()
-        );
-    }
-
-    private URI resolveDownloadURI(AdoptiumVersionRequest request, AdoptiumVersionInfo versionInfo) {
+    private URI resolveDownloadURI(AdoptiumVersionRequest request, String version) {
         return URI.create(
-            "https://api.adoptium.net/v3/binary/version/jdk-"
-                + versionInfo.semver
+            "https://api.adoptium.net/v3/binary/version/"
+                + version
                 + "/"
                 + request.platform
                 + "/"
@@ -118,7 +100,5 @@ private boolean requestIsSupported(JavaToolchainRequest request) {
         return anyVendorOr(request.getJavaToolchainSpec().getVendor().get(), JvmVendorSpec.ADOPTIUM);
     }
 
-    record AdoptiumVersionInfo(int build, int major, int minor, String openjdkVersion, int security, String semver) {}
-
     record AdoptiumVersionRequest(String platform, String arch, JavaLanguageVersion languageVersion) {}
 }