Merge remote-tracking branch 'origin/main' into lucene_snapshot

elastic · May 18, 2024 · c9cd48a · c9cd48a
2 parents 0ce4398 + 064655e
commit c9cd48a
Show file tree

Hide file tree

Showing 207 changed files with 5,017 additions and 1,527 deletions.
diff --git a/...main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java b/...main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java
@@ -26,7 +26,53 @@
 
 public abstract class OracleOpenJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    record JdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) {}
+    interface JdkBuild {
+        JavaLanguageVersion languageVersion();
+
+        String url(String os, String arch, String extension);
+    }
+
+    record ReleasedJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.oracle.com/java/GA/jdk"
+                + version
+                + "/"
+                + hash
+                + "/"
+                + buildNumber
+                + "/GPL/openjdk-"
+                + version
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
+
+    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.java.net/java/early_access/jdk"
+                + version
+                + "/"
+                + version
+                + "/GPL/openjdk-"
+                + version
+                + "-ea+"
+                + buildNumber
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
 
     private static final Pattern VERSION_PATTERN = Pattern.compile(
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
@@ -39,7 +85,11 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(getBundledJdkBuild());
+    List<JdkBuild> builds = List.of(
+        getBundledJdkBuild(),
+        // 23 early access
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "23")
+    );
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();
@@ -51,7 +101,7 @@ private JdkBuild getBundledJdkBuild() {
         String baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
         String build = jdkVersionMatcher.group(3);
         String hash = jdkVersionMatcher.group(5);
-        return new JdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
+        return new ReleasedJdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
     }
 
     /**
@@ -68,24 +118,7 @@ public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         String extension = operatingSystem.equals(OperatingSystem.WINDOWS) ? "zip" : "tar.gz";
         String arch = toArchString(request.getBuildPlatform().getArchitecture());
         String os = toOsString(operatingSystem);
-        return Optional.of(
-            () -> URI.create(
-                "https://download.oracle.com/java/GA/jdk"
-                    + build.version
-                    + "/"
-                    + build.hash
-                    + "/"
-                    + build.buildNumber
-                    + "/GPL/openjdk-"
-                    + build.version
-                    + "_"
-                    + os
-                    + "-"
-                    + arch
-                    + "_bin."
-                    + extension
-            )
-        );
+        return Optional.of(() -> URI.create(build.url(os, arch, extension)));
     }
 
     /**
@@ -113,7 +146,7 @@ private JdkBuild findSupportedBuild(JavaToolchainRequest request) {
 
         JavaLanguageVersion languageVersion = javaToolchainSpec.getLanguageVersion().get();
         for (JdkBuild build : builds) {
-            if (build.languageVersion.equals(languageVersion)) {
+            if (build.languageVersion().equals(languageVersion)) {
                 return build;
             }
         }

diff --git a/...ovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy b/...ovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy
@@ -25,7 +25,8 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
             }
         }
         toolChain.builds = [
-            new OracleOpenJdkToolchainResolver.JdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d")
+            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d"),
+            new OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild(JavaLanguageVersion.of(21), "21", "6")
         ]
         toolChain
     }
@@ -40,7 +41,18 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
          [20, anyVendor(), MAC_OS, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_macos-aarch64_bin.tar.gz"],
          [20, anyVendor(), LINUX, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-x64_bin.tar.gz"],
          [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"],
-         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"]
+         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"],
+         // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz
+         [21, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"],
+         [21, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"]
         ]
     }
 

diff --git a/docs/changelog/107240.yaml b/docs/changelog/107240.yaml
@@ -0,0 +1,6 @@
+pr: 107240
+summary: Include doc size info in ingest stats
+area: Ingest Node
+type: enhancement
+issues:
+ - 106386
diff --git a/docs/changelog/108161.yaml b/docs/changelog/108161.yaml
@@ -0,0 +1,5 @@
+pr: 108161
+summary: Refactor TextEmbeddingResults to use primitives rather than objects
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/108417.yaml b/docs/changelog/108417.yaml
@@ -0,0 +1,6 @@
+pr: 108417
+summary: Track source for arrays of objects
+area: Mapping
+type: enhancement
+issues:
+ - 90708
diff --git a/docs/changelog/108472.yaml b/docs/changelog/108472.yaml
@@ -0,0 +1,5 @@
+pr: 108472
+summary: Add support for Azure AI Studio embeddings and completions to the inference service.
+area: Machine Learning
+type: feature
+issues: []
diff --git a/docs/changelog/108679.yaml b/docs/changelog/108679.yaml
@@ -0,0 +1,6 @@
+pr: 108679
+summary: Suppress deprecation warnings from ingest pipelines when deleting trained model
+area: Machine Learning
+type: bug
+issues:
+ - 105004
diff --git a/docs/changelog/108684.yaml b/docs/changelog/108684.yaml
@@ -0,0 +1,5 @@
+pr: 108684
+summary: Check if `CsvTests` required capabilities exist
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/108726.yaml b/docs/changelog/108726.yaml
@@ -0,0 +1,5 @@
+pr: 108726
+summary: Allow RA metrics to be reported upon parsing completed or accumulated
+area: Infra/Metrics
+type: enhancement
+issues: []
diff --git a/docs/changelog/108761.yaml b/docs/changelog/108761.yaml
@@ -0,0 +1,5 @@
+pr: 108761
+summary: Add some missing timeout params to REST API specs
+area: Infra/REST API
+type: bug
+issues: []
diff --git a/docs/changelog/108780.yaml b/docs/changelog/108780.yaml
@@ -0,0 +1,6 @@
+pr: 108780
+summary: Add `continent_code` support to the geoip processor
+area: Ingest Node
+type: enhancement
+issues:
+ - 85820
diff --git a/docs/changelog/108786.yaml b/docs/changelog/108786.yaml
@@ -0,0 +1,5 @@
+pr: 108786
+summary: Make ingest byte stat names more descriptive
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/reference/cluster/cluster-info.asciidoc b/docs/reference/cluster/cluster-info.asciidoc
@@ -207,6 +207,22 @@ pipeline.
 (integer)
 Total number of failed operations for the ingest pipeline.
 
+`ingested_as_first_pipeline_in_bytes`::
+(Optional, integer)
+Total number of bytes of all documents ingested by the pipeline.
+This field is only present on pipelines which are the first to process a document.
+Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
+a reroute processor, or pipelines in pipeline processors.
+
+`produced_as_first_pipeline_in_bytes`::
+(Optional, integer)
+Total number of bytes of all documents produced by the pipeline.
+This field is only present on pipelines which are the first to process a document.
+Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
+a reroute processor, or pipelines in pipeline processors.
+In situations where there are subsequent pipelines, the value represents the size of the document after all pipelines
+have run.
+
 `processors`::
 (array of objects)
 Contains information for the ingest processors for the ingest pipeline.

diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
@@ -2643,6 +2643,22 @@ pipeline.
 (integer)
 Total number of failed operations for the ingest pipeline.
 
+`ingested_as_first_pipeline_in_bytes`::
+(Optional, integer)
+Total number of bytes of all documents ingested by the pipeline.
+This field is only present on pipelines which are the first to process a document.
+Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
+a reroute processor, or pipelines in pipeline processors.
+
+`produced_as_first_pipeline_in_bytes`::
+(Optional, integer)
+Total number of bytes of all documents produced by the pipeline.
+This field is only present on pipelines which are the first to process a document.
+Thus, it is not present on pipelines which only serve as a final pipeline after a default pipeline, a pipeline run after
+a reroute processor, or pipelines in pipeline processors.
+In situations where there are subsequent pipelines, the value represents the size of the document after all pipelines
+have run.
+
 `processors`::
 (array of objects)
 Contains statistics for the ingest processors for the ingest pipeline.

diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc
@@ -33,9 +33,9 @@ DELETE _connector/my-connector
 
 [[create-connector-api-request]]
 ==== {api-request-title}
-`POST _connector`
+* `POST _connector`
 
-`PUT _connector/<connector_id>`
+* `PUT _connector/<connector_id>`
 
 
 [[create-connector-api-prereqs]]
@@ -54,7 +54,7 @@ Creates a connector document in the internal index and initializes its configura
 ==== {api-path-parms-title}
 
 `<connector_id>`::
-(Required, string) Unique identifier of a connector.
+(Optional, string) Unique identifier of a connector.
 
 
 [role="child_attributes"]
@@ -123,7 +123,8 @@ The API returns the following result:
 [source,console-result]
 ----
 {
-  "result": "created"
+  "result": "created",
+  "id": "my-connector"
 }
 ----
 ////