Merge branch 'main' into disableServerlessMLCache

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolver.java

@@ -26,7 +26,53 @@
 
 public abstract class OracleOpenJdkToolchainResolver extends AbstractCustomJavaToolchainResolver {
 
-    record JdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) {}
+    interface JdkBuild {
+        JavaLanguageVersion languageVersion();
+
+        String url(String os, String arch, String extension);
+    }
+
+    record ReleasedJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber, String hash) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.oracle.com/java/GA/jdk"
+                + version
+                + "/"
+                + hash
+                + "/"
+                + buildNumber
+                + "/GPL/openjdk-"
+                + version
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
+
+    record EarlyAccessJdkBuild(JavaLanguageVersion languageVersion, String version, String buildNumber) implements JdkBuild {
+
+        @Override
+        public String url(String os, String arch, String extension) {
+            return "https://download.java.net/java/early_access/jdk"
+                + version
+                + "/"
+                + version
+                + "/GPL/openjdk-"
+                + version
+                + "-ea+"
+                + buildNumber
+                + "_"
+                + os
+                + "-"
+                + arch
+                + "_bin."
+                + extension;
+        }
+    }
 
     private static final Pattern VERSION_PATTERN = Pattern.compile(
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
@@ -39,7 +85,11 @@ record JdkBuild(JavaLanguageVersion languageVersion, String version, String buil
     );
 
     // package private so it can be replaced by tests
-    List<JdkBuild> builds = List.of(getBundledJdkBuild());
+    List<JdkBuild> builds = List.of(
+        getBundledJdkBuild(),
+        // 23 early access
+        new EarlyAccessJdkBuild(JavaLanguageVersion.of(23), "23", "23")
+    );
 
     private JdkBuild getBundledJdkBuild() {
         String bundledJdkVersion = VersionProperties.getBundledJdkVersion();
@@ -51,7 +101,7 @@ private JdkBuild getBundledJdkBuild() {
         String baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
         String build = jdkVersionMatcher.group(3);
         String hash = jdkVersionMatcher.group(5);
-        return new JdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
+        return new ReleasedJdkBuild(bundledJdkMajorVersion, baseVersion, build, hash);
     }
 
     /**
@@ -68,24 +118,7 @@ public Optional<JavaToolchainDownload> resolve(JavaToolchainRequest request) {
         String extension = operatingSystem.equals(OperatingSystem.WINDOWS) ? "zip" : "tar.gz";
         String arch = toArchString(request.getBuildPlatform().getArchitecture());
         String os = toOsString(operatingSystem);
-        return Optional.of(
-            () -> URI.create(
-                "https://download.oracle.com/java/GA/jdk"
-                    + build.version
-                    + "/"
-                    + build.hash
-                    + "/"
-                    + build.buildNumber
-                    + "/GPL/openjdk-"
-                    + build.version
-                    + "_"
-                    + os
-                    + "-"
-                    + arch
-                    + "_bin."
-                    + extension
-            )
-        );
+        return Optional.of(() -> URI.create(build.url(os, arch, extension)));
     }
 
     /**
@@ -113,7 +146,7 @@ private JdkBuild findSupportedBuild(JavaToolchainRequest request) {
 
         JavaLanguageVersion languageVersion = javaToolchainSpec.getLanguageVersion().get();
         for (JdkBuild build : builds) {
-            if (build.languageVersion.equals(languageVersion)) {
+            if (build.languageVersion().equals(languageVersion)) {
                 return build;
             }
         }

build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/toolchain/OracleOpenJdkToolchainResolverSpec.groovy

@@ -25,7 +25,8 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
             }
         }
         toolChain.builds = [
-            new OracleOpenJdkToolchainResolver.JdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d")
+            new OracleOpenJdkToolchainResolver.ReleasedJdkBuild(JavaLanguageVersion.of(20), "20", "36", "bdc68b4b9cbc4ebcb30745c85038d91d"),
+            new OracleOpenJdkToolchainResolver.EarlyAccessJdkBuild(JavaLanguageVersion.of(21), "21", "6")
         ]
         toolChain
     }
@@ -40,7 +41,18 @@ class OracleOpenJdkToolchainResolverSpec extends AbstractToolchainResolverSpec {
          [20, anyVendor(), MAC_OS, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_macos-aarch64_bin.tar.gz"],
          [20, anyVendor(), LINUX, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-x64_bin.tar.gz"],
          [20, anyVendor(), LINUX, AARCH64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_linux-aarch64_bin.tar.gz"],
-         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"]
+         [20, anyVendor(), WINDOWS, X86_64, "https://download.oracle.com/java/GA/jdk20/bdc68b4b9cbc4ebcb30745c85038d91d/36/GPL/openjdk-20_windows-x64_bin.zip"],
+         // https://download.java.net/java/early_access/jdk23/23/GPL/openjdk-23-ea+23_macos-aarch64_bin.tar.gz
+         [21, ORACLE, MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, ORACLE, MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, ORACLE, LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, ORACLE, LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, ORACLE, WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"],
+         [21, anyVendor(), MAC_OS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-x64_bin.tar.gz"],
+         [21, anyVendor(), MAC_OS, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_macos-aarch64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-x64_bin.tar.gz"],
+         [21, anyVendor(), LINUX, AARCH64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_linux-aarch64_bin.tar.gz"],
+         [21, anyVendor(), WINDOWS, X86_64, "https://download.java.net/java/early_access/jdk21/21/GPL/openjdk-21-ea+6_windows-x64_bin.zip"]
         ]
     }
 

muted-tests.yml

@@ -10,6 +10,12 @@ tests:
 - class: "org.elasticsearch.xpack.core.ssl.SSLConfigurationReloaderTests"
   issue: "https://github.com/elastic/elasticsearch/issues/108774"
   method: "testReloadingKeyStore"
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108808"
+  method: "test {k8s-metrics.MetricsWithAggs}"
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  issue: "https://github.com/elastic/elasticsearch/issues/108809"
+  method: "test {k8s-metrics.MetricsWithoutAggs}"
 # Examples:
 #
 #  Mute a single test case in a YAML test suite:

server/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java

@@ -103,6 +103,7 @@ protected <T> T registerNewPage(Recycler.V<T> v, int page, int expectedSize) {
 
     protected final void releasePage(int page) {
         if (recycler != null) {
+            assert cache[page] != null;
             cache[page].close();
             cache[page] = null;
         }

server/src/main/java/org/elasticsearch/common/util/AbstractBigByteArray.java

@@ -10,21 +10,31 @@
 
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.recycler.Recycler;
 
+import java.io.IOException;
 import java.util.Arrays;
 
 abstract class AbstractBigByteArray extends AbstractBigArray {
 
+    protected static final byte[] ZERO_PAGE = new byte[PageCacheRecycler.BYTE_PAGE_SIZE];
+
     protected byte[][] pages;
 
     protected AbstractBigByteArray(int pageSize, BigArrays bigArrays, boolean clearOnResize, long size) {
         super(pageSize, bigArrays, clearOnResize);
         this.size = size;
         pages = new byte[numPages(size)][];
-        for (int i = 0; i < pages.length; ++i) {
-            pages[i] = newBytePage(i);
+        Arrays.fill(pages, ZERO_PAGE);
+        assert assertZeroPageClean();
+    }
+
+    private static boolean assertZeroPageClean() {
+        for (byte b : ZERO_PAGE) {
+            assert b == 0 : b;
         }
+        return true;
     }
 
     /** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@@ -35,16 +45,17 @@ public void resize(long newSize) {
             pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
         }
         for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
-            pages[i] = newBytePage(i);
+            pages[i] = ZERO_PAGE;
         }
         for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
+            assert pages[i] != ZERO_PAGE;
             pages[i] = null;
             releasePage(i);
         }
         this.size = newSize;
     }
 
-    protected final byte[] newBytePage(int page) {
+    private byte[] newBytePage(int page) {
         if (recycler != null) {
             final Recycler.V<byte[]> v = recycler.bytePage(clearOnResize);
             return registerNewPage(v, page, PageCacheRecycler.BYTE_PAGE_SIZE);
@@ -68,22 +79,40 @@ protected static void fillBySelfCopy(byte[] page, int fromBytes, int toBytes, in
     /**
      * Bulk copies array to paged array
      */
-    protected void set(long index, byte[] buf, int offset, int len, byte[][] pages, int shift) {
+    protected void set(long index, byte[] buf, int offset, int len, int shift) {
         assert index + len <= size();
         int pageIndex = pageIndex(index);
         final int indexInPage = indexInPage(index);
         if (indexInPage + len <= pageSize()) {
-            System.arraycopy(buf, offset << shift, pages[pageIndex], indexInPage << shift, len << shift);
+            System.arraycopy(buf, offset << shift, getPageForWriting(pageIndex), indexInPage << shift, len << shift);
         } else {
             int copyLen = pageSize() - indexInPage;
-            System.arraycopy(buf, offset << shift, pages[pageIndex], indexInPage, copyLen << shift);
+            System.arraycopy(buf, offset << shift, getPageForWriting(pageIndex), indexInPage, copyLen << shift);
             do {
                 ++pageIndex;
                 offset += copyLen;
                 len -= copyLen;
                 copyLen = Math.min(len, pageSize());
-                System.arraycopy(buf, offset << shift, pages[pageIndex], 0, copyLen << shift);
+                System.arraycopy(buf, offset << shift, getPageForWriting(pageIndex), 0, copyLen << shift);
             } while (len > copyLen);
         }
     }
+
+    protected byte[] getPageForWriting(int pageIndex) {
+        byte[] foundPage = pages[pageIndex];
+        if (foundPage == ZERO_PAGE) {
+            foundPage = newBytePage(pageIndex);
+            pages[pageIndex] = foundPage;
+        }
+        return foundPage;
+    }
+
+    protected void readPages(StreamInput in) throws IOException {
+        int remainedBytes = in.readVInt();
+        for (int i = 0; i < pages.length && remainedBytes > 0; i++) {
+            int len = Math.min(remainedBytes, pages[0].length);
+            in.readBytes(getPageForWriting(i), 0, len);
+            remainedBytes -= len;
+        }
+    }
 }

server/src/main/java/org/elasticsearch/common/util/BigArrays.java

@@ -21,9 +21,11 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
+import org.elasticsearch.core.Streams;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Arrays;
 
 import static org.elasticsearch.common.util.BigDoubleArray.VH_PLATFORM_NATIVE_DOUBLE;
@@ -162,8 +164,8 @@ public BytesRef next() {
         }
 
         @Override
-        public void fillWith(StreamInput in) throws IOException {
-            in.readBytes(array, 0, Math.toIntExact(size()));
+        public void fillWith(InputStream in) throws IOException {
+            Streams.readFully(in, array, 0, Math.toIntExact(size()));
         }
 
         @Override

server/src/main/java/org/elasticsearch/common/util/BigByteArray.java

@@ -10,10 +10,11 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Streams;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.Arrays;
 
 import static org.elasticsearch.common.util.BigLongArray.writePages;
@@ -49,7 +50,7 @@ public byte get(long index) {
     public byte set(long index, byte value) {
         final int pageIndex = pageIndex(index);
         final int indexInPage = indexInPage(index);
-        final byte[] page = pages[pageIndex];
+        final byte[] page = getPageForWriting(pageIndex);
         final byte ret = page[indexInPage];
         page[indexInPage] = value;
         return ret;
@@ -90,16 +91,16 @@ public void set(long index, byte[] buf, int offset, int len) {
         int pageIndex = pageIndex(index);
         final int indexInPage = indexInPage(index);
         if (indexInPage + len <= pageSize()) {
-            System.arraycopy(buf, offset, pages[pageIndex], indexInPage, len);
+            System.arraycopy(buf, offset, getPageForWriting(pageIndex), indexInPage, len);
         } else {
             int copyLen = pageSize() - indexInPage;
-            System.arraycopy(buf, offset, pages[pageIndex], indexInPage, copyLen);
+            System.arraycopy(buf, offset, getPageForWriting(pageIndex), indexInPage, copyLen);
             do {
                 ++pageIndex;
                 offset += copyLen;
                 len -= copyLen;
                 copyLen = Math.min(len, pageSize());
-                System.arraycopy(buf, offset, pages[pageIndex], 0, copyLen);
+                System.arraycopy(buf, offset, getPageForWriting(pageIndex), 0, copyLen);
             } while (len > copyLen);
         }
     }
@@ -112,13 +113,13 @@ public void fill(long fromIndex, long toIndex, byte value) {
         final int fromPage = pageIndex(fromIndex);
         final int toPage = pageIndex(toIndex - 1);
         if (fromPage == toPage) {
-            Arrays.fill(pages[fromPage], indexInPage(fromIndex), indexInPage(toIndex - 1) + 1, value);
+            Arrays.fill(getPageForWriting(fromPage), indexInPage(fromIndex), indexInPage(toIndex - 1) + 1, value);
         } else {
-            Arrays.fill(pages[fromPage], indexInPage(fromIndex), pages[fromPage].length, value);
+            Arrays.fill(getPageForWriting(fromPage), indexInPage(fromIndex), pages[fromPage].length, value);
             for (int i = fromPage + 1; i < toPage; ++i) {
-                Arrays.fill(pages[i], value);
+                Arrays.fill(getPageForWriting(i), value);
             }
-            Arrays.fill(pages[toPage], 0, indexInPage(toIndex - 1) + 1, value);
+            Arrays.fill(getPageForWriting(toPage), 0, indexInPage(toIndex - 1) + 1, value);
         }
     }
 
@@ -153,11 +154,11 @@ public BytesRef next() {
     }
 
     @Override
-    public void fillWith(StreamInput in) throws IOException {
+    public void fillWith(InputStream in) throws IOException {
         for (int i = 0; i < pages.length - 1; i++) {
-            in.readBytes(pages[i], 0, PAGE_SIZE_IN_BYTES);
+            Streams.readFully(in, getPageForWriting(i), 0, PAGE_SIZE_IN_BYTES);
         }
-        in.readBytes(pages[pages.length - 1], 0, Math.toIntExact(size - (pages.length - 1L) * PAGE_SIZE_IN_BYTES));
+        Streams.readFully(in, getPageForWriting(pages.length - 1), 0, Math.toIntExact(size - (pages.length - 1L) * PAGE_SIZE_IN_BYTES));
     }
 
     @Override