Merge remote-tracking branch 'upstream/main' into dls-expose-existing-stats

* upstream/main: (36 commits)
  Fix reproducability of builds against Java EA versions (elastic#132847)
  Speed up loading keyword fields with index sorts (elastic#132950)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testSyntheticSourceWithTranslogSnapshot elastic#132964
  Simplify EsqlSession (elastic#132848)
  Implement WriteLoadConstraintDecider#canAllocate (elastic#132041)
  Mute org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT test {p0=search/400_synthetic_source/_doc_count} elastic#132965
  Switch to PR-based benchmark pipeline defined in ES repo (elastic#132941)
  Breakdown undesired allocations by shard routing role (elastic#132235)
  Implement v_magnitude function (elastic#132765)
  Introduce execution location marker for better handling of remote/local compatibility (elastic#132205)
  Mute org.elasticsearch.cluster.ClusterInfoServiceIT testMaxQueueLatenciesInClusterInfo elastic#132957
  Unmuting simulate index data stream mapping overrides yaml rest test (elastic#132946)
  Remove CrossClusterCancellationIT.createLocalIndex() (elastic#132952)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testFetch elastic#132956
  Fix failing UT by adding a required capability (elastic#132947)
  Precompute the BitsetCacheKey hashCode (elastic#132875)
  Adding simulate ingest effective mapping (elastic#132833)
  Mute org.elasticsearch.index.mapper.LongFieldMapperTests testFetchMany elastic#132948
  Rename skipping logic to remove hard link to skip_unavailable (elastic#132861)
  Store ignored source in unique stored fields per entry (elastic#132142)
  ...

Author: szybia

.buildkite/pipelines/pull-request/performance-benchmark.yml

@@ -0,0 +1,3 @@
+steps:
+  - label: ":pipeline: TODO"
+    command: echo TODO

.buildkite/pull-requests.json

@@ -18,7 +18,7 @@
     },
     {
       "enabled": true,
-      "pipeline_slug": "elasticsearch-performance-esbench-pr",
+      "pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
       "allow_org_users": true,
       "allowed_repo_permissions": [
         "admin",

BUILDING.md

@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major
+java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+
+```
+./gradlew clean test -Druntime.java=26-ea
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +280,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramGenerationBench.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.benchmark.exponentialhistogram;
 
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -59,7 +60,7 @@ public class ExponentialHistogramGenerationBench {
     @Setup
     public void setUp() {
         random = ThreadLocalRandom.current();
-        histoGenerator = new ExponentialHistogramGenerator(bucketCount);
+        histoGenerator = ExponentialHistogramGenerator.create(bucketCount, ExponentialHistogramCircuitBreaker.noop());
 
         DoubleSupplier nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
 

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.exponentialhistogram.BucketIterator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -56,13 +57,14 @@ public class ExponentialHistogramMergeBench {
     @Setup
     public void setUp() {
         random = ThreadLocalRandom.current();
-        histoMerger = new ExponentialHistogramMerger(bucketCount);
+        ExponentialHistogramCircuitBreaker breaker = ExponentialHistogramCircuitBreaker.noop();
+        histoMerger = ExponentialHistogramMerger.create(bucketCount, breaker);
 
-        ExponentialHistogramGenerator initial = new ExponentialHistogramGenerator(bucketCount);
+        ExponentialHistogramGenerator initialGenerator = ExponentialHistogramGenerator.create(bucketCount, breaker);
         for (int j = 0; j < bucketCount; j++) {
-            initial.add(Math.pow(1.001, j));
+            initialGenerator.add(Math.pow(1.001, j));
         }
-        ExponentialHistogram initialHisto = initial.get();
+        ExponentialHistogram initialHisto = initialGenerator.getAndClear();
         int cnt = getBucketCount(initialHisto);
         if (cnt < bucketCount) {
             throw new IllegalArgumentException("Expected bucket count to be " + bucketCount + ", but was " + cnt);
@@ -72,14 +74,14 @@ public void setUp() {
         int dataPointSize = (int) Math.round(bucketCount * mergedHistoSizeFactor);
 
         for (int i = 0; i < toMerge.length; i++) {
-            ExponentialHistogramGenerator generator = new ExponentialHistogramGenerator(dataPointSize);
+            ExponentialHistogramGenerator generator = ExponentialHistogramGenerator.create(dataPointSize, breaker);
 
             int bucketIndex = 0;
             for (int j = 0; j < dataPointSize; j++) {
                 bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
                 generator.add(Math.pow(1.001, bucketIndex));
             }
-            toMerge[i] = generator.get();
+            toMerge[i] = generator.getAndClear();
             cnt = getBucketCount(toMerge[i]);
             if (cnt < dataPointSize) {
                 throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java

@@ -82,4 +82,13 @@ public void packAsBinaryLegacy(Blackhole bh) {
             bh.consume(packed);
         }
     }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void packAsBinaryPanama(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQVectorUtils.packAsBinary(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java

@@ -83,4 +83,13 @@ public void transposeHalfByteLegacy(Blackhole bh) {
             bh.consume(packed);
         }
     }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void transposeHalfBytePanama(Blackhole bh) {
+        for (int i = 0; i < numVectors; i++) {
+            BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
+            bh.consume(packed);
+        }
+    }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
-import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
     private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
+    private static final String CATALOG_EA_VERSION = "25-ea+30"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
 
+    def setup() {
+        configurationCacheCompatible = false // JDK class references configurations which break configuration cache
+    }
+
     @Unroll
-    def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
+    def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
-        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
+        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
         def mockedContent = filebytes(jdkVendor, platform)
         buildFile.text = """
             plugins {
@@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         where:
         platform  | arch      | jdkVendor       | jdkVersion           | distributionVersion   | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null                  | "bin/java"               | "(jdk 11)"
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null                  | "bin/java"               | "(jdk 15)"
+        "linux"   | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "darwin"  | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | ""                    | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | ""                    | "bin/java"               | "(jdk 15)"
         "darwin"  | "aarch64" | VENDOR_ZULU     | AZUL_JDK_VERSION_8   | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
     }
 
@@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static String urlPath(final String vendor,
                                   final String version,
                                   final String platform,
-                                  final String arch = 'x64') {
-        if (vendor.equals(VENDOR_ADOPTIUM)) {
+                                  final String arch = 'x64',
+                                  final String distributedVersion = '') {
+        final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
+
+        if (distributedVersion.equals("ea")) {
+            def effectivePlatform = isMac(platform) ? "macos" : platform;
+            def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
+            return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
+        } else if (vendor.equals(VENDOR_ADOPTIUM)) {
             final String module = isMac(platform) ? "mac" : platform;
             return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             final String effectivePlatform = isMac(platform) ? "macos" : platform;
-            final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
             return "/java/GA/" + versionPath + "/GPL/" + filename;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,6 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;
@@ -78,7 +79,9 @@ public String getVersion() {
     }
 
     public void setVersion(String version) {
-        if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
+        if (VERSION_PATTERN.matcher(version).matches() == false
+            && LEGACY_VERSION_PATTERN.matcher(version).matches() == false
+            && EA_VERSION_PATTERN.matcher(version).matches() == false) {
             throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
         }
         parseVersion(version);
@@ -112,7 +115,7 @@ public void setArchitecture(final String architecture) {
     }
 
     public String getDistributionVersion() {
-        return distributionVersion.get();
+        return distributionVersion.getOrNull();
     }
 
     public void setDistributionVersion(String distributionVersion) {
@@ -218,9 +221,17 @@ private void parseVersion(String version) {
         if (jdkVersionMatcher.matches() == false) {
             // Try again with the pre-Java9 version format
             jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
-
             if (jdkVersionMatcher.matches() == false) {
-                throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                // Try again with the pre-Java9 version format
+                jdkVersionMatcher = EA_VERSION_PATTERN.matcher(version);
+                if (jdkVersionMatcher.matches() == false) {
+                    throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                }
+                baseVersion = version;
+                major = jdkVersionMatcher.group(1);
+                build = jdkVersionMatcher.group(2);
+                hash = null;
+                return;
             }
         }