merging main

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.3", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.16", "8.11.4", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1761,6 +1761,22 @@ steps:
         env:
           BWC_VERSION: 8.11.3
 
+      - label: "{{matrix.image}} / 8.11.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.4
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1082,6 +1082,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.3
+      - label: 8.11.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.4#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.4
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -107,5 +107,6 @@ BWC_VERSION:
   - "8.11.1"
   - "8.11.2"
   - "8.11.3"
+  - "8.11.4"
   - "8.12.0"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.16"
-  - "8.11.3"
+  - "8.11.4"
   - "8.12.0"
   - "8.13.0"

build-conventions/build.gradle

@@ -92,3 +92,7 @@ project.getPlugins().withType(JavaBasePlugin.class) {
         }
     }
 }
+
+tasks.withType(JavaCompile).configureEach {
+  options.incremental = System.getenv("JENKINS_URL") == null && System.getenv("BUILDKITE_BUILD_URL") == null && System.getProperty("isCI") == null
+}

build-tools-internal/build.gradle

@@ -229,6 +229,7 @@ sourceSets {
 
 tasks.withType(JavaCompile).configureEach {
   options.encoding = 'UTF-8'
+  options.incremental = System.getenv("JENKINS_URL") == null && System.getenv("BUILDKITE_BUILD_URL") == null && System.getProperty("isCI") == null
 }
 
 tasks.named('licenseHeaders').configure {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -132,6 +132,7 @@ public static void configureCompile(Project project) {
             compileTask.getConventionMapping().map("sourceCompatibility", () -> java.getSourceCompatibility().toString());
             compileTask.getConventionMapping().map("targetCompatibility", () -> java.getTargetCompatibility().toString());
             compileOptions.getRelease().set(releaseVersionProviderFromCompileTask(project, compileTask));
+            compileOptions.setIncremental(BuildParams.isCi() == false);
         });
         // also apply release flag to groovy, which is used in build-tools
         project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -124,7 +124,9 @@ public void apply(Project project) {
             params.setGitOrigin(gitInfo.getOrigin());
             params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC));
             params.setTestSeed(getTestSeed());
-            params.setIsCi(System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null);
+            params.setIsCi(
+                System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null || System.getProperty("isCI") != null
+            );
             params.setDefaultParallel(ParallelDetector.findDefaultParallel(project));
             params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixtureTask.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.gradle.internal.testfixtures;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.tasks.Internal;
+
+public abstract class TestFixtureTask extends DefaultTask {
+
+    @Internal
+    abstract DirectoryProperty getFixturesDir();
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -81,37 +81,36 @@ public void apply(Project project) {
         );
 
         ExtraPropertiesExtension ext = project.getExtensions().getByType(ExtraPropertiesExtension.class);
-        File testfixturesDir = project.file("testfixtures_shared");
-        ext.set("testFixturesDir", testfixturesDir);
+        File testFixturesDir = project.file("testfixtures_shared");
+        ext.set("testFixturesDir", testFixturesDir);
 
         if (project.file(DOCKER_COMPOSE_YML).exists()) {
             project.getPluginManager().apply(BasePlugin.class);
             project.getPluginManager().apply(DockerComposePlugin.class);
-
-            TaskProvider<Task> preProcessFixture = project.getTasks().register("preProcessFixture", t -> {
-                t.doFirst(new Action<Task>() {
-                    @Override
-                    public void execute(Task task) {
-                        try {
-                            Files.createDirectories(testfixturesDir.toPath());
-                        } catch (IOException e) {
-                            throw new UncheckedIOException(e);
-                        }
+            TaskProvider<TestFixtureTask> preProcessFixture = project.getTasks().register("preProcessFixture", TestFixtureTask.class, t -> {
+                t.getFixturesDir().set(testFixturesDir);
+                t.doFirst(task -> {
+                    try {
+                        Files.createDirectories(testFixturesDir.toPath());
+                    } catch (IOException e) {
+                        throw new UncheckedIOException(e);
                     }
                 });
             });
             TaskProvider<Task> buildFixture = project.getTasks()
                 .register("buildFixture", t -> t.dependsOn(preProcessFixture, tasks.named("composeUp")));
 
-            TaskProvider<Task> postProcessFixture = project.getTasks().register("postProcessFixture", task -> {
-                task.dependsOn(buildFixture);
-                configureServiceInfoForTask(
-                    task,
-                    project,
-                    false,
-                    (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
-                );
-            });
+            TaskProvider<TestFixtureTask> postProcessFixture = project.getTasks()
+                .register("postProcessFixture", TestFixtureTask.class, task -> {
+                    task.getFixturesDir().set(testFixturesDir);
+                    task.dependsOn(buildFixture);
+                    configureServiceInfoForTask(
+                        task,
+                        project,
+                        false,
+                        (name, port) -> task.getExtensions().getByType(ExtraPropertiesExtension.class).set(name, port)
+                    );
+                });
 
             maybeSkipTask(dockerSupport, preProcessFixture);
             maybeSkipTask(dockerSupport, postProcessFixture);
@@ -138,7 +137,7 @@ public void execute(Task task) {
                 t.mustRunAfter(preProcessFixture);
             });
             tasks.named("composePull").configure(t -> t.mustRunAfter(preProcessFixture));
-            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testfixturesDir))));
+            tasks.named("composeDown").configure(t -> t.doLast(t2 -> getFileSystemOperations().delete(d -> d.delete(testFixturesDir))));
         } else {
             project.afterEvaluate(spec -> {
                 if (extension.fixtures.isEmpty()) {
@@ -179,7 +178,7 @@ private void maybeSkipTasks(TaskContainer tasks, Provider<DockerSupportService>
         tasks.withType(taskClass).configureEach(t -> maybeSkipTask(dockerSupport, t));
     }
 
-    private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskProvider<Task> task) {
+    private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskProvider<? extends Task> task) {
         task.configure(t -> maybeSkipTask(dockerSupport, t));
     }
 

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -163,6 +163,10 @@ org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures()
 @defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
 org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
 
+@defaultMessage Do not construct this records outside the source files they are declared in
+org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#<init>(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult)
+org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.String, java.util.List, long, long, org.elasticsearch.cluster.SnapshotDeletionsInProgress$State, java.lang.String)
+
 @defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
 java.lang.Thread#<init>(java.lang.Runnable)
 java.lang.Thread#<init>(java.lang.ThreadGroup, java.lang.Runnable)

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.13.0
-lucene            = 9.9.0
+lucene            = 9.9.0-snapshot-bb4fec631e6
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d

build-tools/build.gradle

@@ -158,6 +158,10 @@ dependencies {
 
 }
 
+tasks.withType(JavaCompile).configureEach {
+  options.incremental = System.getenv("JENKINS_URL") == null && System.getenv("BUILDKITE_BUILD_URL") == null && System.getProperty("isCI") == null
+}
+
 tasks.named('test').configure {
     useJUnitPlatform()
 }

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -201,14 +201,20 @@ public void beforeStart() {
                     try {
                         mockServer.start();
                         node.setting("telemetry.metrics.enabled", "true");
+                        node.setting("tracing.apm.agent.enabled", "true");
+                        node.setting("tracing.apm.agent.transaction_sample_rate", "0.10");
+                        node.setting("tracing.apm.agent.metrics_interval", "10s");
                         node.setting("tracing.apm.agent.server_url", "http://127.0.0.1:" + mockServer.getPort());
                     } catch (IOException e) {
                         logger.warn("Unable to start APM server", e);
                     }
-                } else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) {
-                    // in serverless metrics are enabled by default
-                    // if metrics were not enabled explicitly for gradlew run we should disable them
+                }
+                // in serverless metrics are enabled by default
+                // if metrics were not enabled explicitly for gradlew run we should disable them
+                else if (node.getSettingKeys().contains("telemetry.metrics.enabled") == false) { // metrics
                     node.setting("telemetry.metrics.enabled", "false");
+                } else if (node.getSettingKeys().contains("tracing.apm.agent.enabled") == false) { // tracing
+                    node.setting("tracing.apm.agent.enable", "false");
                 }
 
             }

docs/build.gradle

@@ -111,6 +111,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
+  requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 
   extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json")
   extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")

docs/changelog/102078.yaml

@@ -0,0 +1,5 @@
+pr: 102078
+summary: Derive expected replica size from primary
+area: Allocation
+type: enhancement
+issues: []

docs/changelog/102759.yaml

@@ -0,0 +1,6 @@
+pr: 102759
+summary: Close rather than stop `HttpServerTransport` on shutdown
+area: Infra/Node Lifecycle
+type: bug
+issues:
+ - 102501

docs/changelog/102765.yaml

@@ -0,0 +1,5 @@
+pr: 102765
+summary: "Add APM metrics to `HealthPeriodicLogger`"
+area: Health
+type: enhancement
+issues: []

docs/changelog/102967.yaml

@@ -0,0 +1,6 @@
+pr: 102967
+summary: "ES|QL: Improve resolution error management in `mv_expand`"
+area: ES|QL
+type: bug
+issues:
+ - 102964

docs/changelog/103003.yaml

@@ -0,0 +1,6 @@
+pr: 103003
+summary: "Fix: Watcher REST API `GET /_watcher/settings` now includes product header"
+area: "Watcher"
+type: bug
+issues:
+ - 102928

docs/changelog/103035.yaml

@@ -0,0 +1,5 @@
+pr: 103035
+summary: "x-pack/plugin/core: add `match_mapping_type` to `ecs@mappings` dynamic templates"
+area: Data streams
+type: bug
+issues: []

docs/changelog/103084.yaml

@@ -0,0 +1,6 @@
+pr: 103084
+summary: Return `matched_queries` in Percolator
+area: Percolator
+type: enhancement
+issues:
+ - 10163

docs/changelog/103151.yaml

@@ -0,0 +1,6 @@
+pr: 103151
+summary: Wrap painless explain error
+area: Infra/Scripting
+type: bug
+issues:
+ - 103018

docs/changelog/103183.yaml

@@ -0,0 +1,6 @@
+pr: 103183
+summary: "[Connectors API] Handle nullable fields correctly in the `ConnectorSyncJob`\
+  \ parser"
+area: Application
+type: bug
+issues: []

docs/changelog/103185.yaml

@@ -0,0 +1,5 @@
+pr: 103185
+summary: Fix format string in `OldLuceneVersions`
+area: Search
+type: bug
+issues: []

docs/changelog/103190.yaml

@@ -0,0 +1,5 @@
+pr: 103190
+summary: ILM/SLM history policies forcemerge in hot and dsl configuration
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/103203.yaml

@@ -0,0 +1,5 @@
+pr: 103203
+summary: Fix NPE & empty result handling in `CountOnlyQueryPhaseResultConsumer`
+area: Search
+type: bug
+issues: []

docs/changelog/103209.yaml

@@ -0,0 +1,6 @@
+pr: 103209
+summary: "ESQL: Fix `to_degrees()` returning infinity"
+area: ES|QL
+type: bug
+issues:
+ - 102987

docs/changelog/103251.yaml

@@ -0,0 +1,5 @@
+pr: 103251
+summary: Wait for reroute before acking put-shutdown
+area: Infra/Node Lifecycle
+type: bug
+issues: []

docs/changelog/103310.yaml

@@ -0,0 +1,5 @@
+pr: 103310
+summary: Revert "Validate settings in `ReloadSecureSettings` API"
+area: Security
+type: bug
+issues: []