merging main

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -10,6 +10,8 @@
 import org.elasticsearch.gradle.util.Pair
 import org.elasticsearch.gradle.util.GradleUtils
 import org.elasticsearch.gradle.internal.test.TestUtil
+import org.elasticsearch.gradle.internal.idea.EnablePreviewFeaturesTask
+import org.elasticsearch.gradle.internal.idea.IdeaXmlUtil
 import org.jetbrains.gradle.ext.JUnit
 
 import java.nio.file.Files
@@ -144,19 +146,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   }
 
   // modifies the idea module config to enable preview features on ':libs:native' module
-  tasks.register("enablePreviewFeatures") {
+  tasks.register("enablePreviewFeatures", EnablePreviewFeaturesTask) {
     group = 'ide'
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
-
-//    ext {
-      def enablePreview = { moduleFile, languageLevel ->
-        IdeaXmlUtil.modifyXml(moduleFile) { xml ->
-          xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
-        }
-      }
-//    }
-
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.test.iml', 'JDK_21_PREVIEW')
@@ -277,46 +270,6 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   }
 }
 
-/**
- * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
- *
- * @param path Path to existing XML file
- * @param action Action to perform on parsed XML document
- * @param preface optional front matter to add after the XML declaration
- * but before the XML document, e.g. a doctype or comment
- */
-
-class IdeaXmlUtil {
-  static Node parseXml(Object xmlPath) {
-    File xmlFile = new File(xmlPath)
-    XmlParser xmlParser = new XmlParser(false, true, true)
-    xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
-    Node xml = xmlParser.parse(xmlFile)
-    return xml
-  }
-
-  static void modifyXml(Object xmlPath, Action<? super Node> action, String preface = null) {
-    File xmlFile = new File(xmlPath)
-    if (xmlFile.exists()) {
-      Node xml = parseXml(xmlPath)
-      action.execute(xml)
-
-      xmlFile.withPrintWriter { writer ->
-        def printer = new XmlNodePrinter(writer)
-        printer.namespaceAware = true
-        printer.preserveWhitespace = true
-        writer.write("<?xml version=\"1.0\"?>\n")
-
-        if (preface != null) {
-          writer.write(preface)
-        }
-        printer.print(xml)
-      }
-    }
-  }
-}
-
-
 Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
   if (gradle.parent == null) {
     // See if any of these included builds is the Elasticsearch gradle

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/EnablePreviewFeaturesTask.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.idea;
+
+import groovy.util.Node;
+import groovy.util.NodeList;
+
+import org.gradle.api.DefaultTask;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+public class EnablePreviewFeaturesTask extends DefaultTask {
+
+    public void enablePreview(String moduleFile, String languageLevel) throws IOException, ParserConfigurationException, SAXException {
+        IdeaXmlUtil.modifyXml(moduleFile, xml -> {
+            // Find the 'component' node
+            NodeList nodes = (NodeList) xml.depthFirst();
+            Node componentNode = null;
+            for (Object node : nodes) {
+                Node currentNode = (Node) node;
+                if ("component".equals(currentNode.name()) && "NewModuleRootManager".equals(currentNode.attribute("name"))) {
+                    componentNode = currentNode;
+                    break;
+                }
+            }
+
+            // Add the attribute to the 'component' node
+            if (componentNode != null) {
+                componentNode.attributes().put("LANGUAGE_LEVEL", languageLevel);
+            }
+        });
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/idea/IdeaXmlUtil.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.idea;
+
+import groovy.util.Node;
+import groovy.util.XmlParser;
+import groovy.xml.XmlNodePrinter;
+
+import org.gradle.api.Action;
+import org.xml.sax.SAXException;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.xml.parsers.ParserConfigurationException;
+
+public class IdeaXmlUtil {
+
+    static Node parseXml(String xmlPath) throws IOException, SAXException, ParserConfigurationException {
+        File xmlFile = new File(xmlPath);
+        XmlParser xmlParser = new XmlParser(false, true, true);
+        xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        Node xml = xmlParser.parse(xmlFile);
+        return xml;
+    }
+
+    /**
+     * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
+     *
+     * @param path Path to existing XML file
+     * @param action Action to perform on parsed XML document
+     * but before the XML document, e.g. a doctype or comment
+     */
+    static void modifyXml(String xmlPath, Action<? super Node> action) throws IOException, ParserConfigurationException, SAXException {
+        modifyXml(xmlPath, action, null);
+    }
+
+    /**
+     * Parses a given XML file, applies a set of changes, and writes those changes back to the original file.
+     *
+     * @param path Path to existing XML file
+     * @param action Action to perform on parsed XML document
+     * @param preface optional front matter to add after the XML declaration
+     * but before the XML document, e.g. a doctype or comment
+     */
+    static void modifyXml(String xmlPath, Action<? super Node> action, String preface) throws IOException, ParserConfigurationException,
+        SAXException {
+        File xmlFile = new File(xmlPath);
+        if (xmlFile.exists()) {
+            Node xml = parseXml(xmlPath);
+            action.execute(xml);
+
+            try (PrintWriter writer = new PrintWriter(xmlFile)) {
+                var printer = new XmlNodePrinter(writer);
+                printer.setNamespaceAware(true);
+                printer.setPreserveWhitespace(true);
+                writer.write("<?xml version=\"1.0\"?>\n");
+                if (preface != null) {
+                    writer.write(preface);
+                }
+                printer.print(xml);
+            }
+        }
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -31,6 +31,7 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.specs.Specs;
@@ -88,8 +89,8 @@ public void apply(Project project) {
         Map<String, TaskProvider<?>> versionTasks = versionTasks(project, "destructiveDistroUpgradeTest", buildParams.getBwcVersions());
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
 
-        Configuration examplePlugin = configureExamplePlugin(project);
-
+        Configuration examplePluginConfiguration = configureExamplePlugin(project);
+        FileCollection examplePluginFileCollection = examplePluginConfiguration;
         List<TaskProvider<Test>> windowsTestTasks = new ArrayList<>();
         Map<ElasticsearchDistributionType, List<TaskProvider<Test>>> linuxTestTasks = new HashMap<>();
 
@@ -102,9 +103,9 @@ public void apply(Project project) {
                     t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable()
                 );
                 addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
-                addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
+                addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePluginFileCollection.getSingleFile().toString());
                 t.exclude("**/PackageUpgradeTests.class");
-            }, distribution, examplePlugin.getDependencies());
+            }, distribution, examplePluginConfiguration.getDependencies());
 
             if (distribution.getPlatform() == Platform.WINDOWS) {
                 windowsTestTasks.add(destructiveTask);

build.gradle

@@ -365,7 +365,7 @@ tasks.register("verifyBwcTestsEnabled") {
 
 tasks.register("branchConsistency") {
   description = 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
-  group 'Verification'
+  group = 'Verification'
   dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
 }
 

client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java

@@ -89,18 +89,22 @@ public static void stopHttpServers() throws IOException {
     }
 
     public void testBuilderUsesDefaultSSLContext() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             try (RestClient client = buildRestClient()) {
                 try {
                     client.performRequest(new Request("GET", "/"));
                     fail("connection should have been rejected due to SSL handshake");
                 } catch (Exception e) {
-                    assertThat(e, instanceOf(SSLHandshakeException.class));
+                    if (inFipsJvm()) {
+                        // Bouncy Castle throw a different exception
+                        assertThat(e, instanceOf(IOException.class));
+                        assertThat(e.getCause(), instanceOf(javax.net.ssl.SSLException.class));
+                    } else {
+                        assertThat(e, instanceOf(SSLHandshakeException.class));
+                    }
                 }
             }
-
             SSLContext.setDefault(getSslContext());
             try (RestClient client = buildRestClient()) {
                 Response response = client.performRequest(new Request("GET", "/"));
@@ -112,7 +116,6 @@ public void testBuilderUsesDefaultSSLContext() throws Exception {
     }
 
     public void testBuilderSetsThreadName() throws Exception {
-        assumeFalse("https://github.com/elastic/elasticsearch/issues/49094", inFipsJvm());
         final SSLContext defaultSSLContext = SSLContext.getDefault();
         try {
             SSLContext.setDefault(getSslContext());

distribution/docker/build.gradle

@@ -45,7 +45,7 @@ if (useDra == false) {
         ivy {
           name = 'beats'
           if (useLocalArtifacts) {
-            url getLayout().getBuildDirectory().dir("artifacts").get().asFile
+            url = getLayout().getBuildDirectory().dir("artifacts").get().asFile
             patternLayout {
               artifact '/[organisation]/[module]-[revision]-[classifier].[ext]'
             }
@@ -127,7 +127,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'bin_dir'            : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
     'build_date'         : buildDate,
     'config_dir'         : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
-    'git_revision'       : buildParams.gitRevision,
+    'git_revision'       : buildParams.gitRevision.get(),
     'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0',
     'package_manager'    : base.packageManager,
     'docker_base'        : base.name().toLowerCase(),
@@ -551,6 +551,7 @@ subprojects { Project subProject ->
       inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker")
       executable = 'docker'
       outputs.file(tarFile)
+      outputs.doNotCacheIf("Build cache is disabled for export tasks") { true }
       args "save",
         "-o",
         tarFile,

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -71,7 +71,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
             maybeSetActiveProcessorCount(nodeSettings),
             maybeSetReplayFile(distroType, isHotspot),
             maybeWorkaroundG1Bug(),
-            maybeAllowSecurityManager(),
+            maybeAllowSecurityManager(useEntitlements),
             maybeAttachEntitlementAgent(useEntitlements)
         ).flatMap(s -> s).toList();
     }
@@ -140,7 +140,7 @@ private static Stream<String> maybeWorkaroundG1Bug() {
     }
 
     @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    private static Stream<String> maybeAllowSecurityManager() {
+    private static Stream<String> maybeAllowSecurityManager(boolean useEntitlements) {
         if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
             // Will become conditional on useEntitlements once entitlements can run without SM
             return Stream.of("-Djava.security.manager=allow");
@@ -167,12 +167,16 @@ private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlement
         } catch (IOException e) {
             throw new IllegalStateException("Failed to list entitlement jars in: " + dir, e);
         }
+        // We instrument classes in these modules to call the bridge. Because the bridge gets patched
+        // into java.base, we must export the bridge from java.base to these modules.
+        String modulesContainingEntitlementInstrumentation = "java.logging";
         return Stream.of(
             "-Des.entitlements.enabled=true",
             "-XX:+EnableDynamicAgentLoading",
             "-Djdk.attach.allowAttachSelf=true",
             "--patch-module=java.base=" + bridgeJar,
-            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement"
+            "--add-exports=java.base/org.elasticsearch.entitlement.bridge=org.elasticsearch.entitlement,"
+                + modulesContainingEntitlementInstrumentation
         );
     }
 }

docs/build.gradle

@@ -178,8 +178,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 }
 
 tasks.named("yamlRestTest").configure {
+  def repoFolder = "${layout.buildDirectory.asFile.get()}/cluster/shared/repo"
   doFirst {
-    delete("${buildDir}/cluster/shared/repo")
+    delete(repoFolder)
   }
 }
 

docs/changelog/117519.yaml

@@ -0,0 +1,20 @@
+pr: 117519
+summary: Remove `data_frame_transforms` roles
+area: Transform
+type: breaking
+issues: []
+breaking:
+  title: Remove `data_frame_transforms` roles
+  area: Transform
+  details: >-
+    `data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in
+    Elasticsearch 7 and are being removed in Elasticsearch 9.
+    `data_frame_transforms_admin` is now `transform_admin`.
+    `data_frame_transforms_user` is now `transform_user`.
+    Users must call the `_update` API to replace the permissions on the Transform before the
+    Transform can be started.
+  impact: >-
+    Transforms created with either the `data_frame_transforms_admin` or the
+    `data_frame_transforms_user` role will fail to start. The Transform will remain
+    in a `stopped` state, and its health will be red while displaying permission failures.
+  notable: false

docs/changelog/117949.yaml

@@ -0,0 +1,5 @@
+pr: 117949
+summary: Move `SlowLogFieldProvider` instantiation to node construction
+area: Infra/Logging
+type: bug
+issues: []

docs/changelog/118324.yaml

@@ -0,0 +1,6 @@
+pr: 118324
+summary: Allow the data type of `null` in filters
+area: ES|QL
+type: bug
+issues:
+  - 116351

docs/changelog/118652.yaml

@@ -0,0 +1,5 @@
+pr: 118652
+summary: Add Jina AI API to do inference for Embedding and Rerank models
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118774.yaml

@@ -0,0 +1,5 @@
+pr: 118774
+summary: Apply default k for knn query eagerly
+area: Vector Search
+type: bug
+issues: []