Merge branch 'main' into esql_source_meta

.ci/init.gradle

@@ -10,8 +10,6 @@ initscript {
   }
 }
 
-boolean USE_ARTIFACTORY = false
-
 if (System.getenv('VAULT_ADDR') == null) {
   // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
   if (System.getenv('CI') == null) {
@@ -50,75 +48,38 @@ final Vault vault = new Vault(
     .engineVersion(1)
     .token(vaultToken)
     .build()
-)
-  .withRetries(5, 1000)
+).withRetries(5, 1000)
 
 
-if (USE_ARTIFACTORY) {
-  final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("${vaultPathPrefix}/artifactory.elstc.co")
-    .getData()
-  logger.info("Using elastic artifactory repos")
-  Closure configCache = {
-    return {
-      name "artifactory-gradle-release"
-      url "https://artifactory.elstc.co/artifactory/gradle-release"
-      credentials {
-        username artifactoryCredentials.get("username")
-        password artifactoryCredentials.get("token")
-      }
-    }
-  }
-  settingsEvaluated { settings ->
-    settings.pluginManagement {
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-  projectsLoaded {
-    allprojects {
-      buildscript {
-        repositories {
-          maven configCache()
-        }
-      }
-      repositories {
-        maven configCache()
-      }
-    }
-  }
-}
-
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.enterprise") {
-    settings.gradleEnterprise {
-      server = 'https://gradle-enterprise.elastic.co'
-    }
+    configureGradleEnterprise(settings)
   }
 }
 
+void configureGradleEnterprise(def settings) {
+  settings.gradleEnterprise {
+    server = 'https://gradle-enterprise.elastic.co'
+    buildScan.publishAlways()
+  }
 
-final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
-final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
+  def isCI = System.getenv("CI") == "true"
+  settings.buildCache {
+    local {
+      // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
+      enabled = isCI == false
+    }
+    remote(settings.gradleEnterprise.buildCache) {
+      if (isCI) {
+        final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
+        final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
+          .read("${vaultPathPrefix}/gradle-build-cache")
+          .getData()
+        def username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
+        def password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
 
-if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
-    .read("${vaultPathPrefix}/gradle-build-cache")
-    .getData()
-  gradle.settingsEvaluated { settings ->
-    settings.buildCache {
-      local {
-        // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
-        enabled = false
-      }
-      remote(HttpBuildCache) {
-        url = buildCacheUrl
         push = buildCachePush
-        credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
-        }
+        usernameAndPassword(username, password)
       }
     }
   }

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -15,12 +15,6 @@ buildScan {
   URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
   String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
 
-  // Automatically publish scans from Elasticsearch CI
-  if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
-    publishAlways()
-    buildScan.server = 'https://gradle-enterprise.elastic.co'
-  }
-
   background {
     tag OS.current().name()
     tag Architecture.current().name()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -83,6 +83,11 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
                 return getJavaHome(Integer.parseInt(minimumCompilerVersion));
             }));
 
+            // temporally workaround for reworked gradle enterprise setup
+            // removed when PR https://github.com/elastic/elasticsearch/pull/102180 backported
+            // to all BWC branches
+            loggedExec.getEnvironment().put("BUILDKITE", "false");
+
             if (BuildParams.isCi() && OS.current() != OS.WINDOWS) {
                 // TODO: Disabled for now until we can figure out why files are getting corrupted
                 // loggedExec.getEnvironment().put("GRADLE_RO_DEP_CACHE", System.getProperty("user.home") + "/gradle_ro_cache");

docs/changelog/101904.yaml

@@ -0,0 +1,5 @@
+pr: 101904
+summary: Allow granting API keys with JWT as the access_token
+area: Security
+type: feature
+issues: []

docs/changelog/102183.yaml

@@ -0,0 +1,13 @@
+pr: 102183
+summary: "[ES|QL] pow function always returns double"
+area: ES|QL
+type: "breaking"
+issues:
+ - 99055
+breaking:
+  title: "[ES|QL] pow function always returns double"
+  area: REST API
+  details: "In ES|QL, the pow function no longer returns the type of its inputs, instead\
+    \ always returning a double."
+  impact: low. Most queries should continue to function with the change.
+  notable: false

docs/changelog/102379.yaml

@@ -0,0 +1,6 @@
+pr: 102379
+summary: Pass source query to `_field_caps` (as `index_filter`) when deducing destination index mappings for better
+  performance
+area: Transform
+type: enhancement
+issues: []

docs/changelog/102388.yaml

@@ -0,0 +1,6 @@
+pr: 102388
+summary: Add support for `index_filter` to open pit
+area: Search
+type: enhancement
+issues:
+ - 99740

docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc

@@ -127,8 +127,7 @@ init_script::       Executed prior to any collection of documents. Allows the ag
 +
 In the above example, the `init_script` creates an array `transactions` in the `state` object.
 
-map_script::        Executed once per document collected. This is a required script. If no combine_script is specified, the resulting state
-                    needs to be stored in the `state` object.
+map_script::        Executed once per document collected. This is a required script. 
 +
 In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field
 is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added

docs/reference/esql/functions/pow.asciidoc

@@ -5,7 +5,8 @@
 image::esql/functions/signature/pow.svg[Embedded,opts=inline]
 
 Returns the value of a base (first argument) raised to the power of an exponent (second argument).
-Both arguments must be numeric.
+Both arguments must be numeric. The output is always a double.  Note that it is still possible to overflow
+a double result here; in that case, null will be returned.
 
 [source.merge.styled,esql]
 ----
@@ -16,62 +17,6 @@ include::{esql-specs}/math.csv-spec[tag=powDI]
 include::{esql-specs}/math.csv-spec[tag=powDI-result]
 |===
 
-[discrete]
-==== Type rules
-
-The type of the returned value is determined by the types of the base and exponent.
-The following rules are applied to determine the result type:
-
-* If either of the base or exponent are of a floating point type, the result will be a double
-* Otherwise, if either the base or the exponent are 64-bit (long or unsigned long), the result will be a long
-* Otherwise, the result will be a 32-bit integer (this covers all other numeric types, including int, short and byte)
-
-For example, using simple integers as arguments will lead to an integer result:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/math.csv-spec[tag=powII]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/math.csv-spec[tag=powII-result]
-|===
-
-NOTE: The actual power function is performed using double precision values for all cases.
-This means that for very large non-floating point values there is a small chance that the
-operation can lead to slightly different answers than expected.
-However, a more likely outcome of very large non-floating point values is numerical overflow.
-
-[discrete]
-==== Arithmetic errors
-
-Arithmetic errors and numeric overflow do not result in an error. Instead, the result will be `null`
-and a warning for the `ArithmeticException` added.
-For example:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/math.csv-spec[tag=powULOverrun]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/math.csv-spec[tag=powULOverrun-warning]
-|===
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/math.csv-spec[tag=powULOverrun-result]
-|===
-
-If it is desired to protect against numerical overruns, use `TO_DOUBLE` on either of the arguments:
-
-[source.merge.styled,esql]
-----
-include::{esql-specs}/math.csv-spec[tag=pow2d]
-----
-[%header.monospaced.styled,format=dsv,separator=|]
-|===
-include::{esql-specs}/math.csv-spec[tag=pow2d-result]
-|===
 
 [discrete]
 ==== Fractional exponents

docs/reference/esql/functions/types/pow.asciidoc

@@ -4,7 +4,7 @@ base | exponent | result
 double | double | double
 double | integer | double
 integer | double | double
-integer | integer | integer
+integer | integer | double
 long | double | double
-long | integer | long
+long | integer | double
 |===

docs/reference/landing-page.asciidoc

@@ -215,6 +215,12 @@
     <li>
       <a href="https://www.elastic.co/guide/en/elasticsearch/plugins/current/index.html">Plugins and integrations</a>
     </li>
+     <li>
+      <a href="https://www.elastic.co/search-labs">Search Labs</a>
+    </li>
+    <li>
+      <a href="https://www.elastic.co/search-labs/tutorials/examples">Notebook examples</a>
+    </li>
   </ul>
 </div>
 

docs/reference/rest-api/security/grant-api-keys.asciidoc

@@ -15,18 +15,21 @@ Creates an API key on behalf of another user.
 [[security-api-grant-api-key-prereqs]]
 ==== {api-prereq-title}
 
-* To use this API, you must have the `grant_api_key` cluster privilege.
+* To use this API, you must have the `grant_api_key` or the `manage_api_key` cluster privilege.
 
 [[security-api-grant-api-key-desc]]
 ==== {api-description-title}
 
 This API is similar to <<security-api-create-api-key>>, however it creates the
 API key for a user that is different than the user that runs the API.
 
-The caller must have authentication credentials (either an access token,
-or a username and password) for the user on whose behalf the API key will be
-created. It is not possible to use this API to create an API key without that
-user's credentials.
+The caller must have authentication credentials for the user on whose behalf
+the API key will be created. It is not possible to use this API to create an
+API key without that user's credentials.
+The supported user authentication credentials types are:
+ * username and password
+ * <<security-api-get-token, {es} access tokens>>
+ * <<jwt-auth-realm, JWTs>>
 
 The user, for whom the authentication credentials is provided,
 can optionally <<run-as-privilege,"run as">> (impersonate) another user.
@@ -55,8 +58,11 @@ The following parameters can be specified in the body of a POST request:
 
 `access_token`::
 (Required*, string)
-The user's access token. If you specify the `access_token` grant type, this
-parameter is required. It is not valid with other grant types.
+The user's <<security-api-get-token, {es} access token>>, or JWT. Both <<jwt-realm-oauth2, access>> and
+<<jwt-realm-oidc, id>> JWT token types are supported, and they depend on the underlying JWT realm configuration.
+The created API key will have a point in time snapshot of permissions of the user authenticated with this token
+(or even more restricted permissions, see the `role_descriptors` parameter).
+If you specify the `access_token` grant type, this parameter is required. It is not valid with other grant types.
 
 `api_key`::
 (Required, object)
@@ -83,15 +89,32 @@ It supports nested data structure.
 Within the `metadata` object, keys beginning with `_` are reserved for
 system usage.
 
+`client_authentication`::
+(Optional, object) When using the `access_token` grant type, and when supplying a
+JWT, this specifies the client authentication for <<jwt-auth-realm, JWTs>> that
+need it (i.e. what's normally specified by the `ES-Client-Authentication` request header).
+
+`scheme`:::
+(Required, string) The scheme (case-sensitive) as it's supplied in the
+`ES-Client-Authentication` request header. Currently, the only supported
+value is <<jwt-auth-shared-secret-scheme-example, `SharedSecret`>>.
+
+`value`:::
+(Required, string) The value that follows the scheme for the client credentials
+as it's supplied in the `ES-Client-Authentication` request header. For example,
+if the request header would be `ES-Client-Authentication: SharedSecret myShar3dS3cret`
+if the client were to authenticate directly with a JWT, then `value` here should
+be `myShar3dS3cret`.
+
 `grant_type`::
 (Required, string)
 The type of grant. Supported grant types are: `access_token`,`password`.
 
 `access_token`:::
 (Required*, string)
-In this type of grant, you must supply an access token that was created by the
-{es} token service. For more information, see
-<<security-api-get-token>> and <<encrypt-http-communication>>.
+In this type of grant, you must supply either an access token, that was created by the
+{es} token service (see <<security-api-get-token>> and <<encrypt-http-communication>>),
+or a <<jwt-auth-realm, JWT>> (either a JWT `access_token` or a JWT `id_token`).
 
 `password`:::
 In this type of grant, you must supply the user ID and password for which you

docs/reference/search/point-in-time-api.asciidoc

@@ -22,6 +22,13 @@ or alias.
 To search a <<point-in-time-api,point in time (PIT)>> for an alias, you
 must have the `read` index privilege for the alias's data streams or indices.
 
+[[point-in-time-api-request-body]]
+==== {api-request-body-title}
+
+`index_filter`::
+(Optional,  <<query-dsl,query object>> Allows to filter indices if the provided
+query rewrites to `match_none` on every shard.
+
 [[point-in-time-api-example]]
 ==== {api-examples-title}
 
@@ -60,7 +67,7 @@ POST /_search  <1>
 or <<search-preference,`preference`>>
 as these parameters are copied from the point in time.
 <2> Just like regular searches, you can <<paginate-search-results,use `from` and
-`size` to page through search results>>, up to the first 10,000 hits. If you 
+`size` to page through search results>>, up to the first 10,000 hits. If you
 want to retrieve more hits, use PIT with <<search-after,`search_after`>>.
 <3> The `id` parameter tells Elasticsearch to execute the request using contexts
 from this point in time.

docs/reference/security/authentication/jwt-realm.asciidoc

@@ -123,8 +123,9 @@ Instructs the realm to treat and validate incoming JWTs as ID Tokens (`id_token`
 Specifies the client authentication type as `shared_secret`, which means that
 the client is authenticated using an HTTP request header that must match a
 pre-configured secret value. The client must provide this shared secret with
-every request in the `ES-Client-Authentication` header. The header value must be a
-case-sensitive match to the realm's `client_authentication.shared_secret`.
+every request in the `ES-Client-Authentication` header and using the
+`SharedSecret` scheme. The header value must be a case-sensitive match
+to the realm's `client_authentication.shared_secret`.
 
 `allowed_issuer`::
 Sets a verifiable identifier for your JWT issuer. This value is typically a
@@ -519,6 +520,7 @@ After mapping the roles, you can make an
 <<security-api-authenticate,authenticated call>> to {es} using a JWT and include
 the `ES-Client-Authentication` header:
 
+[[jwt-auth-shared-secret-scheme-example]]
 [source,sh]
 ----
 curl -s -X GET -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJhdWQiOlsiZXMwMSIsImVzMDIiLCJlczAzIl0sInN1YiI6InVzZXIyIiwiaXNzIjoibXktaXNzdWVyIiwiZXhwIjo0MDcwOTA4ODAwLCJpYXQiOjk0NjY4NDgwMCwiZW1haWwiOiJ1c2VyMkBzb21ldGhpbmcuZXhhbXBsZS5jb20ifQ.UgO_9w--EoRyUKcWM5xh9SimTfMzl1aVu6ZBsRWhxQA" -H "ES-Client-Authentication: sharedsecret test-secret" https://localhost:9200/_security/_authenticate