Merge branch 'main' into adding-simulate-ingest-api

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java

@@ -224,11 +224,6 @@ public Set<String> getMatchingFieldNames(String pattern) {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public boolean isFieldMapped(String field) {
-            return field.startsWith("int");
-        }
-
         @Override
         public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryType> context) {
             throw new UnsupportedOperationException();

branches.json

@@ -7,9 +7,6 @@
     {
       "branch": "8.11"
     },
-    {
-      "branch": "8.10"
-    },
     {
       "branch": "7.17"
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/StringStatsAggregationBuilder.java

@@ -22,7 +22,6 @@
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.xcontent.ParseField;
@@ -71,12 +70,6 @@ public String getType() {
         return NAME;
     }
 
-    @Override
-    protected ValuesSourceRegistry.RegistryKey<?> getRegistryKey() {
-        // This would be called from the same thing that calls innerBuild, which also throws. So it's "safe" to throw here.
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         return builder.field(StringStatsAggregationBuilder.SHOW_DISTRIBUTION_FIELD.getPreferredName(), showDistribution);

docs/changelog/101333.yaml

@@ -0,0 +1,29 @@
+pr: 101333
+summary: Fixed JWT principal from claims
+area: Authorization
+type: breaking
+issues: []
+breaking:
+  title: Fixed JWT principal from claims
+  area: Authorization
+  details: "This changes the format of a JWT's principal before the JWT is actually\
+    \ validated by any JWT realm. The JWT's principal is a convenient way to refer\
+    \ to a JWT that has not yet been verified by a JWT realm. The JWT's principal\
+    \ is printed in the audit and regular logs (notably for auditing authn failures)\
+    \ as well as the smart realm chain reordering optimization. The JWT principal\
+    \ is NOT required to be identical to the JWT-authenticated user's principal, but\
+    \ in general, they should be similar. Previously, the JWT's principal was built\
+    \ by individual realms in the same way the realms built the authenticated user's\
+    \ principal. This had the advantage that, in simpler JWT realms configurations\
+    \ (e.g. a single JWT realm in the chain), the JWT principal and the authenticated\
+    \ user's principal are very similar. However the drawback is that, in general,\
+    \ the JWT principal and the user principal can be very different (i.e. in the\
+    \ case where one JWT realm builds the JWT principal and a different one builds\
+    \ the user principal). Another downside is that the (unauthenticated) JWT principal\
+    \ depended on realm ordering, which makes identifying the JWT from its principal\
+    \ dependent on the ES authn realm configuration. This PR implements a consistent\
+    \ fixed logic to build the JWT principal, which only depends on the JWT's claims\
+    \ and no ES configuration."
+  impact: "Users will observe changed format and values for the `user.name` attribute\
+    \ of `authentication_failed` audit log events, in the JWT (failed) authn case."
+  notable: false

docs/changelog/101577.yaml

@@ -0,0 +1,5 @@
+pr: 101577
+summary: Add metrics to the shared blob cache
+area: Search
+type: enhancement
+issues: []

docs/changelog/101682.yaml

@@ -0,0 +1,5 @@
+pr: 101682
+summary: "Add manage_enrich cluster privilege to kibana_system role"
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/101753.yaml

@@ -0,0 +1,5 @@
+pr: 101753
+summary: Expose roles by default in cat allocation API
+area: CAT APIs
+type: enhancement
+issues: []

docs/changelog/101868.yaml

@@ -0,0 +1,5 @@
+pr: 101868
+summary: Read scores from downloaded vocabulary for XLM Roberta tokenizers
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/101892.yaml

@@ -0,0 +1,6 @@
+pr: 101892
+summary: Dry up `AsyncTaskIndexService` memory management and fix inefficient circuit
+  breaker use
+area: Search
+type: bug
+issues: []

docs/changelog/101907.yaml

@@ -0,0 +1,6 @@
+pr: 101907
+summary: Fail listener on exception in `TcpTransport#openConnection`
+area: Network
+type: bug
+issues:
+ - 100510

docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc

@@ -68,7 +68,7 @@ POST /_search
 --------------------------------------------------
 // TEST[setup:sales]
 
-<1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-day intervals
+<1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-month intervals
 <2> A `sum` metric is used to calculate the sum of a field. This could be any numeric metric (sum, min, max, etc)
 <3> Finally, we specify a `moving_fn` aggregation which uses "the_sum" metric as its input.
 

docs/reference/cat/allocation.asciidoc

@@ -6,7 +6,7 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or {kib} 
+cat APIs are only intended for human consumption using the command line or {kib}
 console. They are _not_ intended for use by applications.
 ====
 
@@ -113,10 +113,10 @@ The API returns the following response:
 
 [source,txt]
 --------------------------------------------------
-shards disk.indices disk.used disk.avail disk.total disk.percent host      ip        node
-     1         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2
+shards disk.indices disk.used disk.avail disk.total disk.percent host      ip        node    node.role
+     1         260b    47.3gb     43.4gb    100.7gb           46 127.0.0.1 127.0.0.1 CSUXak2 himrst
 --------------------------------------------------
 // TESTRESPONSE[s/\d+(\.\d+)?[tgmk]?b/\\d+(\\.\\d+)?[tgmk]?b/ s/46/\\d+/]
-// TESTRESPONSE[s/CSUXak2/.+/ non_json]
+// TESTRESPONSE[s/CSUXak2 himrst/.+/ non_json]
 
 This response shows a single shard is allocated to the one node available.

docs/reference/esql/esql-limitations.asciidoc

@@ -100,6 +100,12 @@ you query, and query `keyword` sub-fields instead of `text` fields.
 
 {esql} does not support querying time series data streams (TSDS).
 
+[discrete]
+[[esql-limitations-ccs]]
+=== {ccs-cap} is not supported
+
+{esql} does not support {ccs}.
+
 [discrete]
 [[esql-limitations-date-math]]
 === Date math limitations
@@ -130,6 +136,15 @@ now() - 2023-10-26
 
 include::esql-enrich-data.asciidoc[tag=limitations]
 
+[discrete]
+[[esql-limitations-mv]]
+=== Multivalue limitations
+
+{esql} <<esql-multivalued-fields,supports multivalued fields>>, but functions
+return `null` when applied to a multivalued field, unless documented otherwise.
+Work around this limitation by converting the field to single value with one of
+the <<esql-mv-functions,multivalue functions>>.
+
 [discrete]
 [[esql-limitations-kibana]]
 === Kibana limitations

docs/reference/esql/esql-using.asciidoc

@@ -9,13 +9,13 @@ Using {esql} in {kib} to query and aggregate your data, create visualizations,
 and set up alerts.
 
 <<esql-elastic-security>>::
-Using {esql} in {elastic-sec} to investigate events in Timeline and create
-detection rules.
+Using {esql} in {elastic-sec} to investigate events in Timeline, create
+detection rules, and build {esql} queries using Elastic AI Assistant. 
 
 <<esql-task-management>>::
 Using the <<tasks,task management API>> to list and cancel {esql} queries.
 
 include::esql-rest.asciidoc[]
 include::esql-kibana.asciidoc[]
 include::esql-security-solution.asciidoc[]
-include::task-management.asciidoc[]
+include::task-management.asciidoc[]

docs/reference/esql/functions/case.asciidoc

@@ -27,7 +27,8 @@ Accepts pairs of conditions and values. The function returns the value that
 belongs to the first condition that evaluates to `true`.
 
 If the number of arguments is odd, the last argument is the default value which
-is returned when no condition matches.
+is returned when no condition matches. If the number of arguments is even, and
+no condition matches, the function returns `null`.
 
 *Example*
 

docs/reference/esql/functions/starts_with.asciidoc

@@ -2,7 +2,7 @@
 [[esql-starts_with]]
 === `STARTS_WITH`
 [.text-center]
-image::esql/functions/signature/ends_with.svg[Embedded,opts=inline]
+image::esql/functions/signature/starts_with.svg[Embedded,opts=inline]
 
 Returns a boolean that indicates whether a keyword string starts with another
 string:

docs/reference/migration/migrate_8_11.asciidoc

@@ -9,12 +9,61 @@ your application to {es} 8.11.
 
 See also <<release-highlights>> and <<es-release-notes>>.
 
-coming::[8.11.0]
-
 
 [discrete]
 [[breaking-changes-8.11]]
 === Breaking changes
 
-There are no breaking changes in {es} 8.11.
+The following changes in {es} 8.11 might affect your applications
+and prevent them from operating normally.
+Before upgrading to 8.11, review these changes and take the described steps
+to mitigate the impact.
+
+
+There are no notable breaking changes in {es} 8.11.
+But there are some less critical breaking changes.
+
+[discrete]
+[[breaking_811_rest_api_changes]]
+==== REST API changes
+
+[[remove_transport_versions_from_cluster_state_api]]
+.Remove `transport_versions` from cluster state API
+[%collapsible]
+====
+*Details* +
+The `transport_versions` subobject of the response to `GET _cluster/state` has been replaced by the `nodes_versions` subobject.
+
+*Impact* +
+If needed, retrieve the per-node transport versions from the `nodes_versions` subobject.
+====
+
+
+[discrete]
+[[deprecated-8.11]]
+=== Deprecations
+
+The following functionality has been deprecated in {es} 8.11
+and will be removed in a future version.
+While this won't have an immediate impact on your applications,
+we strongly encourage you to take the described steps to update your code
+after upgrading to 8.11.
+
+To find out if you are using any deprecated functionality,
+enable <<deprecation-logging, deprecation logging>>.
+
+[discrete]
+[[deprecations_811_rollup]]
+==== Rollup deprecations
+
+[[rollup_functionality_deprecated]]
+.Rollup functionality is now deprecated
+[%collapsible]
+====
+*Details* +
+<<xpack-rollup,Rollup functionality>> has been deprecated and will be removed in a future release. Previously, rollups were available in technical preview.
+
+*Impact* +
+Use <<downsampling,downsampling>> to reduce storage costs for time series data by storing it at reduced granularity.
+====