diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 6f0657c3d5e8e..06b9f1dfbb6bf 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0", "8.12.0"] + BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0", "8.12.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 844570d945fdf..a265f7cd0cd4c 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1056,6 +1056,22 @@ steps: env: BWC_VERSION: 7.17.14 + - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 + - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0 timeout_in_minutes: 300 @@ -1648,6 +1664,22 @@ steps: env: BWC_VERSION: 8.10.3 + - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 + - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 8e959b07a9bc1..8143110607da2 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -642,6 +642,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 7.17.14 + - label: 7.17.15 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 7.17.15 - label: 8.0.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest timeout_in_minutes: 300 @@ -1012,6 +1022,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.10.3 + - label: 8.10.4 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.10.4 - label: 8.11.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest timeout_in_minutes: 300 diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh index 754c701927185..3571d112c5b6d 100755 --- a/.buildkite/scripts/periodic.trigger.sh +++ b/.buildkite/scripts/periodic.trigger.sh @@ -12,6 +12,18 @@ for BRANCH in "${BRANCHES[@]}"; do LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit') cat < getPluginsToUpgrade( throw new RuntimeException("Couldn't find a PluginInfo for [" + eachPluginId + "], which should be impossible"); }); - if (info.getElasticsearchVersion().before(Version.CURRENT)) { + if (info.getElasticsearchVersion().toString().equals(Build.current().version()) == false) { this.terminal.println( Terminal.Verbosity.VERBOSE, String.format( Locale.ROOT, - "Official plugin [%s] is out-of-date (%s versus %s), upgrading", + "Official plugin [%s] is out-of-sync (%s versus %s), upgrading", eachPluginId, info.getElasticsearchVersion(), - Version.CURRENT + Build.current().version() ) ); return true; @@ -278,14 +278,14 @@ private List getExistingPlugins() throws PluginSyncException { // Check for a version mismatch, unless it's an official plugin since we can upgrade them. if (InstallPluginAction.OFFICIAL_PLUGINS.contains(info.getName()) - && info.getElasticsearchVersion().equals(Version.CURRENT) == false) { + && info.getElasticsearchVersion().toString().equals(Build.current().version()) == false) { this.terminal.errorPrintln( String.format( Locale.ROOT, "WARNING: plugin [%s] was built for Elasticsearch version %s but version %s is required", info.getName(), info.getElasticsearchVersion(), - Version.CURRENT + Build.current().version() ) ); } diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index 2a66ed3cf4349..2da05d87f831f 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -32,7 +32,6 @@ import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyEncryptorBuilder; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.cli.ExitCodes; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.ProcessInfo; @@ -111,6 +110,7 @@ import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; @@ -298,7 +298,7 @@ private static String[] pluginProperties(String name, String[] additionalProps, "version", "1.0", "elasticsearch.version", - Version.CURRENT.toString(), + InstallPluginAction.getSemanticVersion(Build.current().version()), "java.version", System.getProperty("java.specification.version") @@ -724,7 +724,7 @@ public void testPluginPermissions() throws Exception { final Path platformBinDir = platformNameDir.resolve("bin"); Files.createDirectories(platformBinDir); - Files.createFile(tempPluginDir.resolve("fake-" + Version.CURRENT.toString() + ".jar")); + Files.createFile(tempPluginDir.resolve("fake-" + Build.current().version() + ".jar")); Files.createFile(platformBinDir.resolve("fake_executable")); Files.createDirectory(resourcesDir); Files.createFile(resourcesDir.resolve("resource")); @@ -740,7 +740,7 @@ public void testPluginPermissions() throws Exception { final Path platformName = platform.resolve("linux-x86_64"); final Path bin = platformName.resolve("bin"); assert755(fake); - assert644(fake.resolve("fake-" + Version.CURRENT + ".jar")); + assert644(fake.resolve("fake-" + Build.current().version() + ".jar")); assert755(resources); assert644(resources.resolve("resource")); assert755(platform); @@ -1110,8 +1110,8 @@ public void testOfficialPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", - Version.CURRENT, - Build.current().qualifiedVersion() + InstallPluginAction.getSemanticVersion(Build.current().version()), + Build.current().version() ); assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } @@ -1120,8 +1120,8 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s.zip", - Version.CURRENT, - Build.current().qualifiedVersion() + InstallPluginAction.getSemanticVersion(Build.current().version()), + Build.current().version() ); // attempting to install a release build of a plugin (no staging ID) on a snapshot build should throw a user exception final UserException e = expectThrows( @@ -1137,9 +1137,9 @@ public void testInstallReleaseBuildOfPluginOnSnapshotBuild() { public void testOfficialPluginStaging() throws Exception { String url = "https://staging.elastic.co/" - + Version.CURRENT + + InstallPluginAction.getSemanticVersion(Build.current().version()) + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" - + Build.current().qualifiedVersion() + + Build.current().version() + ".zip"; assertInstallPluginFromUrl("analysis-icu", url, "abc123", false); } @@ -1148,7 +1148,7 @@ public void testOfficialPlatformPlugin() throws Exception { String url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-" - + Build.current().qualifiedVersion() + + Build.current().version() + ".zip"; assertInstallPluginFromUrl("analysis-icu", url, null, false); } @@ -1157,16 +1157,16 @@ public void testOfficialPlatformPluginSnapshot() throws Exception { String url = String.format( Locale.ROOT, "https://snapshots.elastic.co/%s-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-%s-%s.zip", - Version.CURRENT, + InstallPluginAction.getSemanticVersion(Build.current().version()), Platforms.PLATFORM_NAME, - Build.current().qualifiedVersion() + Build.current().version() ); assertInstallPluginFromUrl("analysis-icu", url, "abc123", true); } public void testOfficialPlatformPluginStaging() throws Exception { String url = "https://staging.elastic.co/" - + Version.CURRENT + + InstallPluginAction.getSemanticVersion(Build.current().version()) + "-abc123/downloads/elasticsearch-plugins/analysis-icu/analysis-icu-" + Platforms.PLATFORM_NAME + "-" @@ -1580,6 +1580,17 @@ public void testStablePluginWithoutNamedComponentsFile() throws Exception { assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON()); } + public void testGetSemanticVersion() { + assertThat(InstallPluginAction.getSemanticVersion("1.2.3"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("123.456.789"), equalTo("123.456.789")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3-SNAPSHOT"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3foobar"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2.3.4"), equalTo("1.2.3")); + assertThat(InstallPluginAction.getSemanticVersion("1.2"), nullValue()); + assertThat(InstallPluginAction.getSemanticVersion("foo"), nullValue()); + assertThat(InstallPluginAction.getSemanticVersion("foo-1.2.3"), nullValue()); + } + private Map> namedComponentsMap() { Map> result = new LinkedHashMap<>(); Map extensibles = new LinkedHashMap<>(); diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java index e1577f7d101be..b225bc441794a 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java @@ -215,7 +215,7 @@ public void testExistingIncompatiblePlugin() throws Exception { "version", "1.0", "elasticsearch.version", - Version.fromString("1.0.0").toString(), + "1.0.0", "java.version", System.getProperty("java.specification.version"), "classname", diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java index 2c200df2a7d56..9802b4039bb7b 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugins.cli; import org.apache.lucene.tests.util.LuceneTestCase; -import org.elasticsearch.Version; +import org.elasticsearch.Build; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.UserException; import org.elasticsearch.common.settings.Settings; @@ -17,6 +17,7 @@ import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.plugins.cli.SyncPluginsAction.PluginChanges; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; import org.junit.Before; import org.mockito.InOrder; @@ -26,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Objects; import java.util.Optional; import static org.hamcrest.Matchers.containsString; @@ -129,7 +131,7 @@ public void test_getPluginChanges_withPluginToInstall_returnsPluginToInstall() t * since we can't automatically upgrade it. */ public void test_getPluginChanges_withPluginToUpgrade_returnsNoChanges() throws Exception { - createPlugin("my-plugin", Version.CURRENT.previousMajor()); + createPlugin("my-plugin", VersionUtils.getPreviousVersion().toString()); config.setPlugins(List.of(new InstallablePlugin("my-plugin"))); final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); @@ -142,7 +144,7 @@ public void test_getPluginChanges_withPluginToUpgrade_returnsNoChanges() throws * but needs to be upgraded, then we calculate that the plugin needs to be upgraded. */ public void test_getPluginChanges_withOfficialPluginToUpgrade_returnsPluginToUpgrade() throws Exception { - createPlugin("analysis-icu", Version.CURRENT.previousMajor()); + createPlugin("analysis-icu", VersionUtils.getPreviousVersion().toString()); config.setPlugins(List.of(new InstallablePlugin("analysis-icu"))); final PluginChanges pluginChanges = action.getPluginChanges(config, Optional.empty()); @@ -329,10 +331,11 @@ public void test_performSync_withPluginsToUpgrade_callsUpgradeAction() throws Ex } private void createPlugin(String name) throws IOException { - createPlugin(name, Version.CURRENT); + String semanticVersion = InstallPluginAction.getSemanticVersion(Build.current().version()); + createPlugin(name, Objects.nonNull(semanticVersion) ? semanticVersion : Build.current().version()); } - private void createPlugin(String name, Version version) throws IOException { + private void createPlugin(String name, String version) throws IOException { PluginTestUtil.writePluginProperties( env.pluginsFile().resolve(name), "description", @@ -342,7 +345,7 @@ private void createPlugin(String name, Version version) throws IOException { "version", "1.0", "elasticsearch.version", - version.toString(), + version, "java.version", System.getProperty("java.specification.version"), "classname", diff --git a/docs/changelog/100106.yaml b/docs/changelog/100106.yaml deleted file mode 100644 index c3e3d50d2597a..0000000000000 --- a/docs/changelog/100106.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100106 -summary: Validate enrich index before completing policy execution -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/100134.yaml b/docs/changelog/100134.yaml deleted file mode 100644 index 3836ec2793050..0000000000000 --- a/docs/changelog/100134.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100134 -summary: Implement matches() on `SourceConfirmedTextQuery` -area: Highlighting -type: enhancement -issues: [] diff --git a/docs/changelog/100179.yaml b/docs/changelog/100179.yaml deleted file mode 100644 index 2b7824c1575e6..0000000000000 --- a/docs/changelog/100179.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 100179 -summary: ILM introduce the `check-ts-end-time-passed` step -area: ILM+SLM -type: bug -issues: - - 99696 diff --git a/docs/changelog/100207.yaml b/docs/changelog/100207.yaml deleted file mode 100644 index 10e55992f0e45..0000000000000 --- a/docs/changelog/100207.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100207 -summary: ILM the delete action waits for a TSDS index time/bounds to lapse -area: ILM+SLM -type: bug -issues: [] diff --git a/docs/changelog/100284.yaml b/docs/changelog/100284.yaml deleted file mode 100644 index 956fc472d6656..0000000000000 --- a/docs/changelog/100284.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 100284 -summary: Defend against negative datafeed start times -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/100470.yaml b/docs/changelog/100470.yaml new file mode 100644 index 0000000000000..3408ae06f7fe9 --- /dev/null +++ b/docs/changelog/100470.yaml @@ -0,0 +1,6 @@ +pr: 100470 +summary: DSL waits for the tsdb time boundaries to lapse +area: Data streams +type: bug +issues: + - 99696 diff --git a/docs/changelog/100519.yaml b/docs/changelog/100519.yaml new file mode 100644 index 0000000000000..086c6962b3a95 --- /dev/null +++ b/docs/changelog/100519.yaml @@ -0,0 +1,5 @@ +pr: 100519 +summary: Disallow vectors whose magnitudes will not fit in a float +area: Vector Search +type: bug +issues: [] diff --git a/docs/changelog/100565.yaml b/docs/changelog/100565.yaml new file mode 100644 index 0000000000000..066e9bbb4b227 --- /dev/null +++ b/docs/changelog/100565.yaml @@ -0,0 +1,5 @@ +pr: 100565 +summary: "[Monitoring] Dont get cluster state until recovery" +area: Monitoring +type: bug +issues: [] diff --git a/docs/changelog/100609.yaml b/docs/changelog/100609.yaml new file mode 100644 index 0000000000000..c1c63c1af5d4d --- /dev/null +++ b/docs/changelog/100609.yaml @@ -0,0 +1,5 @@ +pr: 100609 +summary: Fix metric gauge creation model +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/100610.yaml b/docs/changelog/100610.yaml new file mode 100644 index 0000000000000..7423ce9225868 --- /dev/null +++ b/docs/changelog/100610.yaml @@ -0,0 +1,7 @@ +pr: 100610 +summary: Fix interruption of `markAllocationIdAsInSync` +area: Recovery +type: bug +issues: + - 96578 + - 100589 diff --git a/docs/changelog/100624.yaml b/docs/changelog/100624.yaml new file mode 100644 index 0000000000000..247343bf03ed8 --- /dev/null +++ b/docs/changelog/100624.yaml @@ -0,0 +1,5 @@ +pr: 100624 +summary: Make Transform Feature Reset really wait for all the tasks +area: Transform +type: bug +issues: [] diff --git a/docs/changelog/100645.yaml b/docs/changelog/100645.yaml new file mode 100644 index 0000000000000..e6bb6ab0fd653 --- /dev/null +++ b/docs/changelog/100645.yaml @@ -0,0 +1,7 @@ +pr: 100645 +summary: "ESQL: Graceful handling of non-bool condition in the filter" +area: ES|QL +type: bug +issues: + - 100049 + - 100409 diff --git a/docs/changelog/100647.yaml b/docs/changelog/100647.yaml new file mode 100644 index 0000000000000..399407146af68 --- /dev/null +++ b/docs/changelog/100647.yaml @@ -0,0 +1,6 @@ +pr: 100647 +summary: "ESQL: Handle queries with non-existing enrich policies and no field" +area: ES|QL +type: bug +issues: + - 100593 diff --git a/docs/changelog/100656.yaml b/docs/changelog/100656.yaml new file mode 100644 index 0000000000000..1ee9a2ad0e47a --- /dev/null +++ b/docs/changelog/100656.yaml @@ -0,0 +1,6 @@ +pr: 100656 +summary: "ESQL: fix non-null value being returned for unsupported data types in `ValueSources`" +area: ES|QL +type: bug +issues: + - 100048 diff --git a/docs/changelog/96968.yaml b/docs/changelog/96968.yaml new file mode 100644 index 0000000000000..8cc6d4ac4c284 --- /dev/null +++ b/docs/changelog/96968.yaml @@ -0,0 +1,6 @@ +pr: 96968 +summary: Allow prefix index naming while reindexing from remote +area: Reindex +type: bug +issues: + - 89120 diff --git a/docs/changelog/99231.yaml b/docs/changelog/99231.yaml deleted file mode 100644 index 9f5dfa1137587..0000000000000 --- a/docs/changelog/99231.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99231 -summary: Add manage permission for fleet managed threat intel indices -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/99604.yaml b/docs/changelog/99604.yaml deleted file mode 100644 index 0bace7aef1b26..0000000000000 --- a/docs/changelog/99604.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99604 -summary: Show a concrete error when the enrich index does not exist rather than a NullPointerException -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/99660.yaml b/docs/changelog/99660.yaml deleted file mode 100644 index ea19e24d51fff..0000000000000 --- a/docs/changelog/99660.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99660 -summary: Close expired search contexts on SEARCH thread -area: Search -type: bug -issues: [] diff --git a/docs/changelog/99673.yaml b/docs/changelog/99673.yaml deleted file mode 100644 index b48d620b21f49..0000000000000 --- a/docs/changelog/99673.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99673 -summary: Adding retry logic for start model deployment API -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99677.yaml b/docs/changelog/99677.yaml deleted file mode 100644 index 04c1c28cf2e12..0000000000000 --- a/docs/changelog/99677.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99677 -summary: Using 1 MB chunks for elser model storage -area: Machine Learning -type: bug -issues: [ ] diff --git a/docs/changelog/99724.yaml b/docs/changelog/99724.yaml deleted file mode 100644 index 4fe78687bf72b..0000000000000 --- a/docs/changelog/99724.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99724 -summary: Upgrade bundled JDK to Java 21 -area: Packaging -type: upgrade -issues: [] diff --git a/docs/changelog/99738.yaml b/docs/changelog/99738.yaml deleted file mode 100644 index 1b65926aed741..0000000000000 --- a/docs/changelog/99738.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99738 -summary: Ignore "index not found" error when `delete_dest_index` flag is set but the - dest index doesn't exist -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99803.yaml b/docs/changelog/99803.yaml deleted file mode 100644 index ce0929eb20e07..0000000000000 --- a/docs/changelog/99803.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99803 -summary: Do not use PIT in the presence of remote indices in source -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99814.yaml b/docs/changelog/99814.yaml deleted file mode 100644 index 1632be42b4e4c..0000000000000 --- a/docs/changelog/99814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99814 -summary: Fix cardinality agg for `const_keyword` -area: Aggregations -type: bug -issues: - - 99776 diff --git a/docs/changelog/99818.yaml b/docs/changelog/99818.yaml deleted file mode 100644 index 8835bcf28e050..0000000000000 --- a/docs/changelog/99818.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99818 -summary: Add checks in term and terms queries that input terms are not too long -area: Search -type: enhancement -issues: - - 99802 diff --git a/docs/changelog/99846.yaml b/docs/changelog/99846.yaml deleted file mode 100644 index 198b0b6f939ac..0000000000000 --- a/docs/changelog/99846.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99846 -summary: Update version range in `jvm.options` for the Panama Vector API -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/99868.yaml b/docs/changelog/99868.yaml deleted file mode 100644 index 33d582f9ebd0a..0000000000000 --- a/docs/changelog/99868.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99868 -summary: Fix fields API for `geo_point` fields inside other arrays -area: Search -type: bug -issues: - - 99781 diff --git a/docs/changelog/99892.yaml b/docs/changelog/99892.yaml deleted file mode 100644 index 5090d1d888b65..0000000000000 --- a/docs/changelog/99892.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 99892 -summary: Support $ and / in restore rename replacements -area: Snapshot/Restore -type: bug -issues: - - 99078 diff --git a/docs/changelog/99914.yaml b/docs/changelog/99914.yaml deleted file mode 100644 index 8b0026a8ff9ca..0000000000000 --- a/docs/changelog/99914.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99914 -summary: Let `_stats` internally timeout if checkpoint information can not be retrieved -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/99946.yaml b/docs/changelog/99946.yaml deleted file mode 100644 index 11dc4090baa0e..0000000000000 --- a/docs/changelog/99946.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 99946 -summary: Skip settings validation during desired nodes updates -area: Distributed -type: bug -issues: [] diff --git a/docs/reference/release-notes/8.10.3.asciidoc b/docs/reference/release-notes/8.10.3.asciidoc index a09beb26b4d27..b7828f52ad082 100644 --- a/docs/reference/release-notes/8.10.3.asciidoc +++ b/docs/reference/release-notes/8.10.3.asciidoc @@ -1,7 +1,11 @@ [[release-notes-8.10.3]] == {es} version 8.10.3 -coming[8.10.3] +[[known-issues-8.10.3]] +[float] +=== Known issues + +include::8.10.0.asciidoc[tag=repositorydata-format-change] Also see <>. diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6c7fa4d4653d2..01f330a93e8fa 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 0adc8e1a53214..1aa94a4269074 100755 --- a/gradlew +++ b/gradlew @@ -145,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -153,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -202,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java index 9ded2106b2500..526a621674f6b 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/WellKnownBinary.java @@ -24,6 +24,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; @@ -40,10 +41,13 @@ private WellKnownBinary() {} /** * Converts the given {@link Geometry} to WKB with the provided {@link ByteOrder} */ - public static byte[] toWKB(Geometry geometry, ByteOrder byteOrder) throws IOException { + public static byte[] toWKB(Geometry geometry, ByteOrder byteOrder) { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { toWKB(geometry, outputStream, ByteBuffer.allocate(8).order(byteOrder)); return outputStream.toByteArray(); + } catch (IOException ioe) { + // Should never happen as the only method throwing IOException is ByteArrayOutputStream#close and it is a NOOP + throw new UncheckedIOException(ioe); } } diff --git a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java index 9ede3d9db8126..5369475e4ed4f 100644 --- a/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java +++ b/libs/geo/src/test/java/org/elasticsearch/geometry/utils/WKBTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; -import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; @@ -35,47 +34,47 @@ public void testEmptyPoint() { assertEquals("Empty POINT cannot be represented in WKB", ex.getMessage()); } - public void testPoint() throws IOException { + public void testPoint() { Point point = GeometryTestUtils.randomPoint(randomBoolean()); assertWKB(point); } - public void testEmptyMultiPoint() throws IOException { + public void testEmptyMultiPoint() { MultiPoint multiPoint = MultiPoint.EMPTY; assertWKB(multiPoint); } - public void testMultiPoint() throws IOException { + public void testMultiPoint() { MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(randomBoolean()); assertWKB(multiPoint); } - public void testEmptyLine() throws IOException { + public void testEmptyLine() { Line line = Line.EMPTY; assertWKB(line); } - public void testLine() throws IOException { + public void testLine() { Line line = GeometryTestUtils.randomLine(randomBoolean()); assertWKB(line); } - public void tesEmptyMultiLine() throws IOException { + public void tesEmptyMultiLine() { MultiLine multiLine = MultiLine.EMPTY; assertWKB(multiLine); } - public void testMultiLine() throws IOException { + public void testMultiLine() { MultiLine multiLine = GeometryTestUtils.randomMultiLine(randomBoolean()); assertWKB(multiLine); } - public void testEmptyPolygon() throws IOException { + public void testEmptyPolygon() { Polygon polygon = Polygon.EMPTY; assertWKB(polygon); } - public void testPolygon() throws IOException { + public void testPolygon() { final boolean hasZ = randomBoolean(); Polygon polygon = GeometryTestUtils.randomPolygon(hasZ); if (randomBoolean()) { @@ -89,22 +88,22 @@ public void testPolygon() throws IOException { assertWKB(polygon); } - public void testEmptyMultiPolygon() throws IOException { + public void testEmptyMultiPolygon() { MultiPolygon multiPolygon = MultiPolygon.EMPTY; assertWKB(multiPolygon); } - public void testMultiPolygon() throws IOException { + public void testMultiPolygon() { MultiPolygon multiPolygon = GeometryTestUtils.randomMultiPolygon(randomBoolean()); assertWKB(multiPolygon); } - public void testEmptyGeometryCollection() throws IOException { + public void testEmptyGeometryCollection() { GeometryCollection collection = GeometryCollection.EMPTY; assertWKB(collection); } - public void testGeometryCollection() throws IOException { + public void testGeometryCollection() { GeometryCollection collection = GeometryTestUtils.randomGeometryCollection(randomBoolean()); assertWKB(collection); } @@ -115,7 +114,7 @@ public void testEmptyCircle() { assertEquals("Empty CIRCLE cannot be represented in WKB", ex.getMessage()); } - public void testCircle() throws IOException { + public void testCircle() { Circle circle = GeometryTestUtils.randomCircle(randomBoolean()); assertWKB(circle); } @@ -129,7 +128,7 @@ public void testEmptyRectangle() { assertEquals("Empty ENVELOPE cannot be represented in WKB", ex.getMessage()); } - public void testRectangle() throws IOException { + public void testRectangle() { Rectangle rectangle = GeometryTestUtils.randomRectangle(); assertWKB(rectangle); } @@ -138,7 +137,7 @@ private ByteOrder randomByteOrder() { return randomBoolean() ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; } - private void assertWKB(Geometry geometry) throws IOException { + private void assertWKB(Geometry geometry) { final boolean hasZ = geometry.hasZ(); final ByteOrder byteOrder = randomByteOrder(); final byte[] b = WellKnownBinary.toWKB(geometry, byteOrder); diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java index 9d55d475d4a93..54f33be21698b 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/DoubleGaugeAdapter.java @@ -10,33 +10,46 @@ import io.opentelemetry.api.metrics.Meter; +import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; /** * DoubleGaugeAdapter wraps an otel ObservableDoubleMeasurement */ -public class DoubleGaugeAdapter extends AbstractInstrument +public class DoubleGaugeAdapter extends AbstractInstrument implements org.elasticsearch.telemetry.metric.DoubleGauge { + private final AtomicReference valueWithAttributes; + public DoubleGaugeAdapter(Meter meter, String name, String description, String unit) { super(meter, name, description, unit); + this.valueWithAttributes = new AtomicReference<>(new ValueWithAttributes(0.0, Collections.emptyMap())); } @Override - io.opentelemetry.api.metrics.ObservableDoubleMeasurement buildInstrument(Meter meter) { - var builder = Objects.requireNonNull(meter).gaugeBuilder(getName()); - return builder.setDescription(getDescription()).setUnit(getUnit()).buildObserver(); + io.opentelemetry.api.metrics.ObservableDoubleGauge buildInstrument(Meter meter) { + return Objects.requireNonNull(meter) + .gaugeBuilder(getName()) + .setDescription(getDescription()) + .setUnit(getUnit()) + .buildWithCallback(measurement -> { + var localValueWithAttributed = valueWithAttributes.get(); + measurement.record(localValueWithAttributed.value(), OtelHelper.fromMap(localValueWithAttributed.attributes())); + }); } @Override public void record(double value) { - getInstrument().record(value); + record(value, Collections.emptyMap()); } @Override public void record(double value, Map attributes) { - getInstrument().record(value, OtelHelper.fromMap(attributes)); + this.valueWithAttributes.set(new ValueWithAttributes(value, attributes)); } + + private record ValueWithAttributes(double value, Map attributes) {} } diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java index 48430285a5173..66d2287a765dc 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/metrics/LongGaugeAdapter.java @@ -10,37 +10,47 @@ import io.opentelemetry.api.metrics.Meter; +import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; /** * LongGaugeAdapter wraps an otel ObservableLongMeasurement */ -public class LongGaugeAdapter extends AbstractInstrument +public class LongGaugeAdapter extends AbstractInstrument implements org.elasticsearch.telemetry.metric.LongGauge { + private final AtomicReference valueWithAttributes; public LongGaugeAdapter(Meter meter, String name, String description, String unit) { super(meter, name, description, unit); + this.valueWithAttributes = new AtomicReference<>(new ValueWithAttributes(0L, Collections.emptyMap())); } @Override - io.opentelemetry.api.metrics.ObservableLongMeasurement buildInstrument(Meter meter) { + io.opentelemetry.api.metrics.ObservableLongGauge buildInstrument(Meter meter) { + return Objects.requireNonNull(meter) .gaugeBuilder(getName()) .ofLongs() .setDescription(getDescription()) .setUnit(getUnit()) - .buildObserver(); + .buildWithCallback(measurement -> { + var localValueWithAttributed = valueWithAttributes.get(); + measurement.record(localValueWithAttributed.value(), OtelHelper.fromMap(localValueWithAttributed.attributes())); + }); } @Override public void record(long value) { - getInstrument().record(value); + record(value, Collections.emptyMap()); } @Override public void record(long value, Map attributes) { - getInstrument().record(value, OtelHelper.fromMap(attributes)); + this.valueWithAttributes.set(new ValueWithAttributes(value, attributes)); } + + private record ValueWithAttributes(long value, Map attributes) {} } diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java new file mode 100644 index 0000000000000..1e230eefe32dc --- /dev/null +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/metrics/GaugeAdapterTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.telemetry.apm.internal.metrics; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.LongGaugeBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.Map; +import java.util.function.Consumer; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class GaugeAdapterTests extends ESTestCase { + Meter testMeter = Mockito.mock(Meter.class); + LongGaugeBuilder longGaugeBuilder = Mockito.mock(LongGaugeBuilder.class); + DoubleGaugeBuilder mockDoubleGaugeBuilder = Mockito.mock(DoubleGaugeBuilder.class); + + @Before + public void init() { + when(longGaugeBuilder.setDescription(Mockito.anyString())).thenReturn(longGaugeBuilder); + when(longGaugeBuilder.setUnit(Mockito.anyString())).thenReturn(longGaugeBuilder); + + + when(mockDoubleGaugeBuilder.ofLongs()).thenReturn(longGaugeBuilder); + when(mockDoubleGaugeBuilder.setUnit(Mockito.anyString())).thenReturn(mockDoubleGaugeBuilder); + when(mockDoubleGaugeBuilder.setDescription(Mockito.anyString())).thenReturn(mockDoubleGaugeBuilder); + when(testMeter.gaugeBuilder(anyString())).thenReturn(mockDoubleGaugeBuilder); + } + + // testing that a value reported is then used in a callback + @SuppressWarnings("unchecked") + public void testLongGaugeRecord() { + LongGaugeAdapter longGaugeAdapter = new LongGaugeAdapter(testMeter, "name", "desc", "unit"); + + // recording a value + longGaugeAdapter.record(1L, Map.of("k", 1L)); + + // upon metric export, the consumer will be called + ArgumentCaptor> captor = ArgumentCaptor.forClass(Consumer.class); + verify(longGaugeBuilder).buildWithCallback(captor.capture()); + + Consumer value = captor.getValue(); + // making sure that a consumer will fetch the value passed down upon recording of a value + TestLongMeasurement testLongMeasurement = new TestLongMeasurement(); + value.accept(testLongMeasurement); + + assertThat(testLongMeasurement.value, Matchers.equalTo(1L)); + assertThat(testLongMeasurement.attributes, Matchers.equalTo(Attributes.builder().put("k", 1).build())); + } + + // testing that a value reported is then used in a callback + @SuppressWarnings("unchecked") + public void testDoubleGaugeRecord() { + DoubleGaugeAdapter doubleGaugeAdapter = new DoubleGaugeAdapter(testMeter, "name", "desc", "unit"); + + // recording a value + doubleGaugeAdapter.record(1.0, Map.of("k", 1.0)); + + // upon metric export, the consumer will be called + ArgumentCaptor> captor = ArgumentCaptor.forClass(Consumer.class); + verify(mockDoubleGaugeBuilder).buildWithCallback(captor.capture()); + + Consumer value = captor.getValue(); + // making sure that a consumer will fetch the value passed down upon recording of a value + TestDoubleMeasurement testLongMeasurement = new TestDoubleMeasurement(); + value.accept(testLongMeasurement); + + assertThat(testLongMeasurement.value, Matchers.equalTo(1.0)); + assertThat(testLongMeasurement.attributes, Matchers.equalTo(Attributes.builder().put("k", 1.0).build())); + } + + private static class TestDoubleMeasurement implements ObservableDoubleMeasurement { + double value; + Attributes attributes; + + @Override + public void record(double value) { + this.value = value; + } + + @Override + public void record(double value, Attributes attributes) { + this.value = value; + this.attributes = attributes; + + } + } + + private static class TestLongMeasurement implements ObservableLongMeasurement { + long value; + Attributes attributes; + + @Override + public void record(long value) { + this.value = value; + } + + @Override + public void record(long value, Attributes attributes) { + this.value = value; + this.attributes = attributes; + + } + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java new file mode 100644 index 0000000000000..7de4ed2f2843c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/EcsLogsDataStreamIT.java @@ -0,0 +1,433 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.core.PathUtils; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.datastreams.LogsDataStreamIT.createDataStream; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getMappingProperties; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getValueFromPath; +import static org.elasticsearch.datastreams.LogsDataStreamIT.getWriteBackingIndex; +import static org.elasticsearch.datastreams.LogsDataStreamIT.indexDoc; +import static org.elasticsearch.datastreams.LogsDataStreamIT.searchDocs; +import static org.elasticsearch.datastreams.LogsDataStreamIT.waitForLogs; +import static org.hamcrest.Matchers.is; + +public class EcsLogsDataStreamIT extends DisabledSecurityDataStreamTestCase { + + private static final String DATA_STREAM_NAME = "logs-generic-default"; + private RestClient client; + private String backingIndex; + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + createDataStream(client, DATA_STREAM_NAME); + backingIndex = getWriteBackingIndex(client, DATA_STREAM_NAME); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + } + + @SuppressWarnings("unchecked") + public void testElasticAgentLogEcsMappings() throws Exception { + { + Path path = PathUtils.get(Thread.currentThread().getContextClassLoader().getResource("ecs-logs/es-agent-ecs-log.json").toURI()); + String agentLog = Files.readString(path); + indexDoc(client, DATA_STREAM_NAME, agentLog); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "elastic-agent-log" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // timestamp from deserialized JSON message field should win + assertThat(source.get("@timestamp"), is("2023-05-16T13:49:40.374Z")); + assertThat( + ((Map>) source.get("kubernetes")).get("pod").get("name"), + is("elastic-agent-managed-daemonset-jwktj") + ); + // expecting the extracted message from within the original JSON-formatted message + assertThat(((List) fields.get("message")).get(0), is("Non-zero metrics in the last 30s")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("@timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "name", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("kubernetes", "properties", "pod", "properties", "test_ip", "type")), is("ip")); + assertThat( + getValueFromPath( + properties, + List.of("kubernetes", "properties", "labels", "properties", "pod-template-generation", "type") + ), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("log", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "os", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + } + } + + @SuppressWarnings("unchecked") + public void testGeneralMockupEcsMappings() throws Exception { + { + indexDoc(client, DATA_STREAM_NAME, """ + { + "start_timestamp": "not a date", + "start-timestamp": "not a date", + "timestamp.us": 1688550340718000, + "test": "mockup-ecs-log", + "registry": { + "data": { + "strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + } + }, + "process": { + "title": "ssh", + "executable": "/usr/bin/ssh", + "name": "ssh", + "command_line": "/usr/bin/ssh -l user 10.0.0.16", + "working_directory": "/home/ekoren", + "io": { + "text": "test" + } + }, + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "email": { + "message_id": "81ce15$8r2j59@mail01.example.com" + }, + "parent": { + "url": { + "path": "/page", + "full": "https://mydomain.com/app/page", + "original": "https://mydomain.com/app/original" + }, + "body": { + "content": "Some content" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "code_signature.timestamp": "2023-07-05", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"] + }, + "error": { + "stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "message": "Error occurred" + }, + "file": { + "path": "/path/to/my/file", + "target_path": "/path/to/my/file" + }, + "os": { + "full": "Mac OS Mojave" + }, + "user_agent": { + "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" + }, + "user": { + "full_name": "John Doe" + }, + "vulnerability": { + "score": { + "base": 5.5, + "temporal": 5.5, + "version": "2.0" + }, + "textual_score": "bad" + }, + "host": { + "cpu": { + "usage": 0.68 + } + }, + "geo": { + "location": { + "lon": -73.614830, + "lat": 45.505918 + } + }, + "data_stream": { + "dataset": "nginx.access", + "namespace": "production", + "custom": "whatever" + }, + "structured_data": { + "key1": "value1", + "key2": ["value2", "value3"] + }, + "exports": { + "key": "value" + }, + "top_level_imports": { + "key": "value" + }, + "nested": { + "imports": { + "key": "value" + } + }, + "numeric_as_string": "42", + "socket": { + "ip": "127.0.0.1", + "remote_ip": "187.8.8.8" + } + } + """); + List results = searchDocs(client, DATA_STREAM_NAME, """ + { + "query": { + "term": { + "test": { + "value": "mockup-ecs-log" + } + } + }, + "fields": ["start-timestamp", "start_timestamp"], + "script_fields": { + "data_stream_type": { + "script": { + "source": "doc['data_stream.type'].value" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map fields = ((Map>) results.get(0)).get("fields"); + List ignored = ((Map>) results.get(0)).get("_ignored"); + Map ignoredFieldValues = ((Map>) results.get(0)).get("ignored_field_values"); + + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("start_timestamp")); + List startTimestampValues = (List) ignoredFieldValues.get("start_timestamp"); + assertThat(startTimestampValues.size(), is(1)); + assertThat(startTimestampValues.get(0), is("not a date")); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("start-timestamp"), is(List.of("not a date"))); + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream_type"), is(List.of("logs"))); + assertThat(ignored.get(1), is("vulnerability.textual_score")); + + Map properties = getMappingProperties(client, backingIndex); + assertThat(getValueFromPath(properties, List.of("error", "properties", "message", "type")), is("match_only_text")); + assertThat( + getValueFromPath(properties, List.of("registry", "properties", "data", "properties", "strings", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "registry", "properties", "data", "properties", "strings", "type") + ), + is("wildcard") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "io", "properties", "text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email", "properties", "message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "full", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("url", "properties", "original", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "url", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "type")), + is("wildcard") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "body", "properties", "content", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "command_line", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "command_line", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("error", "properties", "stack_trace", "type")), is("wildcard")); + assertThat( + getValueFromPath(properties, List.of("error", "properties", "stack_trace", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("file", "properties", "target_path", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("file", "properties", "target_path", "fields", "text", "type")), + is("match_only_text") + ); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "file", "properties", "target_path", "type")), + is("keyword") + ); + assertThat( + getValueFromPath( + properties, + List.of("parent", "properties", "file", "properties", "target_path", "fields", "text", "type") + ), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os", "properties", "full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent", "properties", "original", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user_agent", "properties", "original", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "title", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "title", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "executable", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "executable", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("process", "properties", "working_directory", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("process", "properties", "working_directory", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("user", "properties", "full_name", "type")), is("keyword")); + assertThat( + getValueFromPath(properties, List.of("user", "properties", "full_name", "fields", "text", "type")), + is("match_only_text") + ); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("timestamp", "properties", "us", "type")), is("long")); + assertThat( + getValueFromPath(properties, List.of("parent", "properties", "code_signature", "properties", "timestamp", "type")), + is("date") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "base", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "temporal", "type")), + is("float") + ); + assertThat( + getValueFromPath(properties, List.of("vulnerability", "properties", "score", "properties", "version", "type")), + is("keyword") + ); + assertThat(getValueFromPath(properties, List.of("vulnerability", "properties", "textual_score", "type")), is("float")); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "type")), + is("scaled_float") + ); + assertThat( + getValueFromPath(properties, List.of("host", "properties", "cpu", "properties", "usage", "scaling_factor")), + is(1000.0) + ); + assertThat(getValueFromPath(properties, List.of("geo", "properties", "location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream", "properties", "custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested", "properties", "imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket", "properties", "remote_ip", "type")), is("ip")); + } + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 5bb9c8b340ee9..cc8695b9e0e5b 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.nullValue; public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase { @@ -45,8 +46,8 @@ public void testDefaultLogsSettingAndMapping() throws Exception { // Extend the mapping and verify putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("ignore_malformed"), equalTo(false)); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "ignore_malformed")), equalTo(false)); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); // Insert valid doc and verify successful indexing { @@ -149,11 +150,8 @@ public void testCustomMapping() throws Exception { // Verify that the new field from the custom component template is applied putMapping(client, backingIndex); Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("numeric_field")).get("type"), equalTo("integer")); - assertThat( - ((Map) mappingProperties.get("socket")).get("properties"), - equalTo(Map.of("ip", Map.of("type", "keyword"))) - ); + assertThat(getValueFromPath(mappingProperties, List.of("numeric_field", "type")), equalTo("integer")); + assertThat(getValueFromPath(mappingProperties, List.of("socket", "properties", "ip", "type")), is("keyword")); // Insert valid doc and verify successful indexing { @@ -227,7 +225,7 @@ public void testLogsDefaultPipeline() throws Exception { // Verify mapping from custom logs Map mappingProperties = getMappingProperties(client, backingIndex); - assertThat(((Map) mappingProperties.get("@timestamp")).get("type"), equalTo("date")); + assertThat(getValueFromPath(mappingProperties, List.of("@timestamp", "type")), equalTo("date")); // no timestamp - testing default pipeline's @timestamp set processor { @@ -284,7 +282,358 @@ public void testLogsDefaultPipeline() throws Exception { } } - private static void waitForLogs(RestClient client) throws Exception { + @SuppressWarnings("unchecked") + public void testLogsMessagePipeline() throws Exception { + RestClient client = client(); + waitForLogs(client); + + { + Request request = new Request("PUT", "/_ingest/pipeline/logs@custom"); + request.setJsonEntity(""" + { + "processors": [ + { + "pipeline" : { + "name": "logs@json-message", + "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" + } + } + ] + } + """); + assertOK(client.performRequest(request)); + } + + String dataStreamName = "logs-generic-default"; + createDataStream(client, dataStreamName); + + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-09T16:48:34.135Z", + "message":"json", + "log.level": "INFO", + "ecs.version": "1.6.0", + "service.name":"my-app", + "event.dataset":"my-app.RollingFile", + "process.thread.name":"main", + "log.logger":"root.pkg.MyApp" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "message": { + "value": "json" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-09T16:48:34.135Z")); + assertThat(source.get("message"), is("json")); + assertThat(((List) fields.get("message")).get(0), is("json")); + + // successful access to subfields verifies that dot expansion is part of the pipeline + assertThat(source.get("log.level"), is("INFO")); + assertThat(source.get("ecs.version"), is("1.6.0")); + assertThat(source.get("service.name"), is("my-app")); + assertThat(source.get("event.dataset"), is("my-app.RollingFile")); + assertThat(source.get("process.thread.name"), is("main")); + assertThat(source.get("log.logger"), is("root.pkg.MyApp")); + // _tmp_json_message should be removed by the pipeline + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message + { + indexDoc(client, dataStreamName, """ + { + "@timestamp":"2023-05-10", + "test":"malformed_json", + "message": "{\\"@timestamp\\":\\"2023-05-09T16:48:34.135Z\\", \\"message\\":\\"malformed_json\\"}}" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "malformed_json" + } + } + } + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + + // root field parsed from JSON should win + assertThat(source.get("@timestamp"), is("2023-05-10")); + assertThat(source.get("message"), is("{\"@timestamp\":\"2023-05-09T16:48:34.135Z\", \"message\":\"malformed_json\"}}")); + assertThat(source.get("_tmp_json_message"), is(nullValue())); + } + + // test non-string message field + { + indexDoc(client, dataStreamName, """ + { + "message": 42, + "test": "numeric_message" + } + """); + List results = searchDocs(client, dataStreamName, """ + { + "query": { + "term": { + "test": { + "value": "numeric_message" + } + } + }, + "fields": ["message"] + } + """); + assertThat(results.size(), is(1)); + Map source = ((Map>) results.get(0)).get("_source"); + Map fields = ((Map>) results.get(0)).get("fields"); + + assertThat(source.get("message"), is(42)); + assertThat(((List) fields.get("message")).get(0), is("42")); + } + } + + @SuppressWarnings("unchecked") + public void testNoSubobjects() throws Exception { + RestClient client = client(); + waitForLogs(client); + { + Request request = new Request("POST", "/_component_template/logs-test-subobjects-mappings"); + request.setJsonEntity(""" + { + "template": { + "settings": { + "mapping": { + "ignore_malformed": true + } + }, + "mappings": { + "subobjects": false, + "date_detection": false, + "properties": { + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + } + } + """); + assertOK(client.performRequest(request)); + } + { + Request request = new Request("POST", "/_index_template/logs-ecs-test-template"); + request.setJsonEntity(""" + { + "priority": 200, + "data_stream": {}, + "index_patterns": ["logs-*-*"], + "composed_of": ["logs-test-subobjects-mappings", "ecs@dynamic_templates"] + } + """); + assertOK(client.performRequest(request)); + } + String dataStream = "logs-ecs-test-subobjects"; + createDataStream(client, dataStream); + String backingIndexName = getWriteBackingIndex(client, dataStream); + + indexDoc(client, dataStream, """ + { + "@timestamp": "2023-06-12", + "start_timestamp": "2023-06-08", + "location" : "POINT (-71.34 41.12)", + "test": "flattened", + "test.start_timestamp": "not a date", + "test.start-timestamp": "not a date", + "registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "process.title": "ssh", + "process.executable": "/usr/bin/ssh", + "process.name": "ssh", + "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", + "process.working_directory": "/home/ekoren", + "process.io.text": "test", + "url.path": "/page", + "url.full": "https://mydomain.com/app/page", + "url.original": "https://mydomain.com/app/original", + "email.message_id": "81ce15$8r2j59@mail01.example.com", + "parent.url.path": "/page", + "parent.url.full": "https://mydomain.com/app/page", + "parent.url.original": "https://mydomain.com/app/original", + "parent.body.content": "Some content", + "parent.file.path": "/path/to/my/file", + "parent.file.target_path": "/path/to/my/file", + "parent.registry.data.strings": ["C:\\\\rta\\\\red_ttp\\\\bin\\\\myapp.exe"], + "error.stack_trace": "co.elastic.test.TestClass error:\\n at co.elastic.test.BaseTestClass", + "error.message": "Error occurred", + "file.path": "/path/to/my/file", + "file.target_path": "/path/to/my/file", + "os.full": "Mac OS Mojave", + "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", + "user.full_name": "John Doe", + "vulnerability.score.base": 5.5, + "vulnerability.score.temporal": 5.5, + "vulnerability.score.version": "2.0", + "vulnerability.textual_score": "bad", + "host.cpu.usage": 0.68, + "geo.location": [-73.614830, 45.505918], + "data_stream.dataset": "nginx.access", + "data_stream.namespace": "production", + "data_stream.custom": "whatever", + "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, + "exports": {"key": "value"}, + "top_level_imports": {"key": "value"}, + "nested.imports": {"key": "value"}, + "numeric_as_string": "42", + "socket.ip": "127.0.0.1", + "socket.remote_ip": "187.8.8.8" + } + """); + List hits = searchDocs(client, dataStream, """ + { + "query": { + "term": { + "test": { + "value": "flattened" + } + } + }, + "fields": [ + "data_stream.type", + "location", + "geo.location", + "test.start-timestamp", + "test.start_timestamp", + "vulnerability.textual_score" + ] + } + """); + assertThat(hits.size(), is(1)); + Map fields = ((Map>) hits.get(0)).get("fields"); + List ignored = ((Map>) hits.get(0)).get("_ignored"); + Map> ignoredFieldValues = ((Map>>) hits.get(0)).get("ignored_field_values"); + + // verify that data_stream.type has the correct constant_keyword value + assertThat(fields.get("data_stream.type"), is(List.of("logs"))); + // verify geo_point subfields evaluation + assertThat(((List>) fields.get("location")).get(0).get("type"), is("Point")); + List coordinates = ((List>>) fields.get("location")).get(0).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-71.34)); + assertThat(coordinates.get(1), equalTo(41.12)); + List geoLocation = (List) fields.get("geo.location"); + assertThat(((Map) geoLocation.get(0)).get("type"), is("Point")); + coordinates = ((Map>) geoLocation.get(0)).get("coordinates"); + assertThat(coordinates.size(), is(2)); + assertThat(coordinates.get(0), equalTo(-73.614830)); + assertThat(coordinates.get(1), equalTo(45.505918)); + // "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" + assertThat(fields.get("test.start-timestamp"), is(List.of("not a date"))); + assertThat(ignored.size(), is(2)); + assertThat(ignored.get(0), is("vulnerability.textual_score")); + // the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type + assertThat(ignored.get(1), is("test.start_timestamp")); + assertThat(ignoredFieldValues.get("test.start_timestamp").size(), is(1)); + assertThat(ignoredFieldValues.get("test.start_timestamp"), is(List.of("not a date"))); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").size(), is(1)); + assertThat(ignoredFieldValues.get("vulnerability.textual_score").get(0), is("bad")); + + Map properties = getMappingProperties(client, backingIndexName); + assertThat(getValueFromPath(properties, List.of("error.message", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.registry.data.strings", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.io.text", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("email.message_id", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.path", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.url.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("parent.body.content", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("process.command_line", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "type")), is("wildcard")); + assertThat(getValueFromPath(properties, List.of("error.stack_trace", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("parent.file.target_path", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("os.full", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("os.full", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user_agent.original", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.title", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.title", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.executable", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.executable", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("process.working_directory", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("user.full_name", "fields", "text", "type")), is("match_only_text")); + assertThat(getValueFromPath(properties, List.of("start_timestamp", "type")), is("date")); + assertThat(getValueFromPath(properties, List.of("test.start_timestamp", "type")), is("date")); + // testing the default mapping of string input fields to keyword if not matching any pattern + assertThat(getValueFromPath(properties, List.of("test.start-timestamp", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.base", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.temporal", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("vulnerability.score.version", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("vulnerability.textual_score", "type")), is("float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "type")), is("scaled_float")); + assertThat(getValueFromPath(properties, List.of("host.cpu.usage", "scaling_factor")), is(1000.0)); + assertThat(getValueFromPath(properties, List.of("location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("geo.location", "type")), is("geo_point")); + assertThat(getValueFromPath(properties, List.of("data_stream.dataset", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.namespace", "type")), is("constant_keyword")); + assertThat(getValueFromPath(properties, List.of("data_stream.type", "type")), is("constant_keyword")); + // not one of the three data_stream fields that are explicitly mapped to constant_keyword + assertThat(getValueFromPath(properties, List.of("data_stream.custom", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("structured_data", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("exports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("top_level_imports", "type")), is("flattened")); + assertThat(getValueFromPath(properties, List.of("nested.imports", "type")), is("flattened")); + // verifying the default mapping for strings into keyword, overriding the automatic numeric string detection + assertThat(getValueFromPath(properties, List.of("numeric_as_string", "type")), is("keyword")); + assertThat(getValueFromPath(properties, List.of("socket.ip", "type")), is("ip")); + assertThat(getValueFromPath(properties, List.of("socket.remote_ip", "type")), is("ip")); + + } + + static void waitForLogs(RestClient client) throws Exception { assertBusy(() -> { try { Request request = new Request("GET", "_index_template/logs"); @@ -295,13 +644,13 @@ private static void waitForLogs(RestClient client) throws Exception { }); } - private static void createDataStream(RestClient client, String name) throws IOException { + static void createDataStream(RestClient client, String name) throws IOException { Request request = new Request("PUT", "_data_stream/" + name); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static String getWriteBackingIndex(RestClient client, String name) throws IOException { + static String getWriteBackingIndex(RestClient client, String name) throws IOException { Request request = new Request("GET", "_data_stream/" + name); List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); Map dataStream = (Map) dataStreams.get(0); @@ -310,12 +659,12 @@ private static String getWriteBackingIndex(RestClient client, String name) throw } @SuppressWarnings("unchecked") - private static Map getSettings(RestClient client, String indexName) throws IOException { + static Map getSettings(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings"); return ((Map>) entityAsMap(client.performRequest(request)).get(indexName)).get("settings"); } - private static void putMapping(RestClient client, String indexName) throws IOException { + static void putMapping(RestClient client, String indexName) throws IOException { Request request = new Request("PUT", "/" + indexName + "/_mapping"); request.setJsonEntity(""" { @@ -330,24 +679,51 @@ private static void putMapping(RestClient client, String indexName) throws IOExc } @SuppressWarnings("unchecked") - private static Map getMappingProperties(RestClient client, String indexName) throws IOException { + static Map getMappingProperties(RestClient client, String indexName) throws IOException { Request request = new Request("GET", "/" + indexName + "/_mapping"); Map map = (Map) entityAsMap(client.performRequest(request)).get(indexName); Map mappings = (Map) map.get("mappings"); return (Map) mappings.get("properties"); } - private static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { + static void indexDoc(RestClient client, String dataStreamName, String doc) throws IOException { Request request = new Request("POST", "/" + dataStreamName + "/_doc?refresh=true"); request.setJsonEntity(doc); assertOK(client.performRequest(request)); } @SuppressWarnings("unchecked") - private static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { + static List searchDocs(RestClient client, String dataStreamName, String query) throws IOException { Request request = new Request("GET", "/" + dataStreamName + "/_search"); request.setJsonEntity(query); Map hits = (Map) entityAsMap(client.performRequest(request)).get("hits"); return (List) hits.get("hits"); } + + @SuppressWarnings("unchecked") + static Object getValueFromPath(Map map, List path) { + Map current = map; + for (int i = 0; i < path.size(); i++) { + Object value = current.get(path.get(i)); + if (i == path.size() - 1) { + return value; + } + if (value == null) { + throw new IllegalStateException("Path " + String.join(".", path) + " was not found in " + map); + } + if (value instanceof Map next) { + current = (Map) next; + } else { + throw new IllegalStateException( + "Failed to reach the end of the path " + + String.join(".", path) + + " last reachable field was " + + path.get(i) + + " in " + + map + ); + } + } + return current; + } } diff --git a/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json new file mode 100644 index 0000000000000..29ae669e1290d --- /dev/null +++ b/modules/data-streams/src/javaRestTest/resources/ecs-logs/es-agent-ecs-log.json @@ -0,0 +1,118 @@ +{ + "@timestamp": "2023-05-16T13:49:40.377Z", + "test": "elastic-agent-log", + "container": { + "image": { + "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" + }, + "runtime": "containerd", + "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" + }, + "kubernetes": { + "container": { + "name": "elastic-agent" + }, + "node": { + "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", + "hostname": "multi-v1.27.1-worker", + "name": "multi-v1.27.1-worker", + "labels": { + "kubernetes_io/hostname": "multi-v1.27.1-worker", + "beta_kubernetes_io/os": "linux", + "kubernetes_io/arch": "arm64", + "kubernetes_io/os": "linux", + "beta_kubernetes_io/arch": "arm64" + } + }, + "pod": { + "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", + "ip": "172.18.0.4", + "test_ip": "172.18.0.5", + "name": "elastic-agent-managed-daemonset-jwktj" + }, + "namespace": "kube-system", + "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", + "daemonset": { + "name": "elastic-agent-managed-daemonset" + }, + "namespace_labels": { + "kubernetes_io/metadata_name": "kube-system" + }, + "labels": { + "controller-revision-hash": "7ff74fcd4b", + "pod-template-generation": "1", + "k8s-app": "elastic-agent" + } + }, + "agent": { + "name": "multi-v1.27.1-worker", + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", + "type": "filebeat", + "version": "8.9.0" + }, + "log": { + "file": { + "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" + }, + "offset": 635247 + }, + "elastic_agent": { + "id": "230358e2-6c5d-4675-9069-04feaddad64b", + "version": "8.9.0", + "snapshot": true + }, + "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", + "orchestrator": { + "cluster": { + "name": "multi-v1.27.1", + "url": "multi-v1.27.1-control-plane:6443" + } + }, + "input": { + "type": "filestream" + }, + "ecs": { + "version": "8.0.0" + }, + "stream": "stderr", + "data_stream": { + "namespace": "default", + "dataset": "kubernetes.container_logs" + }, + "host": { + "hostname": "multi-v1.27.1-worker", + "os": { + "kernel": "5.15.49-linuxkit", + "codename": "focal", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "20.04.6 LTS (Focal Fossa)", + "platform": "ubuntu" + }, + "ip": [ + "10.244.2.1", + "10.244.2.1", + "172.18.0.4", + "fc00:f853:ccd:e793::4", + "fe80::42:acff:fe12:4", + "172.21.0.9" + ], + "containerized": false, + "name": "multi-v1.27.1-worker", + "id": "b2c527655d7746328f0686e25d3c413a", + "mac": [ + "02-42-AC-12-00-04", + "02-42-AC-15-00-09", + "32-7E-AA-73-39-04", + "EA-F3-80-1D-88-E3" + ], + "architecture": "aarch64" + }, + "event": { + "agent_id_status": "verified", + "ingested": "2023-05-16T13:49:47Z", + "dataset": "kubernetes.container_logs" + } +} \ No newline at end of file diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 5d85a199c4e3d..d1ea1b589b5a5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -63,7 +63,9 @@ import org.elasticsearch.datastreams.lifecycle.downsampling.DeleteSourceAndAddDownsampleToDS; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.snapshots.SnapshotInProgressException; import org.elasticsearch.threadpool.ThreadPool; @@ -71,6 +73,7 @@ import java.io.Closeable; import java.time.Clock; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -140,7 +143,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab private final ThreadPool threadPool; final ResultDeduplicator transportActionsDeduplicator; final ResultDeduplicator clusterStateChangesDeduplicator; - private final LongSupplier nowSupplier; + private LongSupplier nowSupplier; private final Clock clock; private final DataStreamLifecycleErrorStore errorStore; private volatile boolean isMaster = false; @@ -304,11 +307,24 @@ void run(ClusterState state) { } } } - Set indicesBeingRemoved; + + Set indicesToExcludeForRemainingRun = new HashSet<>(); + // the following indices should not be considered for the remainder of this service run: + // 1) the write index as it's still getting writes and we'll have to roll it over when the conditions are met + // 2) tsds indices that are still within their time bounds (i.e. now < time_series.end_time) - we don't want these indices to be + // deleted, forcemerged, or downsampled as they're still expected to receive large amounts of writes + indicesToExcludeForRemainingRun.add(currentRunWriteIndex); + indicesToExcludeForRemainingRun.addAll( + timeSeriesIndicesStillWithinTimeBounds( + state.metadata(), + getTargetIndices(dataStream, indicesToExcludeForRemainingRun, state.metadata()::index), + nowSupplier + ) + ); + try { - indicesBeingRemoved = maybeExecuteRetention(state, dataStream); + indicesToExcludeForRemainingRun.addAll(maybeExecuteRetention(state, dataStream, indicesToExcludeForRemainingRun)); } catch (Exception e) { - indicesBeingRemoved = Set.of(); // individual index errors would be reported via the API action listener for every delete call // we could potentially record errors at a data stream level and expose it via the _data_stream API? logger.error( @@ -321,13 +337,6 @@ void run(ClusterState state) { ); } - // the following indices should not be considered for the remainder of this service run: - // 1) the write index as it's still getting writes and we'll have to roll it over when the conditions are met - // 2) we exclude any indices that we're in the process of deleting because they'll be gone soon anyway - Set indicesToExcludeForRemainingRun = new HashSet<>(); - indicesToExcludeForRemainingRun.add(currentRunWriteIndex); - indicesToExcludeForRemainingRun.addAll(indicesBeingRemoved); - try { indicesToExcludeForRemainingRun.addAll( maybeExecuteForceMerge(state, getTargetIndices(dataStream, indicesToExcludeForRemainingRun, state.metadata()::index)) @@ -372,6 +381,30 @@ void run(ClusterState state) { ); } + // visible for testing + static Set timeSeriesIndicesStillWithinTimeBounds(Metadata metadata, List targetIndices, LongSupplier nowSupplier) { + Set tsIndicesWithinBounds = new HashSet<>(); + for (Index index : targetIndices) { + IndexMetadata backingIndex = metadata.index(index); + assert backingIndex != null : "the data stream backing indices must exist"; + if (IndexSettings.MODE.get(backingIndex.getSettings()) == IndexMode.TIME_SERIES) { + Instant configuredEndTime = IndexSettings.TIME_SERIES_END_TIME.get(backingIndex.getSettings()); + assert configuredEndTime != null + : "a time series index must have an end time configured but [" + index.getName() + "] does not"; + if (nowSupplier.getAsLong() <= configuredEndTime.toEpochMilli()) { + logger.trace( + "Data stream lifecycle will not perform any operations in this run on time series index [{}] because " + + "its configured [{}] end time has not lapsed", + index.getName(), + configuredEndTime + ); + tsIndicesWithinBounds.add(index); + } + } + } + return tsIndicesWithinBounds; + } + /** * Data stream lifecycle supports configuring multiple rounds of downsampling for each managed index. When attempting to execute * downsampling we iterate through the ordered rounds of downsampling that match an index (ordered ascending according to the `after` @@ -716,11 +749,13 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) { /** * This method sends requests to delete any indices in the datastream that exceed its retention policy. It returns the set of indices * it has sent delete requests for. - * @param state The cluster state from which to get index metadata - * @param dataStream The datastream + * + * @param state The cluster state from which to get index metadata + * @param dataStream The datastream + * @param indicesToExcludeForRemainingRun Indices to exclude from retention even if it would be time for them to be deleted * @return The set of indices that delete requests have been sent for */ - private Set maybeExecuteRetention(ClusterState state, DataStream dataStream) { + private Set maybeExecuteRetention(ClusterState state, DataStream dataStream, Set indicesToExcludeForRemainingRun) { TimeValue retention = getRetentionConfiguration(dataStream); Set indicesToBeRemoved = new HashSet<>(); if (retention != null) { @@ -728,14 +763,16 @@ private Set maybeExecuteRetention(ClusterState state, DataStream dataStre List backingIndicesOlderThanRetention = dataStream.getIndicesPastRetention(metadata::index, nowSupplier); for (Index index : backingIndicesOlderThanRetention) { - indicesToBeRemoved.add(index); - IndexMetadata backingIndex = metadata.index(index); - assert backingIndex != null : "the data stream backing indices must exist"; - - // there's an opportunity here to batch the delete requests (i.e. delete 100 indices / request) - // let's start simple and reevaluate - String indexName = backingIndex.getIndex().getName(); - deleteIndexOnce(indexName, "the lapsed [" + retention + "] retention period"); + if (indicesToExcludeForRemainingRun.contains(index) == false) { + indicesToBeRemoved.add(index); + IndexMetadata backingIndex = metadata.index(index); + assert backingIndex != null : "the data stream backing indices must exist"; + + // there's an opportunity here to batch the delete requests (i.e. delete 100 indices / request) + // let's start simple and reevaluate + String indexName = backingIndex.getIndex().getName(); + deleteIndexOnce(indexName, "the lapsed [" + retention + "] retention period"); + } } } return indicesToBeRemoved; @@ -1227,6 +1264,11 @@ public DataStreamLifecycleErrorStore getErrorStore() { return errorStore; } + // visible for testing + public void setNowSupplier(LongSupplier nowSupplier) { + this.nowSupplier = nowSupplier; + } + /** * This is a ClusterStateTaskListener that writes the force_merge_completed_timestamp into the cluster state. It is meant to run in * STATE_UPDATE_TASK_EXECUTOR. diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index b1679b5fa6701..f1e74a936e781 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling; import org.elasticsearch.cluster.metadata.DataStreamLifecycle.Downsampling.Round; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -59,6 +60,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -80,6 +82,7 @@ import java.time.Clock; import java.time.Instant; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -105,6 +108,7 @@ import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService.TARGET_MERGE_FACTOR_VALUE; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -246,6 +250,49 @@ public void testRetentionNotExecutedDueToAge() { assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); } + public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + Instant start3 = currentTime.plus(2, ChronoUnit.HOURS); + Instant end3 = currentTime.plus(4, ChronoUnit.HOURS); + + String dataStreamName = "logs_my-app_prod"; + var clusterState = DataStreamTestHelper.getClusterStateWithDataStream( + dataStreamName, + List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2), Tuple.tuple(start3, end3)) + ); + Metadata.Builder builder = Metadata.builder(clusterState.metadata()); + DataStream dataStream = builder.dataStream(dataStreamName); + builder.put( + new DataStream( + dataStreamName, + dataStream.getIndices(), + dataStream.getGeneration() + 1, + dataStream.getMetadata(), + dataStream.isHidden(), + dataStream.isReplicated(), + dataStream.isSystem(), + dataStream.isAllowCustomRouting(), + dataStream.getIndexMode(), + DataStreamLifecycle.newBuilder().dataRetention(0L).build() + ) + ); + clusterState = ClusterState.builder(clusterState).metadata(builder).build(); + + dataStreamLifecycleService.run(clusterState); + assertThat(clientSeenRequests.size(), is(2)); // rollover the write index and one delete request for the index that's out of the + // TS time bounds + assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class)); + TransportRequest deleteIndexRequest = clientSeenRequests.get(1); + assertThat(deleteIndexRequest, instanceOf(DeleteIndexRequest.class)); + // only the first generation index should be eligible for retention + assertThat(((DeleteIndexRequest) deleteIndexRequest).indices(), is(new String[] { dataStream.getIndices().get(0).getName() })); + } + public void testIlmManagedIndicesAreSkipped() { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); int numBackingIndices = 3; @@ -1186,6 +1233,69 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc assertThat(error, containsString("resource_already_exists_exception")); } + public void testTimeSeriesIndicesStillWithinTimeBounds() { + Instant currentTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); + // These ranges are on the edge of each other temporal boundaries. + Instant start1 = currentTime.minus(6, ChronoUnit.HOURS); + Instant end1 = currentTime.minus(4, ChronoUnit.HOURS); + Instant start2 = currentTime.minus(4, ChronoUnit.HOURS); + Instant end2 = currentTime.plus(2, ChronoUnit.HOURS); + Instant start3 = currentTime.plus(2, ChronoUnit.HOURS); + Instant end3 = currentTime.plus(4, ChronoUnit.HOURS); + + String dataStreamName = "logs_my-app_prod"; + var clusterState = DataStreamTestHelper.getClusterStateWithDataStream( + dataStreamName, + List.of(Tuple.tuple(start1, end1), Tuple.tuple(start2, end2), Tuple.tuple(start3, end3)) + ); + DataStream dataStream = clusterState.getMetadata().dataStreams().get(dataStreamName); + + { + // test for an index for which `now` is outside its time bounds + Index firstGenIndex = dataStream.getIndices().get(0); + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + clusterState.metadata(), + // the end_time for the first generation has lapsed + List.of(firstGenIndex), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(0)); + } + + { + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + clusterState.metadata(), + // the end_time for the first generation has lapsed, but the other 2 generations are still within bounds + dataStream.getIndices(), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(2)); + assertThat(indices, containsInAnyOrder(dataStream.getIndices().get(1), dataStream.getIndices().get(2))); + } + + { + // non time_series indices are not within time bounds (they don't have any) + IndexMetadata indexMeta = IndexMetadata.builder(randomAlphaOfLengthBetween(10, 30)) + .settings( + Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersion.current()) + .build() + ) + .build(); + + Metadata newMetadata = Metadata.builder(clusterState.metadata()).put(indexMeta, true).build(); + + Set indices = DataStreamLifecycleService.timeSeriesIndicesStillWithinTimeBounds( + newMetadata, + List.of(indexMeta.getIndex()), + currentTime::toEpochMilli + ); + assertThat(indices.size(), is(0)); + } + } + /* * Creates a test cluster state with the given indexName. If customDataStreamLifecycleMetadata is not null, it is added as the value * of the index's custom metadata named "data_stream_lifecycle". diff --git a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java index 43438bfe9e5fb..fa7b4ca1a80c0 100644 --- a/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java +++ b/modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java @@ -9,7 +9,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -20,7 +19,6 @@ import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.ClassRule; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99764") public class DataStreamsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public DataStreamsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml deleted file mode 100644 index 6fd6f24a4ea14..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/230_logs_message_pipeline.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -Test log message JSON-parsing pipeline: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - - is_true: acknowledged - - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - message: |- - { - "@timestamp":"2023-05-09T16:48:34.135Z", - "message":"json", - "log.level": "INFO", - "ecs.version": "1.6.0", - "service.name":"my-app", - "event.dataset":"my-app.RollingFile", - "process.thread.name":"main", - "log.logger":"root.pkg.MyApp" - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - message: - value: 'json' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # root field parsed from JSON should win - - match: { hits.hits.0._source.@timestamp: '2023-05-09T16:48:34.135Z' } - - match: { hits.hits.0._source.message: 'json' } - - match: { hits.hits.0.fields.message.0: 'json' } - # successful access to subfields verifies that dot expansion is part of the pipeline - - match: { hits.hits.0._source.log.level: 'INFO' } - - match: { hits.hits.0._source.ecs.version: '1.6.0' } - - match: { hits.hits.0._source.service.name: 'my-app' } - - match: { hits.hits.0._source.event.dataset: 'my-app.RollingFile' } - - match: { hits.hits.0._source.process.thread.name: 'main' } - - match: { hits.hits.0._source.log.logger: 'root.pkg.MyApp' } - # _tmp_json_message should be removed by the pipeline - - match: { hits.hits.0._source._tmp_json_message: null } - - # test malformed-JSON parsing - parsing error should be ignored and the document should be indexed with original message - - do: - index: - index: logs-generic-default - refresh: true - body: - '@timestamp': '2023-05-10' - test: 'malformed_json' - message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'malformed_json' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.@timestamp: '2023-05-10' } - - match: { hits.hits.0._source.message: '{"@timestamp":"2023-05-09T16:48:34.135Z", "message":"malformed_json"}}' } - - match: { hits.hits.0._source._tmp_json_message: null } - - # test non-string message field - - do: - index: - index: logs-generic-default - refresh: true - body: - test: 'numeric_message' - message: 42 - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'numeric_message' - fields: - - field: 'message' - - length: { hits.hits: 1 } - - match: { hits.hits.0._source.message: 42 } - - match: { hits.hits.0.fields.message.0: '42' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml deleted file mode 100644 index 538e362ed9ec0..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_logs_ecs_mappings.yml +++ /dev/null @@ -1,406 +0,0 @@ -setup: - - do: - ingest.put_pipeline: - # opting in to use the JSON parsing pipeline for message field - id: "logs@custom" - body: > - { - "processors": [ - { - "pipeline" : { - "name": "logs@json-message", - "description": "A pipeline that automatically parses JSON log events into top-level fields if they are such" - } - } - ] - } - - - do: - indices.create_data_stream: - name: logs-generic-default - ---- -Test Elastic Agent log ECS mappings: - - skip: - version: all - reason: https://github.com/elastic/elasticsearch/issues/97795 - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "@timestamp": "2023-05-16T13:49:40.377Z", - "test": "elastic-agent-log", - "container": { - "image": { - "name": "docker.elastic.co/beats/elastic-agent:8.9.0-SNAPSHOT" - }, - "runtime": "containerd", - "id": "bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d" - }, - "kubernetes": { - "container": { - "name": "elastic-agent" - }, - "node": { - "uid": "0f4dd3b8-0b29-418e-ad7a-ebc55bc279ff", - "hostname": "multi-v1.27.1-worker", - "name": "multi-v1.27.1-worker", - "labels": { - "kubernetes_io/hostname": "multi-v1.27.1-worker", - "beta_kubernetes_io/os": "linux", - "kubernetes_io/arch": "arm64", - "kubernetes_io/os": "linux", - "beta_kubernetes_io/arch": "arm64" - } - }, - "pod": { - "uid": "c91d1354-27cf-40f3-a2d6-e2b75aa96bf2", - "ip": "172.18.0.4", - "test_ip": "172.18.0.5", - "name": "elastic-agent-managed-daemonset-jwktj" - }, - "namespace": "kube-system", - "namespace_uid": "63294aeb-b23f-429d-827c-e793ccf91024", - "daemonset": { - "name": "elastic-agent-managed-daemonset" - }, - "namespace_labels": { - "kubernetes_io/metadata_name": "kube-system" - }, - "labels": { - "controller-revision-hash": "7ff74fcd4b", - "pod-template-generation": "1", - "k8s-app": "elastic-agent" - } - }, - "agent": { - "name": "multi-v1.27.1-worker", - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "ephemeral_id": "e0934bfb-7e35-4bcc-a935-803643841213", - "type": "filebeat", - "version": "8.9.0" - }, - "log": { - "file": { - "path": "/var/log/containers/elastic-agent-managed-daemonset-jwktj_kube-system_elastic-agent-bdabf58305b2b537d06b85764c588ff659190d875cb5470214bc16ba50ea1a4d.log" - }, - "offset": 635247 - }, - "elastic_agent": { - "id": "230358e2-6c5d-4675-9069-04feaddad64b", - "version": "8.9.0", - "snapshot": true - }, - "message": "{\"log.level\":\"info\",\"@timestamp\":\"2023-05-16T13:49:40.374Z\",\"message\":\"Non-zero metrics in the last 30s\",\"component\":{\"binary\":\"metricbeat\",\"dataset\":\"elastic_agent.metricbeat\",\"id\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\",\"type\":\"kubernetes/metrics\"},\"log\":{\"source\":\"kubernetes/metrics-a92ab320-f3ed-11ed-9c8d-45656839f031\"},\"log.logger\":\"monitoring\",\"log.origin\":{\"file.line\":187,\"file.name\":\"log/log.go\"},\"service.name\":\"metricbeat\",\"ecs.version\":\"1.6.0\"}", - "orchestrator": { - "cluster": { - "name": "multi-v1.27.1", - "url": "multi-v1.27.1-control-plane:6443" - } - }, - "input": { - "type": "filestream" - }, - "ecs": { - "version": "8.0.0" - }, - "stream": "stderr", - "data_stream": { - "namespace": "default", - "dataset": "kubernetes.container_logs" - }, - "host": { - "hostname": "multi-v1.27.1-worker", - "os": { - "kernel": "5.15.49-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.6 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "ip": [ - "10.244.2.1", - "10.244.2.1", - "172.18.0.4", - "fc00:f853:ccd:e793::4", - "fe80::42:acff:fe12:4", - "172.21.0.9" - ], - "containerized": false, - "name": "multi-v1.27.1-worker", - "id": "b2c527655d7746328f0686e25d3c413a", - "mac": [ - "02-42-AC-12-00-04", - "02-42-AC-15-00-09", - "32-7E-AA-73-39-04", - "EA-F3-80-1D-88-E3" - ], - "architecture": "aarch64" - }, - "event": { - "agent_id_status": "verified", - "ingested": "2023-05-16T13:49:47Z", - "dataset": "kubernetes.container_logs" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'elastic-agent-log' - fields: - - field: 'message' - - length: { hits.hits: 1 } - # timestamp from deserialized JSON message field should win - - match: { hits.hits.0._source.@timestamp: '2023-05-16T13:49:40.374Z' } - - match: { hits.hits.0._source.kubernetes.pod.name: 'elastic-agent-managed-daemonset-jwktj' } - # expecting the extracted message from within the original JSON-formatted message - - match: { hits.hits.0.fields.message.0: 'Non-zero metrics in the last 30s' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.@timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.pod.properties.test_ip.type: "ip" } - - match: { .$idx0name.mappings.properties.kubernetes.properties.labels.properties.pod-template-generation.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.host.properties.os.properties.name.fields.text.type: "match_only_text" } - ---- -Test general mockup ECS mappings: - - do: - indices.get_data_stream: - name: logs-generic-default - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-generic-default - refresh: true - body: > - { - "start_timestamp": "not a date", - "start-timestamp": "not a date", - "timestamp.us": 1688550340718000, - "test": "mockup-ecs-log", - "registry": { - "data": { - "strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - } - }, - "process": { - "title": "ssh", - "executable": "/usr/bin/ssh", - "name": "ssh", - "command_line": "/usr/bin/ssh -l user 10.0.0.16", - "working_directory": "/home/ekoren", - "io": { - "text": "test" - } - }, - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "email": { - "message_id": "81ce15$8r2j59@mail01.example.com" - }, - "parent": { - "url": { - "path": "/page", - "full": "https://mydomain.com/app/page", - "original": "https://mydomain.com/app/original" - }, - "body": { - "content": "Some content" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "code_signature.timestamp": "2023-07-05", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"] - }, - "error": { - "stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "message": "Error occurred" - }, - "file": { - "path": "/path/to/my/file", - "target_path": "/path/to/my/file" - }, - "os": { - "full": "Mac OS Mojave" - }, - "user_agent": { - "original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15" - }, - "user": { - "full_name": "John Doe" - }, - "vulnerability": { - "score": { - "base": 5.5, - "temporal": 5.5, - "version": "2.0" - }, - "textual_score": "bad" - }, - "host": { - "cpu": { - "usage": 0.68 - } - }, - "geo": { - "location": { - "lon": -73.614830, - "lat": 45.505918 - } - }, - "data_stream": { - "dataset": "nginx.access", - "namespace": "production", - "custom": "whatever" - }, - "structured_data": { - "key1": "value1", - "key2": ["value2", "value3"] - }, - "exports": { - "key": "value" - }, - "top_level_imports": { - "key": "value" - }, - "nested": { - "imports": { - "key": "value" - } - }, - "numeric_as_string": "42", - "socket": { - "ip": "127.0.0.1", - "remote_ip": "187.8.8.8" - } - } - - match: {result: "created"} - - - do: - search: - index: logs-generic-default - body: - query: - term: - test: - value: 'mockup-ecs-log' - fields: - - field: 'start_timestamp' - - field: 'start-timestamp' - script_fields: - data_stream_type: - script: - source: "doc['data_stream.type'].value" - - length: { hits.hits: 1 } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'start_timestamp' } - - length: { hits.hits.0.ignored_field_values.start_timestamp: 1 } - - match: { hits.hits.0.ignored_field_values.start_timestamp.0: 'not a date' } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - - match: { hits.hits.0.fields.start-timestamp.0: 'not a date' } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream_type.0: 'logs' } - - match: { hits.hits.0._ignored.1: 'vulnerability.textual_score' } - - - do: - indices.get_mapping: - index: logs-generic-default - - match: { .$idx0name.mappings.properties.error.properties.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.registry.properties.data.properties.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.io.properties.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email.properties.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.url.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent.properties.body.properties.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process.properties.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error.properties.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent.properties.file.properties.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os.properties.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os.properties.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent.properties.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process.properties.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user.properties.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.timestamp.properties.us.type: "long" } - - match: { .$idx0name.mappings.properties.parent.properties.code_signature.properties.timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.score.properties.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability.properties.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host.properties.cpu.properties.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.geo.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream.properties.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested.properties.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket.properties.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket.properties.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml deleted file mode 100644 index 607693e9f9955..0000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_logs_no_subobjects.yml +++ /dev/null @@ -1,218 +0,0 @@ ---- -Test flattened document with subobjects-false: -# NOTE: this doesn't work. In order to run this test set "subobjects: false" through logs-mappings.json - - skip: - features: allowed_warnings - - - do: - cluster.put_component_template: - name: logs-test-subobjects-mappings - body: - template: - settings: - mapping: - ignore_malformed: true - mappings: - subobjects: false - date_detection: false - properties: - data_stream.type: - type: constant_keyword - value: logs - data_stream.dataset: - type: constant_keyword - data_stream.namespace: - type: constant_keyword - - - do: - allowed_warnings: - - "index template [logs-ecs-test-template] has index patterns [logs-*-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs-ecs-test-template] will take precedence during new index creation" - indices.put_index_template: - name: logs-ecs-test-template - body: - priority: 200 - data_stream: {} - index_patterns: - - logs-*-* - composed_of: - - logs-test-subobjects-mappings - - ecs@dynamic_templates - - - do: - indices.create_data_stream: - name: logs-ecs-test-subobjects - - is_true: acknowledged - - - do: - indices.get_data_stream: - name: logs-ecs-test-subobjects - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - index: - index: logs-ecs-test-subobjects - refresh: true - body: > - { - "@timestamp": "2023-06-12", - "start_timestamp": "2023-06-08", - "location" : "POINT (-71.34 41.12)", - "test": "flattened", - "test.start_timestamp": "not a date", - "test.start-timestamp": "not a date", - "registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "process.title": "ssh", - "process.executable": "/usr/bin/ssh", - "process.name": "ssh", - "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", - "process.working_directory": "/home/ekoren", - "process.io.text": "test", - "url.path": "/page", - "url.full": "https://mydomain.com/app/page", - "url.original": "https://mydomain.com/app/original", - "email.message_id": "81ce15$8r2j59@mail01.example.com", - "parent.url.path": "/page", - "parent.url.full": "https://mydomain.com/app/page", - "parent.url.original": "https://mydomain.com/app/original", - "parent.body.content": "Some content", - "parent.file.path": "/path/to/my/file", - "parent.file.target_path": "/path/to/my/file", - "parent.registry.data.strings": ["C:\\rta\\red_ttp\\bin\\myapp.exe"], - "error.stack_trace": "co.elastic.test.TestClass error:\n at co.elastic.test.BaseTestClass", - "error.message": "Error occurred", - "file.path": "/path/to/my/file", - "file.target_path": "/path/to/my/file", - "os.full": "Mac OS Mojave", - "user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15", - "user.full_name": "John Doe", - "vulnerability.score.base": 5.5, - "vulnerability.score.temporal": 5.5, - "vulnerability.score.version": "2.0", - "vulnerability.textual_score": "bad", - "host.cpu.usage": 0.68, - "geo.location": [-73.614830, 45.505918], - "data_stream.dataset": "nginx.access", - "data_stream.namespace": "production", - "data_stream.custom": "whatever", - "structured_data": {"key1": "value1", "key2": ["value2", "value3"]}, - "exports": {"key": "value"}, - "top_level_imports": {"key": "value"}, - "nested.imports": {"key": "value"}, - "numeric_as_string": "42", - "socket.ip": "127.0.0.1", - "socket.remote_ip": "187.8.8.8" - } - - match: {result: "created"} - - - do: - search: - index: logs-ecs-test-subobjects - body: - query: - term: - test: - value: 'flattened' - fields: - - field: 'data_stream.type' - - field: 'location' - - field: 'geo.location' - - field: 'test.start-timestamp' - - field: 'test.start_timestamp' - - field: 'vulnerability.textual_score' - - length: { hits.hits: 1 } - # verify that data_stream.type has the correct constant_keyword value - - match: { hits.hits.0.fields.data_stream\.type.0: 'logs' } - # verify geo_point subfields evaluation - - match: { hits.hits.0.fields.location.0.type: 'Point' } - - length: { hits.hits.0.fields.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.location.0.coordinates.0: -71.34 } - - match: { hits.hits.0.fields.location.0.coordinates.1: 41.12 } - - match: { hits.hits.0.fields.geo\.location.0.type: 'Point' } - - length: { hits.hits.0.fields.geo\.location.0.coordinates: 2 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.0: -73.614830 } - - match: { hits.hits.0.fields.geo\.location.0.coordinates.1: 45.505918 } - # "start-timestamp" doesn't match the ECS dynamic mapping pattern "*_timestamp" - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.fields.test\.start-timestamp.0: 'not a date' } - - length: { hits.hits.0._ignored: 2 } - - match: { hits.hits.0._ignored.0: 'vulnerability.textual_score' } - # the ECS date dynamic template enforces mapping of "*_timestamp" fields to a date type - - match: { hits.hits.0._ignored.1: 'test.start_timestamp' } - - length: { hits.hits.0.ignored_field_values.test\.start_timestamp: 1 } - # TODO: uncomment once https://github.com/elastic/elasticsearch/issues/96700 gets resolved - # - match: { hits.hits.0.ignored_field_values.test\.start_timestamp.0: 'not a date' } - - length: { hits.hits.0.ignored_field_values.vulnerability\.textual_score: 1 } - - match: { hits.hits.0.ignored_field_values.vulnerability\.textual_score.0: 'bad' } - - - do: - indices.get_mapping: - index: logs-ecs-test-subobjects - - match: { .$idx0name.mappings.properties.error\.message.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.registry\.data\.strings.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.io\.text.type: "wildcard" } - - match: { .$idx0name.mappings.properties.email\.message_id.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.path.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.url\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.type: "wildcard" } - - match: { .$idx0name.mappings.properties.parent\.body\.content.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.command_line.type: "wildcard" } - - match: { .$idx0name.mappings.properties.process\.command_line.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.type: "wildcard" } - - match: { .$idx0name.mappings.properties.error\.stack_trace.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.type: "keyword" } - - match: { .$idx0name.mappings.properties.parent\.file\.target_path.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.os\.full.type: "keyword" } - - match: { .$idx0name.mappings.properties.os\.full.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user_agent\.original.type: "keyword" } - - match: { .$idx0name.mappings.properties.user_agent\.original.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.title.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.title.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.executable.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.executable.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.name.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.process\.working_directory.type: "keyword" } - - match: { .$idx0name.mappings.properties.process\.working_directory.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.user\.full_name.type: "keyword" } - - match: { .$idx0name.mappings.properties.user\.full_name.fields.text.type: "match_only_text" } - - match: { .$idx0name.mappings.properties.start_timestamp.type: "date" } - - match: { .$idx0name.mappings.properties.test\.start_timestamp.type: "date" } - # testing the default mapping of string input fields to keyword if not matching any pattern - - match: { .$idx0name.mappings.properties.test\.start-timestamp.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.base.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.temporal.type: "float" } - - match: { .$idx0name.mappings.properties.vulnerability\.score\.version.type: "keyword" } - - match: { .$idx0name.mappings.properties.vulnerability\.textual_score.type: "float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.type: "scaled_float" } - - match: { .$idx0name.mappings.properties.host\.cpu\.usage.scaling_factor: 1000 } - - match: { .$idx0name.mappings.properties.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.geo\.location.type: "geo_point" } - - match: { .$idx0name.mappings.properties.data_stream\.dataset.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.namespace.type: "constant_keyword" } - - match: { .$idx0name.mappings.properties.data_stream\.type.type: "constant_keyword" } - # not one of the three data_stream fields that are explicitly mapped to constant_keyword - - match: { .$idx0name.mappings.properties.data_stream\.custom.type: "keyword" } - - match: { .$idx0name.mappings.properties.structured_data.type: "flattened" } - - match: { .$idx0name.mappings.properties.exports.type: "flattened" } - - match: { .$idx0name.mappings.properties.top_level_imports.type: "flattened" } - - match: { .$idx0name.mappings.properties.nested\.imports.type: "flattened" } - # verifying the default mapping for strings into keyword, overriding the automatic numeric string detection - - match: { .$idx0name.mappings.properties.numeric_as_string.type: "keyword" } - - match: { .$idx0name.mappings.properties.socket\.ip.type: "ip" } - - match: { .$idx0name.mappings.properties.socket\.remote_ip.type: "ip" } - diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 296c692fa2d49..1ea39087211dd 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -51,6 +51,9 @@ setup: --- "Get data stream with default lifecycle": + - skip: + version: all + reason: https://github.com/elastic/elasticsearch/pull/100187 - do: indices.get_data_lifecycle: diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 46860ff38b8ca..51cc7541a9a4d 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -583,7 +583,7 @@ public String strategy() { } @Override - protected void index(DocumentParserContext context, ShapeBuilder shapeBuilder) throws IOException { + protected void index(DocumentParserContext context, ShapeBuilder shapeBuilder) { if (shapeBuilder == null) { return; } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java new file mode 100644 index 0000000000000..b182d9e8c2bde --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/CrossClusterReindexIT.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.AbstractMultiClustersTestCase; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class CrossClusterReindexIT extends AbstractMultiClustersTestCase { + + private static final String REMOTE_CLUSTER = "remote-cluster"; + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return List.of(ReindexPlugin.class); + } + + private int indexDocs(Client client, String index) { + int numDocs = between(1, 100); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("f", "v").get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + public void testReindexFromRemoteGivenIndexExists() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("source-index-001")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "source-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "source-index-001"; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("desc-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + + public void testReindexFromRemoteGivenSameIndexNames() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("test-index-001")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "test-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "test-index-001"; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("test-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("test-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + + public void testReindexManyTimesFromRemoteGivenSameIndexNames() throws Exception { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("test-index-001")); + final long docsNumber = indexDocs(client(REMOTE_CLUSTER), "test-index-001"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "test-index-001"; + + int N = randomIntBetween(2, 10); + for (int attempt = 0; attempt < N; attempt++) { + + BulkByScrollResponse response = new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source( + sourceIndexInRemote + ).destination("test-index-001").get(); + + if (attempt == 0) { + assertThat(response.getCreated(), equalTo(docsNumber)); + assertThat(response.getUpdated(), equalTo(0L)); + } else { + assertThat(response.getCreated(), equalTo(0L)); + assertThat(response.getUpdated(), equalTo(docsNumber)); + } + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("test-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + } + + public void testReindexFromRemoteThrowOnUnavailableIndex() throws Exception { + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + "no-such-source-index-001"; + expectThrows( + IndexNotFoundException.class, + () -> new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get() + ); + + // assert that local index was not created either + final IndexNotFoundException e = expectThrows( + IndexNotFoundException.class, + () -> client(LOCAL_CLUSTER).prepareSearch("desc-index-001").setQuery(new MatchAllQueryBuilder()).setSize(1000).get() + ); + assertThat(e.getMessage(), containsString("no such index [desc-index-001]")); + } + + public void testReindexFromRemoteGivenSimpleDateMathIndexName() throws InterruptedException { + assertAcked(client(REMOTE_CLUSTER).admin().indices().prepareCreate("datemath-2001-01-02")); + final int docsNumber = indexDocs(client(REMOTE_CLUSTER), "datemath-2001-01-02"); + + final String sourceIndexInRemote = REMOTE_CLUSTER + ":" + ""; + new ReindexRequestBuilder(client(LOCAL_CLUSTER), ReindexAction.INSTANCE).source(sourceIndexInRemote) + .destination("desc-index-001") + .get(); + + assertTrue("Number of documents in source and desc indexes does not match", waitUntil(() -> { + SearchResponse resp = client(LOCAL_CLUSTER).prepareSearch("desc-index-001") + .setQuery(new MatchAllQueryBuilder()) + .setSize(1000) + .get(); + final TotalHits totalHits = resp.getHits().getTotalHits(); + return totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value == docsNumber; + })); + } + +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java index aad38f64f64a5..a874dd1846e68 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexValidator.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.reindex.RemoteInfo; import org.elasticsearch.search.builder.SearchSourceBuilder; +import java.util.Arrays; import java.util.List; public class ReindexValidator { @@ -138,7 +139,12 @@ static void validateAgainstAliases( */ target = indexNameExpressionResolver.concreteWriteIndex(clusterState, destination).getName(); } - for (String sourceIndex : indexNameExpressionResolver.concreteIndexNames(clusterState, source)) { + SearchRequest filteredSource = skipRemoteIndexNames(source); + if (filteredSource.indices().length == 0) { + return; + } + String[] sourceIndexNames = indexNameExpressionResolver.concreteIndexNames(clusterState, filteredSource); + for (String sourceIndex : sourceIndexNames) { if (sourceIndex.equals(target)) { ActionRequestValidationException e = new ActionRequestValidationException(); e.addValidationError("reindex cannot write into an index its reading from [" + target + ']'); @@ -146,4 +152,14 @@ static void validateAgainstAliases( } } } + + private static SearchRequest skipRemoteIndexNames(SearchRequest source) { + return new SearchRequest(source).indices( + Arrays.stream(source.indices()).filter(name -> isRemoteExpression(name) == false).toArray(String[]::new) + ); + } + + private static boolean isRemoteExpression(String expression) { + return expression.contains(":"); + } } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index f371d6f354763..3ff0497b42719 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -82,6 +82,10 @@ class S3BlobStore implements BlobStore { private final StatsCollectors statsCollectors = new StatsCollectors(); + private static final TimeValue RETRY_STATS_WINDOW = TimeValue.timeValueMinutes(5); + + private volatile S3RequestRetryStats s3RequestRetryStats; + S3BlobStore( S3Service service, String bucket, @@ -105,10 +109,23 @@ class S3BlobStore implements BlobStore { this.threadPool = threadPool; this.snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); this.meter = meter; + s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); + threadPool.scheduleWithFixedDelay(() -> { + var priorRetryStats = s3RequestRetryStats; + s3RequestRetryStats = new S3RequestRetryStats(getMaxRetries()); + priorRetryStats.emitMetrics(); + }, RETRY_STATS_WINDOW, threadPool.generic()); } RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) { - return statsCollectors.getMetricCollector(operation, purpose); + var collector = statsCollectors.getMetricCollector(operation, purpose); + return new RequestMetricCollector() { + @Override + public void collectMetrics(Request request, Response response) { + s3RequestRetryStats.addRequest(request); + collector.collectMetrics(request, response); + } + }; } public Executor getSnapshotExecutor() { @@ -178,7 +195,7 @@ public AmazonS3Reference clientReference() { return service.client(repositoryMetadata); } - int getMaxRetries() { + final int getMaxRetries() { return service.settings(repositoryMetadata).maxRetries; } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java new file mode 100644 index 0000000000000..952668f370161 --- /dev/null +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RequestRetryStats.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories.s3; + +import com.amazonaws.Request; +import com.amazonaws.util.AWSRequestMetrics; +import com.amazonaws.util.TimingInfo; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.ESLogMessage; +import org.elasticsearch.common.util.Maps; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicLongArray; + +/** + * This class emit aws s3 metrics as logs until we have a proper apm integration + */ +public class S3RequestRetryStats { + + private static final Logger logger = LogManager.getLogger(S3RequestRetryStats.class); + + private final AtomicLong requests = new AtomicLong(); + private final AtomicLong exceptions = new AtomicLong(); + private final AtomicLong throttles = new AtomicLong(); + private final AtomicLongArray exceptionsHistogram; + private final AtomicLongArray throttlesHistogram; + + public S3RequestRetryStats(int maxRetries) { + this.exceptionsHistogram = new AtomicLongArray(maxRetries + 1); + this.throttlesHistogram = new AtomicLongArray(maxRetries + 1); + } + + public void addRequest(Request request) { + if (request == null) { + return; + } + var info = request.getAWSRequestMetrics().getTimingInfo(); + long requests = getCounter(info, AWSRequestMetrics.Field.RequestCount); + long exceptions = getCounter(info, AWSRequestMetrics.Field.Exception); + long throttles = getCounter(info, AWSRequestMetrics.Field.ThrottleException); + + this.requests.addAndGet(requests); + this.exceptions.addAndGet(exceptions); + this.throttles.addAndGet(throttles); + if (exceptions >= 0 && exceptions < this.exceptionsHistogram.length()) { + this.exceptionsHistogram.incrementAndGet((int) exceptions); + } + if (throttles >= 0 && throttles < this.throttlesHistogram.length()) { + this.throttlesHistogram.incrementAndGet((int) throttles); + } + } + + private static long getCounter(TimingInfo info, AWSRequestMetrics.Field field) { + var counter = info.getCounter(field.name()); + return counter != null ? counter.longValue() : 0L; + } + + public void emitMetrics() { + if (logger.isDebugEnabled()) { + var metrics = Maps.newMapWithExpectedSize(3); + metrics.put("elasticsearch.metrics.s3.requests", requests.get()); + metrics.put("elasticsearch.metrics.s3.exceptions", exceptions.get()); + metrics.put("elasticsearch.metrics.s3.throttles", throttles.get()); + for (int i = 0; i < exceptionsHistogram.length(); i++) { + long exceptions = exceptionsHistogram.get(i); + if (exceptions != 0) { + metrics.put("elasticsearch.metrics.s3.exceptions.h" + i, exceptions); + } + } + for (int i = 0; i < throttlesHistogram.length(); i++) { + long throttles = throttlesHistogram.get(i); + if (throttles != 0) { + metrics.put("elasticsearch.metrics.s3.throttles.h" + i, throttles); + } + } + logger.debug(new ESLogMessage().withFields(metrics)); + } + } +} diff --git a/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml index 4508dacbfe7e9..01152a5930f47 100644 --- a/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml +++ b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml @@ -167,7 +167,7 @@ teardown: - match: {count: 3} - do: - catch: /cannot delete snapshot from a readonly repository/ + catch: /repository is readonly/ snapshot.delete: repository: repository-url snapshot: snapshot-two @@ -229,7 +229,7 @@ teardown: - match: {count: 3} - do: - catch: /cannot delete snapshot from a readonly repository/ + catch: /repository is readonly/ snapshot.delete: repository: repository-file snapshot: snapshot-one diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index 6c7fa4d4653d2..01f330a93e8fa 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 08d64e2b9353b..13256179b0a2b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -41,6 +41,12 @@ excludeList.add('aggregations/filter/Standard queries get cached') excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') +// The test checks that tsdb mappings report source as synthetic. +// It is supposed to be skipped (not needed) for versions before +// 8.10 but mixed cluster tests may not respect that - see the +// comment above. +excludeList.add('tsdb/20_mapping/Synthetic source') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java index 1b53a64fb096d..f7caf4805be15 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/MixedClusterClientYamlTestSuiteIT.java @@ -15,7 +15,7 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -@TimeoutSuite(millis = 40 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell +@TimeoutSuite(millis = 60 * TimeUnits.MINUTE) // some of the windows test VMs are slow as hell public class MixedClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { public MixedClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle index 80d316536e09e..8398e3b8aeb1a 100644 --- a/qa/repository-multi-version/build.gradle +++ b/qa/repository-multi-version/build.gradle @@ -29,7 +29,7 @@ BuildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName -> numberOfNodes = 2 setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'xpack.security.enabled', 'false' - if (v.equals('8.10.0') || v.equals('8.10.1') || v.equals('8.10.2')) { + if (v.equals('8.10.0') || v.equals('8.10.1') || v.equals('8.10.2') || v.equals('8.10.3')) { // 8.10.x versions contain a bogus assertion that trips when reading repositories touched by newer versions // see https://github.com/elastic/elasticsearch/issues/98454 for details jvmArgs '-da' diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json index ffcd30fa6c717..c038ac4f3b749 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/esql.query.json @@ -5,7 +5,7 @@ "description":"Executes an ESQL request" }, "stability":"experimental", - "visibility":"private", + "visibility":"public", "headers":{ "accept": [ "application/json"], "content_type": ["application/json"] @@ -32,7 +32,7 @@ } }, "body":{ - "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and 'columnar' to format the answer.", + "description":"Use the `query` element to start a query. Use `time_zone` to specify an execution time zone and `columnar` to format the answer.", "required":true } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml index d2c02fcbff38e..4ef700f807c13 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/60_dense_vector_dynamic_mapping.yml @@ -3,6 +3,30 @@ setup: version: ' - 8.10.99' reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' + # Additional logging for issue: https://github.com/elastic/elasticsearch/issues/100502 + - do: + cluster.put_settings: + body: > + { + "persistent": { + "logger.org.elasticsearch.index": "TRACE" + } + } + +--- +teardown: + - skip: + version: ' - 8.10.99' + reason: 'Dynamic mapping of floats to dense_vector was added in 8.11' + + - do: + cluster.put_settings: + body: > + { + "persistent": { + "logger.org.elasticsearch.index": null + } + } --- "Fields with float arrays below the threshold still map as float": diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index d3aed4a3e2bf2..f556486795c2a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -35,11 +35,15 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; @@ -70,6 +74,8 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.gateway.ReplicaShardAllocatorIT; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -85,6 +91,7 @@ import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.index.shard.GlobalCheckpointListeners; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; @@ -122,7 +129,9 @@ import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -132,6 +141,7 @@ import static java.util.stream.Collectors.toList; import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; import static org.elasticsearch.node.RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING; @@ -1688,6 +1698,104 @@ public void testWaitForClusterStateToBeAppliedOnSourceNode() throws Exception { } } + public void testDeleteIndexDuringFinalization() throws Exception { + internalCluster().startMasterOnlyNode(); + final var primaryNode = internalCluster().startDataOnlyNode(); + String indexName = "test-index"; + createIndex(indexName, indexSettings(1, 0).build()); + ensureGreen(indexName); + final List indexRequests = IntStream.range(0, between(10, 500)) + .mapToObj(n -> client().prepareIndex(indexName).setSource("foo", "bar")) + .toList(); + indexRandom(randomBoolean(), true, true, indexRequests); + assertThat(indicesAdmin().prepareFlush(indexName).get().getFailedShards(), equalTo(0)); + + final var replicaNode = internalCluster().startDataOnlyNode(); + + final SubscribableListener recoveryCompleteListener = new SubscribableListener<>(); + final PlainActionFuture deleteListener = new PlainActionFuture<>(); + + final var threadPool = internalCluster().clusterService().threadPool(); + + final var indexId = internalCluster().clusterService().state().routingTable().index(indexName).getIndex(); + final var primaryIndexShard = internalCluster().getInstance(IndicesService.class, primaryNode) + .indexServiceSafe(indexId) + .getShard(0); + final var globalCheckpointBeforeRecovery = primaryIndexShard.getLastSyncedGlobalCheckpoint(); + + final var replicaNodeTransportService = asInstanceOf( + MockTransportService.class, + internalCluster().getInstance(TransportService.class, replicaNode) + ); + replicaNodeTransportService.addRequestHandlingBehavior( + PeerRecoveryTargetService.Actions.TRANSLOG_OPS, + (handler, request, channel, task) -> handler.messageReceived( + request, + new TestTransportChannel(ActionTestUtils.assertNoFailureListener(response -> { + // Process the TRANSLOG_OPS response on the replica (avoiding failing it due to a concurrent delete) but + // before sending the response back send another document to the primary, advancing the GCP to prevent the replica + // being marked as in-sync (NB below we delay the replica write until after the index is deleted) + client().prepareIndex(indexName).setSource("foo", "baz").execute(ActionListener.noop()); + + primaryIndexShard.addGlobalCheckpointListener( + globalCheckpointBeforeRecovery + 1, + new GlobalCheckpointListeners.GlobalCheckpointListener() { + @Override + public Executor executor() { + return EsExecutors.DIRECT_EXECUTOR_SERVICE; + } + + @Override + public void accept(long globalCheckpoint, Exception e) { + assertNull(e); + + // Now the GCP has advanced the replica won't be marked in-sync so respond to the TRANSLOG_OPS request + // to start recovery finalization + try { + channel.sendResponse(response); + } catch (IOException ex) { + fail(ex); + } + + // Wait a short while for finalization to block on advancing the replica's GCP and then delete the index + threadPool.schedule( + () -> client().admin().indices().prepareDelete(indexName).execute(deleteListener), + TimeValue.timeValueMillis(100), + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + } + }, + TimeValue.timeValueSeconds(10) + ); + })), + task + ) + ); + + // delay the delivery of the replica write until the end of the test so the replica never becomes in-sync + replicaNodeTransportService.addRequestHandlingBehavior( + BulkAction.NAME + "[s][r]", + (handler, request, channel, task) -> recoveryCompleteListener.addListener( + assertNoFailureListener(ignored -> handler.messageReceived(request, channel, task)) + ) + ); + + // Create the replica to trigger the whole process + assertAcked( + client().admin() + .indices() + .prepareUpdateSettings(indexName) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + ); + + // Wait for the index to be deleted + assertTrue(deleteListener.get(20, TimeUnit.SECONDS).isAcknowledged()); + + final var peerRecoverySourceService = internalCluster().getInstance(PeerRecoverySourceService.class, primaryNode); + assertBusy(() -> assertEquals(0, peerRecoverySourceService.numberOfOngoingRecoveries())); + recoveryCompleteListener.onResponse(null); + } + private void assertGlobalCheckpointIsStableAndSyncedInAllNodes(String indexName, List nodes, int shard) throws Exception { assertThat(nodes, is(not(empty()))); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 7fa59f0b47b61..71d036cc6b0f0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -1050,7 +1050,7 @@ public void testReadonlyRepository() throws Exception { assertRequestBuilderThrows( client.admin().cluster().prepareDeleteSnapshot("readonly-repo", "test-snap"), RepositoryException.class, - "cannot delete snapshot from a readonly repository" + "repository is readonly" ); logger.info("--> try making another snapshot"); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 6267fb3b86ae4..6d323c4fc2ea7 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -134,6 +134,7 @@ static TransportVersion def(int id) { public static final TransportVersion NODE_INFO_REQUEST_SIMPLIFIED = def(8_510_00_0); public static final TransportVersion NESTED_KNN_VECTOR_QUERY_V = def(8_511_00_0); public static final TransportVersion ML_PACKAGE_LOADER_PLATFORM_ADDED = def(8_512_00_0); + public static final TransportVersion ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH = def(8_512_00_1); public static final TransportVersion PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME = def(8_513_00_0); public static final TransportVersion UNIVERSAL_PROFILING_LICENSE_ADDED = def(8_514_00_0); public static final TransportVersion ELSER_SERVICE_MODEL_VERSION_ADDED = def(8_515_00_0); @@ -198,7 +199,7 @@ static TransportVersion def(int id) { * Reference to the minimum transport version that can be used with CCS. * This should be the transport version used by the previous minor release. */ - public static final TransportVersion MINIMUM_CCS_VERSION = V_8_500_061; + public static final TransportVersion MINIMUM_CCS_VERSION = ML_PACKAGE_LOADER_PLATFORM_ADDED; static final NavigableMap VERSION_IDS = getAllVersionIds(TransportVersions.class); diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 4d578e77e56bc..69eaf17addb88 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -114,6 +114,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_12 = new Version(7_17_12_99); public static final Version V_7_17_13 = new Version(7_17_13_99); public static final Version V_7_17_14 = new Version(7_17_14_99); + public static final Version V_7_17_15 = new Version(7_17_15_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); public static final Version V_8_1_0 = new Version(8_01_00_99); @@ -151,6 +152,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_10_1 = new Version(8_10_01_99); public static final Version V_8_10_2 = new Version(8_10_02_99); public static final Version V_8_10_3 = new Version(8_10_03_99); + public static final Version V_8_10_4 = new Version(8_10_04_99); public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version V_8_12_0 = new Version(8_12_00_99); public static final Version CURRENT = V_8_12_0; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index bd9382aeaa758..1a626fe4dce31 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -213,6 +213,8 @@ public void onFailure(Exception e) { public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { startedCleanup = true; logger.debug("Initialized repository cleanup in cluster state for [{}][{}]", repositoryName, repositoryStateId); + // We fork here just to call SnapshotsService#minCompatibleVersion (which may be to expensive to run directly) but + // BlobStoreRepository#cleanup forks again straight away. TODO reduce the forking here. threadPool.executor(ThreadPool.Names.SNAPSHOT) .execute( ActionRunnable.wrap( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 7ae410a1a9dcb..ad287e1c6b005 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -22,7 +22,6 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -191,7 +190,7 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { if (geometry == null) { return; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index aeab22a6f5f35..c5d5dbec1ef15 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -101,7 +101,20 @@ public static class Builder extends MetadataFieldMapper.Builder { (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false) ); - private final Parameter mode; + /* + * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic + * source mode. + */ + private final Parameter mode = new Parameter<>( + "mode", + true, + () -> null, + (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), + m -> toType(m).enabled.explicit() ? null : toType(m).mode, + (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), + v -> v.toString().toLowerCase(Locale.ROOT) + ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) + .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured private final Parameter> includes = Parameter.stringArrayParam( "includes", false, @@ -115,22 +128,9 @@ public static class Builder extends MetadataFieldMapper.Builder { private final IndexMode indexMode; - public Builder(IndexMode indexMode, IndexVersion indexVersion) { + public Builder(IndexMode indexMode) { super(Defaults.NAME); this.indexMode = indexMode; - this.mode = new Parameter<>( - "mode", - true, - // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. - () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) - ? Mode.SYNTHETIC - : null, - (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), - m -> toType(m).enabled.explicit() ? null : toType(m).mode, - (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), - v -> v.toString().toLowerCase(Locale.ROOT) - ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) - .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured } public Builder setSynthetic() { @@ -188,7 +188,7 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) + c -> new Builder(c.getIndexSettings().getMode()) ); static final class SourceFieldType extends MappedFieldType { @@ -313,7 +313,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, IndexVersion.current()).init(this); + return new Builder(indexMode).init(this); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index deb178ff724bb..ee144b25f4507 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -458,6 +458,15 @@ void checkVectorMagnitude( ) { StringBuilder errorBuilder = null; + if (Float.isNaN(squaredMagnitude) || Float.isInfinite(squaredMagnitude)) { + errorBuilder = new StringBuilder( + "NaN or Infinite magnitude detected, this usually means the vector values are too extreme to fit within a float." + ); + } + if (errorBuilder != null) { + throw new IllegalArgumentException(appender.apply(errorBuilder).toString()); + } + if (similarity == VectorSimilarity.DOT_PRODUCT && Math.abs(squaredMagnitude - 1.0f) > 1e-4f) { errorBuilder = new StringBuilder( "The [" + VectorSimilarity.DOT_PRODUCT + "] similarity can only be used with unit-length vectors." @@ -886,7 +895,9 @@ public Query createKnnQuery( } elementType.checkVectorBounds(queryVector); - if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { + if (similarity == VectorSimilarity.DOT_PRODUCT + || similarity == VectorSimilarity.COSINE + || similarity == VectorSimilarity.MAX_INNER_PRODUCT) { float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); elementType.checkVectorMagnitude(similarity, ElementType.errorFloatElementsAppender(queryVector), squaredMagnitude); } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index fc5df1a4aa282..81bc226102f62 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -426,7 +425,7 @@ public void onFailure(Exception e) { } static void runUnderPrimaryPermit( - CheckedRunnable action, + Runnable action, IndexShard primary, CancellableThreads cancellableThreads, ActionListener listener @@ -1260,7 +1259,7 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis */ final SubscribableListener markInSyncStep = new SubscribableListener<>(); runUnderPrimaryPermit( - () -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint), + () -> cancellableThreads.execute(() -> shard.markAllocationIdAsInSync(request.targetAllocationId(), targetLocalCheckpoint)), shard, cancellableThreads, markInSyncStep diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 9a2d53312d577..39d11e9d9a4f3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -785,16 +785,16 @@ public RepositoryStats stats() { /** * Loads {@link RepositoryData} ensuring that it is consistent with the given {@code rootBlobs} as well of the assumed generation. * - * @param repositoryStateId Expected repository generation - * @param rootBlobs Blobs at the repository root + * @param repositoryDataGeneration Expected repository generation + * @param rootBlobs Blobs at the repository root * @return RepositoryData */ - private RepositoryData safeRepositoryData(long repositoryStateId, Map rootBlobs) { + private RepositoryData safeRepositoryData(long repositoryDataGeneration, Map rootBlobs) { final long generation = latestGeneration(rootBlobs.keySet()); final long genToLoad; final RepositoryData cached; if (bestEffortConsistency) { - genToLoad = latestKnownRepoGen.accumulateAndGet(repositoryStateId, Math::max); + genToLoad = latestKnownRepoGen.accumulateAndGet(repositoryDataGeneration, Math::max); cached = null; } else { genToLoad = latestKnownRepoGen.get(); @@ -813,11 +813,11 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map rootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); - final RepositoryData repositoryData = safeRepositoryData(repositoryDataGeneration, rootBlobs); - // Cache the indices that were found before writing out the new index-N blob so that a stuck master will never - // delete an index that was created by another master node after writing this index-N blob. - final Map foundIndices = blobStore().blobContainer(indicesPath()) - .children(OperationPurpose.SNAPSHOT); - doDeleteShardSnapshots( - snapshotIds, - repositoryDataGeneration, - foundIndices, - rootBlobs, - repositoryData, - repositoryFormatIndexVersion, - listener - ); - } + createSnapshotsDeletion(snapshotIds, repositoryDataGeneration, repositoryFormatIndexVersion, new ActionListener<>() { + @Override + public void onResponse(SnapshotsDeletion snapshotsDeletion) { + snapshotsDeletion.runDelete(listener); + } - @Override - public void onFailure(Exception e) { - listener.onFailure(new RepositoryException(metadata.name(), "failed to delete snapshots " + snapshotIds, e)); - } - }); - } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); } /** - * The result of removing a snapshot from a shard folder in the repository. + * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the + * repository. + * TODO: Add shard level cleanups + * TODO: Add unreferenced index metadata cleanup + *
    + *
  • Deleting stale indices
    • + *
  • Deleting unreferenced root level blobs
    • + *
* - * @param indexId Index that the snapshot was removed from - * @param shardId Shard id that the snapshot was removed from - * @param newGeneration Id of the new index-${uuid} blob that does not include the snapshot any more - * @param blobsToDelete Blob names in the shard directory that have become unreferenced in the new shard generation + * @param repositoryDataGeneration Generation of {@link RepositoryData} at start of process + * @param repositoryFormatIndexVersion Repository format version + * @param listener Listener to complete when done */ - private record ShardSnapshotMetaDeleteResult( - IndexId indexId, - int shardId, - ShardGeneration newGeneration, - Collection blobsToDelete - ) {} - - // --------------------------------------------------------------------------------------------------------------------------------- - // The overall flow of execution + public void cleanup( + long repositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + ActionListener listener + ) { + createSnapshotsDeletion( + List.of(), + repositoryDataGeneration, + repositoryFormatIndexVersion, + listener.delegateFailureAndWrap((delegate, snapshotsDeletion) -> snapshotsDeletion.runCleanup(delegate)) + ); + } - /** - * After updating the {@link RepositoryData} each of the shards directories is individually first moved to the next shard generation - * and then has all now unreferenced blobs in it deleted. - * - * @param snapshotIds SnapshotIds to delete - * @param originalRepositoryDataGeneration {@link RepositoryData} generation at the start of the process. - * @param originalIndexContainers All index containers at the start of the operation, obtained by listing the repository - * contents. - * @param originalRootBlobs All blobs found at the root of the repository at the start of the operation, obtained by - * listing the repository contents. - * @param originalRepositoryData {@link RepositoryData} at the start of the operation. - * @param repositoryFormatIndexVersion The minimum {@link IndexVersion} of the nodes in the cluster and the snapshots remaining in - * the repository. - * @param listener Listener to invoke once finished - */ - private void doDeleteShardSnapshots( + private void createSnapshotsDeletion( Collection snapshotIds, - long originalRepositoryDataGeneration, - Map originalIndexContainers, - Map originalRootBlobs, - RepositoryData originalRepositoryData, + long repositoryDataGeneration, IndexVersion repositoryFormatIndexVersion, - SnapshotDeleteListener listener + ActionListener listener ) { - if (SnapshotsService.useShardGenerations(repositoryFormatIndexVersion)) { - // First write the new shard state metadata (with the removed snapshot) and compute deletion targets - final ListenableFuture> writeShardMetaDataAndComputeDeletesStep = - new ListenableFuture<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, originalRepositoryData, true, writeShardMetaDataAndComputeDeletesStep); - // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: - // 1. Remove the snapshots from the list of existing snapshots - // 2. Update the index shard generations of all updated shard folders - // - // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created - // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only - // written if all shard paths have been successfully updated. - final ListenableFuture writeUpdatedRepoDataStep = new ListenableFuture<>(); - writeShardMetaDataAndComputeDeletesStep.addListener(ActionListener.wrap(shardDeleteResults -> { - final ShardGenerations.Builder builder = ShardGenerations.builder(); - for (ShardSnapshotMetaDeleteResult newGen : shardDeleteResults) { - builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); - } - final RepositoryData newRepositoryData = originalRepositoryData.removeSnapshots(snapshotIds, builder.build()); - writeIndexGen( - newRepositoryData, - originalRepositoryDataGeneration, + if (isReadOnly()) { + listener.onFailure(new RepositoryException(metadata.name(), "repository is readonly")); + } else { + threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.supply(listener, () -> { + final var originalRootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); + return new SnapshotsDeletion( + snapshotIds, + repositoryDataGeneration, repositoryFormatIndexVersion, - Function.identity(), - ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure) + originalRootBlobs, + blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT), + safeRepositoryData(repositoryDataGeneration, originalRootBlobs) ); - }, listener::onFailure)); - // Once we have updated the repository, run the clean-ups - writeUpdatedRepoDataStep.addListener(ActionListener.wrap(newRepositoryData -> { - listener.onRepositoryDataWritten(newRepositoryData); - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - try (var refs = new RefCountingRunnable(listener::onDone)) { - cleanupUnlinkedRootAndIndicesBlobs( - snapshotIds, - originalIndexContainers, - originalRootBlobs, + })); + } + } + + /** + *

+ * Represents the process of deleting some collection of snapshots within this repository which since 7.6.0 looks like this: + *

+ *
    + *
  • Write a new {@link BlobStoreIndexShardSnapshots} for each affected shard, and compute the blobs to delete.
    • + *
  • Update the {@link RepositoryData} to remove references to deleted snapshots/indices and point to the new + * {@link BlobStoreIndexShardSnapshots} files.
    • + *
  • Remove up any now-unreferenced blobs.
    • + *
+ *

+ * Until the {@link RepositoryData} is updated there should be no other activities in the repository, and in particular the root + * blob must not change until it is updated by this deletion and {@link SnapshotDeleteListener#onRepositoryDataWritten} is called. + *

+ */ + class SnapshotsDeletion { + + /** + * The IDs of the snapshots to delete. This collection is empty if the deletion is a repository cleanup. + */ + private final Collection snapshotIds; + + /** + * The {@link RepositoryData} generation at the start of the process, to ensure that the {@link RepositoryData} does not change + * while the new {@link BlobStoreIndexShardSnapshots} are being written. + */ + private final long originalRepositoryDataGeneration; + + /** + * The minimum {@link IndexVersion} of the nodes in the cluster and the snapshots remaining in the repository. The repository must + * remain readable by all node versions which support this {@link IndexVersion}. + */ + private final IndexVersion repositoryFormatIndexVersion; + + /** + * Whether the {@link #repositoryFormatIndexVersion} is new enough to support naming {@link BlobStoreIndexShardSnapshots} blobs with + * UUIDs (i.e. does not need to remain compatible with versions before v7.6.0). Older repositories use (unsafe) numeric indices for + * these blobs instead. + */ + private final boolean useShardGenerations; + + /** + * All blobs in the repository root at the start of the operation, obtained by listing the repository contents. Note that this may + * include some blobs which are no longer referenced by the current {@link RepositoryData}, but which have not yet been removed by + * the cleanup that follows an earlier deletion. This cleanup may still be ongoing (we do not wait for it to complete before + * starting the next repository operation) or it may have failed before completion (it could have been running on a different node, + * which crashed for unrelated reasons) so we track all the blobs here and clean them up again at the end. + */ + private final Map originalRootBlobs; + + /** + * All index containers at the start of the operation, obtained by listing the repository contents. Note that this may include some + * containers which are no longer referenced by the current {@link RepositoryData}, but which have not yet been removed by + * the cleanup that follows an earlier deletion. This cleanup may or may not still be ongoing (it could have been running on a + * different node, which died before completing it) so we track all the blobs here and clean them up again at the end. + */ + private final Map originalIndexContainers; + + /** + * The {@link RepositoryData} at the start of the operation, obtained after verifying that {@link #originalRootBlobs} contains no + * {@link RepositoryData} blob newer than the one identified by {@link #originalRepositoryDataGeneration}. + */ + private final RepositoryData originalRepositoryData; + + /** + * Executor to use for all repository interactions. + */ + private final Executor snapshotExecutor = threadPool.executor(ThreadPool.Names.SNAPSHOT); + + SnapshotsDeletion( + Collection snapshotIds, + long originalRepositoryDataGeneration, + IndexVersion repositoryFormatIndexVersion, + Map originalRootBlobs, + Map originalIndexContainers, + RepositoryData originalRepositoryData + ) { + this.snapshotIds = snapshotIds; + this.originalRepositoryDataGeneration = originalRepositoryDataGeneration; + this.repositoryFormatIndexVersion = repositoryFormatIndexVersion; + this.useShardGenerations = SnapshotsService.useShardGenerations(repositoryFormatIndexVersion); + this.originalRootBlobs = originalRootBlobs; + this.originalIndexContainers = originalIndexContainers; + this.originalRepositoryData = originalRepositoryData; + } + + /** + * The result of removing a snapshot from a shard folder in the repository. + * + * @param indexId Index that the snapshot was removed from + * @param shardId Shard id that the snapshot was removed from + * @param newGeneration Id of the new index-${uuid} blob that does not include the snapshot any more + * @param blobsToDelete Blob names in the shard directory that have become unreferenced in the new shard generation + */ + private record ShardSnapshotMetaDeleteResult( + IndexId indexId, + int shardId, + ShardGeneration newGeneration, + Collection blobsToDelete + ) {} + + // --------------------------------------------------------------------------------------------------------------------------------- + // The overall flow of execution + + void runDelete(SnapshotDeleteListener listener) { + if (useShardGenerations) { + // First write the new shard state metadata (with the removed snapshot) and compute deletion targets + final ListenableFuture> writeShardMetaDataAndComputeDeletesStep = + new ListenableFuture<>(); + writeUpdatedShardMetaDataAndComputeDeletes(writeShardMetaDataAndComputeDeletesStep); + // Once we have put the new shard-level metadata into place, we can update the repository metadata as follows: + // 1. Remove the snapshots from the list of existing snapshots + // 2. Update the index shard generations of all updated shard folders + // + // Note: If we fail updating any of the individual shard paths, none of them are changed since the newly created + // index-${gen_uuid} will not be referenced by the existing RepositoryData and new RepositoryData is only + // written if all shard paths have been successfully updated. + final ListenableFuture writeUpdatedRepoDataStep = new ListenableFuture<>(); + writeShardMetaDataAndComputeDeletesStep.addListener(ActionListener.wrap(shardDeleteResults -> { + final ShardGenerations.Builder builder = ShardGenerations.builder(); + for (ShardSnapshotMetaDeleteResult newGen : shardDeleteResults) { + builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); + } + final RepositoryData newRepositoryData = originalRepositoryData.removeSnapshots(snapshotIds, builder.build()); + writeIndexGen( newRepositoryData, - refs.acquireListener() - ); - cleanupUnlinkedShardLevelBlobs( - originalRepositoryData, - snapshotIds, - writeShardMetaDataAndComputeDeletesStep.result(), - refs.acquireListener() + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, + Function.identity(), + ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure) ); - } - }, listener::onFailure)); - } else { - // Write the new repository data first (with the removed snapshot), using no shard generations - writeIndexGen( - originalRepositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY), - originalRepositoryDataGeneration, - repositoryFormatIndexVersion, - Function.identity(), - ActionListener.wrap(newRepositoryData -> { - try (var refs = new RefCountingRunnable(() -> { - listener.onRepositoryDataWritten(newRepositoryData); - listener.onDone(); - })) { - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - cleanupUnlinkedRootAndIndicesBlobs( - snapshotIds, - originalIndexContainers, - originalRootBlobs, - newRepositoryData, - refs.acquireListener() - ); - - // writeIndexGen finishes on master-service thread so must fork here. - threadPool.executor(ThreadPool.Names.SNAPSHOT) - .execute( + }, listener::onFailure)); + // Once we have updated the repository, run the clean-ups + writeUpdatedRepoDataStep.addListener(ActionListener.wrap(newRepositoryData -> { + listener.onRepositoryDataWritten(newRepositoryData); + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion + try (var refs = new RefCountingRunnable(listener::onDone)) { + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener().map(ignored -> null)); + cleanupUnlinkedShardLevelBlobs(writeShardMetaDataAndComputeDeletesStep.result(), refs.acquireListener()); + } + }, listener::onFailure)); + } else { + // Write the new repository data first (with the removed snapshot), using no shard generations + writeIndexGen( + originalRepositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY), + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, + Function.identity(), + ActionListener.wrap(newRepositoryData -> { + try (var refs = new RefCountingRunnable(() -> { + listener.onRepositoryDataWritten(newRepositoryData); + listener.onDone(); + })) { + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion + cleanupUnlinkedRootAndIndicesBlobs(newRepositoryData, refs.acquireListener().map(ignored -> null)); + + // writeIndexGen finishes on master-service thread so must fork here. + snapshotExecutor.execute( ActionRunnable.wrap( refs.acquireListener(), l0 -> writeUpdatedShardMetaDataAndComputeDeletes( - snapshotIds, - originalRepositoryData, - false, - l0.delegateFailure( - (l, deleteResults) -> cleanupUnlinkedShardLevelBlobs( - originalRepositoryData, - snapshotIds, - deleteResults, - l - ) - ) + l0.delegateFailure((l, shardDeleteResults) -> cleanupUnlinkedShardLevelBlobs(shardDeleteResults, l)) ) ) ); - } - }, listener::onFailure) - ); + } + }, listener::onFailure) + ); + } } - } - - // --------------------------------------------------------------------------------------------------------------------------------- - // Updating the shard-level metadata and accumulating results - // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. - private void writeUpdatedShardMetaDataAndComputeDeletes( - Collection snapshotIds, - RepositoryData originalRepositoryData, - boolean useShardGenerations, - ActionListener> onAllShardsCompleted - ) { + void runCleanup(ActionListener listener) { + final Set survivingIndexIds = originalRepositoryData.getIndices() + .values() + .stream() + .map(IndexId::getId) + .collect(Collectors.toSet()); + final List staleRootBlobs = staleRootBlobs(originalRepositoryData, originalRootBlobs.keySet()); + if (survivingIndexIds.equals(originalIndexContainers.keySet()) && staleRootBlobs.isEmpty()) { + // Nothing to clean up we return + listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); + } else { + // write new index-N blob to ensure concurrent operations will fail + writeIndexGen( + originalRepositoryData, + originalRepositoryDataGeneration, + repositoryFormatIndexVersion, + Function.identity(), + listener.delegateFailureAndWrap( + // TODO should we pass newRepositoryData to cleanupStaleBlobs()? + (l, newRepositoryData) -> cleanupUnlinkedRootAndIndicesBlobs( + originalRepositoryData, + l.map(RepositoryCleanupResult::new) + ) + ) + ); + } + } - final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); - final List indices = originalRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); + // --------------------------------------------------------------------------------------------------------------------------------- + // Updating the shard-level metadata and accumulating results - if (indices.isEmpty()) { - onAllShardsCompleted.onResponse(Collections.emptyList()); - return; - } + // updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up. + private void writeUpdatedShardMetaDataAndComputeDeletes( + ActionListener> onAllShardsCompleted + ) { - // Listener that flattens out the delete results for each index - final ActionListener> deleteIndexMetadataListener = new GroupedActionListener<>( - indices.size(), - onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList()) - ); + final List indices = originalRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds); - for (IndexId indexId : indices) { - final Set snapshotsWithIndex = Set.copyOf(originalRepositoryData.getSnapshots(indexId)); - final Set survivingSnapshots = snapshotsWithIndex.stream() - .filter(id -> snapshotIds.contains(id) == false) - .collect(Collectors.toSet()); - final ListenableFuture> shardCountListener = new ListenableFuture<>(); - final Collection indexMetaGenerations = snapshotIds.stream() - .filter(snapshotsWithIndex::contains) - .map(id -> originalRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)) - .collect(Collectors.toSet()); - final ActionListener allShardCountsListener = new GroupedActionListener<>( - indexMetaGenerations.size(), - shardCountListener - ); - final BlobContainer indexContainer = indexContainer(indexId); - for (String indexMetaGeneration : indexMetaGenerations) { - executor.execute(ActionRunnable.supply(allShardCountsListener, () -> { - try { - return INDEX_METADATA_FORMAT.read(metadata.name(), indexContainer, indexMetaGeneration, namedXContentRegistry) - .getNumberOfShards(); - } catch (Exception ex) { - logger.warn( - () -> format("[%s] [%s] failed to read metadata for index", indexMetaGeneration, indexId.getName()), - ex - ); - // Just invoke the listener without any shard generations to count it down, this index will be cleaned up - // by the stale data cleanup in the end. - // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just - // ignoring it and letting the cleanup deal with it. - return null; - } - })); + if (indices.isEmpty()) { + onAllShardsCompleted.onResponse(Collections.emptyList()); + return; } - // ----------------------------------------------------------------------------------------------------------------------------- - // Determining the shard count - - shardCountListener.addListener(deleteIndexMetadataListener.delegateFailureAndWrap((delegate, counts) -> { - final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0); - if (shardCount == 0) { - delegate.onResponse(null); - return; - } - // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index - final ActionListener allShardsListener = new GroupedActionListener<>(shardCount, delegate); - for (int i = 0; i < shardCount; i++) { - final int shardId = i; - executor.execute(new AbstractRunnable() { - @Override - protected void doRun() throws Exception { - final BlobContainer shardContainer = shardContainer(indexId, shardId); - final Set originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT).keySet(); - final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; - final long newGen; - if (useShardGenerations) { - newGen = -1L; - blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( - originalShardBlobs, - shardContainer, - originalRepositoryData.shardGenerations().getShardGen(indexId, shardId) - ).v1(); - } else { - Tuple tuple = buildBlobStoreIndexShardSnapshots( - originalShardBlobs, - shardContainer - ); - newGen = tuple.v2() + 1; - blobStoreIndexShardSnapshots = tuple.v1(); - } - allShardsListener.onResponse( - deleteFromShardSnapshotMeta( - survivingSnapshots, - indexId, - shardId, - snapshotIds, - shardContainer, - originalShardBlobs, - blobStoreIndexShardSnapshots, - newGen - ) - ); - } + // Listener that flattens out the delete results for each index + final ActionListener> deleteIndexMetadataListener = new GroupedActionListener<>( + indices.size(), + onAllShardsCompleted.map(res -> res.stream().flatMap(Collection::stream).toList()) + ); - @Override - public void onFailure(Exception ex) { + for (IndexId indexId : indices) { + final Set snapshotsWithIndex = Set.copyOf(originalRepositoryData.getSnapshots(indexId)); + final Set survivingSnapshots = snapshotsWithIndex.stream() + .filter(id -> snapshotIds.contains(id) == false) + .collect(Collectors.toSet()); + final ListenableFuture> shardCountListener = new ListenableFuture<>(); + final Collection indexMetaGenerations = snapshotIds.stream() + .filter(snapshotsWithIndex::contains) + .map(id -> originalRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)) + .collect(Collectors.toSet()); + final ActionListener allShardCountsListener = new GroupedActionListener<>( + indexMetaGenerations.size(), + shardCountListener + ); + final BlobContainer indexContainer = indexContainer(indexId); + for (String indexMetaGeneration : indexMetaGenerations) { + snapshotExecutor.execute(ActionRunnable.supply(allShardCountsListener, () -> { + try { + return INDEX_METADATA_FORMAT.read(metadata.name(), indexContainer, indexMetaGeneration, namedXContentRegistry) + .getNumberOfShards(); + } catch (Exception ex) { logger.warn( - () -> format("%s failed to delete shard data for shard [%s][%s]", snapshotIds, indexId.getName(), shardId), + () -> format("[%s] [%s] failed to read metadata for index", indexMetaGeneration, indexId.getName()), ex ); - // Just passing null here to count down the listener instead of failing it, the stale data left behind - // here will be retried in the next delete or repository cleanup - allShardsListener.onResponse(null); + // Just invoke the listener without any shard generations to count it down, this index will be cleaned up + // by the stale data cleanup in the end. + // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just + // ignoring it and letting the cleanup deal with it. + return null; } - }); + })); } - })); - } - } - // ----------------------------------------------------------------------------------------------------------------------------- - // Updating each shard + // ------------------------------------------------------------------------------------------------------------------------- + // Determining the shard count - /** - * Delete snapshot from shard level metadata. - * - * @param indexGeneration generation to write the new shard level level metadata to. If negative a uuid id shard generation should be - * used - */ - private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( - Set survivingSnapshots, - IndexId indexId, - int shardId, - Collection snapshotIds, - BlobContainer shardContainer, - Set originalShardBlobs, - BlobStoreIndexShardSnapshots snapshots, - long indexGeneration - ) { - // Build a list of snapshots that should be preserved - final BlobStoreIndexShardSnapshots updatedSnapshots = snapshots.withRetainedSnapshots(survivingSnapshots); - ShardGeneration writtenGeneration = null; - try { - if (updatedSnapshots.snapshots().isEmpty()) { - return new ShardSnapshotMetaDeleteResult(indexId, shardId, ShardGenerations.DELETED_SHARD_GEN, originalShardBlobs); - } else { - if (indexGeneration < 0L) { - writtenGeneration = ShardGeneration.newGeneration(); - INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration.toBlobNamePart(), compress); + shardCountListener.addListener(deleteIndexMetadataListener.delegateFailureAndWrap((delegate, counts) -> { + final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0); + if (shardCount == 0) { + delegate.onResponse(null); + return; + } + // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index + final ActionListener allShardsListener = new GroupedActionListener<>( + shardCount, + delegate + ); + for (int i = 0; i < shardCount; i++) { + final int shardId = i; + snapshotExecutor.execute(new AbstractRunnable() { + @Override + protected void doRun() throws Exception { + final BlobContainer shardContainer = shardContainer(indexId, shardId); + final Set originalShardBlobs = shardContainer.listBlobs(OperationPurpose.SNAPSHOT).keySet(); + final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots; + final long newGen; + if (useShardGenerations) { + newGen = -1L; + blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots( + originalShardBlobs, + shardContainer, + originalRepositoryData.shardGenerations().getShardGen(indexId, shardId) + ).v1(); + } else { + Tuple tuple = buildBlobStoreIndexShardSnapshots( + originalShardBlobs, + shardContainer + ); + newGen = tuple.v2() + 1; + blobStoreIndexShardSnapshots = tuple.v1(); + } + allShardsListener.onResponse( + deleteFromShardSnapshotMeta( + survivingSnapshots, + indexId, + shardId, + snapshotIds, + shardContainer, + originalShardBlobs, + blobStoreIndexShardSnapshots, + newGen + ) + ); + } + + @Override + public void onFailure(Exception ex) { + logger.warn( + () -> format( + "%s failed to delete shard data for shard [%s][%s]", + snapshotIds, + indexId.getName(), + shardId + ), + ex + ); + // Just passing null here to count down the listener instead of failing it, the stale data left behind + // here will be retried in the next delete or repository cleanup + allShardsListener.onResponse(null); + } + }); + } + })); + } + } + + // ----------------------------------------------------------------------------------------------------------------------------- + // Updating each shard + + /** + * Delete snapshot from shard level metadata. + * + * @param indexGeneration generation to write the new shard level level metadata to. If negative a uuid id shard generation should + * be used + */ + private ShardSnapshotMetaDeleteResult deleteFromShardSnapshotMeta( + Set survivingSnapshots, + IndexId indexId, + int shardId, + Collection snapshotIds, + BlobContainer shardContainer, + Set originalShardBlobs, + BlobStoreIndexShardSnapshots snapshots, + long indexGeneration + ) { + // Build a list of snapshots that should be preserved + final BlobStoreIndexShardSnapshots updatedSnapshots = snapshots.withRetainedSnapshots(survivingSnapshots); + ShardGeneration writtenGeneration = null; + try { + if (updatedSnapshots.snapshots().isEmpty()) { + return new ShardSnapshotMetaDeleteResult(indexId, shardId, ShardGenerations.DELETED_SHARD_GEN, originalShardBlobs); } else { - writtenGeneration = new ShardGeneration(indexGeneration); - writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots, Collections.emptyMap()); + if (indexGeneration < 0L) { + writtenGeneration = ShardGeneration.newGeneration(); + INDEX_SHARD_SNAPSHOTS_FORMAT.write(updatedSnapshots, shardContainer, writtenGeneration.toBlobNamePart(), compress); + } else { + writtenGeneration = new ShardGeneration(indexGeneration); + writeShardIndexBlobAtomic(shardContainer, indexGeneration, updatedSnapshots, Collections.emptyMap()); + } + final Set survivingSnapshotUUIDs = survivingSnapshots.stream() + .map(SnapshotId::getUUID) + .collect(Collectors.toSet()); + return new ShardSnapshotMetaDeleteResult( + indexId, + shardId, + writtenGeneration, + unusedBlobs(originalShardBlobs, survivingSnapshotUUIDs, updatedSnapshots) + ); } - final Set survivingSnapshotUUIDs = survivingSnapshots.stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); - return new ShardSnapshotMetaDeleteResult( - indexId, - shardId, - writtenGeneration, - unusedBlobs(originalShardBlobs, survivingSnapshotUUIDs, updatedSnapshots) + } catch (IOException e) { + throw new RepositoryException( + metadata.name(), + "Failed to finalize snapshot deletion " + + snapshotIds + + " with shard index [" + + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.toBlobNamePart()) + + "]", + e ); } - } catch (IOException e) { - throw new RepositoryException( - metadata.name(), - "Failed to finalize snapshot deletion " - + snapshotIds - + " with shard index [" - + INDEX_SHARD_SNAPSHOTS_FORMAT.blobName(writtenGeneration.toBlobNamePart()) - + "]", - e - ); } - } - - // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all - // temporary blobs - private static List unusedBlobs( - Set originalShardBlobs, - Set survivingSnapshotUUIDs, - BlobStoreIndexShardSnapshots updatedSnapshots - ) { - return originalShardBlobs.stream() - .filter( - blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX) - || (blob.startsWith(SNAPSHOT_PREFIX) - && blob.endsWith(".dat") - && survivingSnapshotUUIDs.contains( - blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) - ) == false) - || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) - || FsBlobContainer.isTempBlobName(blob) - ) - .toList(); - } - // --------------------------------------------------------------------------------------------------------------------------------- - // Cleaning up dangling blobs + // Unused blobs are all previous index-, data- and meta-blobs and that are not referenced by the new index- as well as all + // temporary blobs + private static List unusedBlobs( + Set originalShardBlobs, + Set survivingSnapshotUUIDs, + BlobStoreIndexShardSnapshots updatedSnapshots + ) { + return originalShardBlobs.stream() + .filter( + blob -> blob.startsWith(SNAPSHOT_INDEX_PREFIX) + || (blob.startsWith(SNAPSHOT_PREFIX) + && blob.endsWith(".dat") + && survivingSnapshotUUIDs.contains( + blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()) + ) == false) + || (blob.startsWith(UPLOADED_DATA_BLOB_PREFIX) && updatedSnapshots.findNameFile(canonicalName(blob)) == null) + || FsBlobContainer.isTempBlobName(blob) + ) + .toList(); + } - /** - * Delete any dangling blobs in the repository root (i.e. {@link RepositoryData}, {@link SnapshotInfo} and {@link Metadata} blobs) - * as well as any containers for indices that are now completely unreferenced. - */ - private void cleanupUnlinkedRootAndIndicesBlobs( - Collection snapshotIds, - Map originalIndexContainers, - Map originalRootBlobs, - RepositoryData newRepositoryData, - ActionListener listener - ) { - cleanupStaleBlobs(snapshotIds, originalIndexContainers, originalRootBlobs, newRepositoryData, listener.map(ignored -> null)); - } + // --------------------------------------------------------------------------------------------------------------------------------- + // Cleaning up dangling blobs - private void cleanupUnlinkedShardLevelBlobs( - RepositoryData originalRepositoryData, - Collection snapshotIds, - Collection shardDeleteResults, - ActionListener listener - ) { - final Iterator filesToDelete = resolveFilesToDelete(originalRepositoryData, snapshotIds, shardDeleteResults); - if (filesToDelete.hasNext() == false) { - listener.onResponse(null); - return; - } - threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> { - try { - deleteFromContainer(blobContainer(), filesToDelete); - l.onResponse(null); - } catch (Exception e) { - logger.warn(() -> format("%s Failed to delete some blobs during snapshot delete", snapshotIds), e); - throw e; + private void cleanupUnlinkedShardLevelBlobs( + Collection shardDeleteResults, + ActionListener listener + ) { + final Iterator filesToDelete = resolveFilesToDelete(shardDeleteResults); + if (filesToDelete.hasNext() == false) { + listener.onResponse(null); + return; } - })); - } + snapshotExecutor.execute(ActionRunnable.wrap(listener, l -> { + try { + deleteFromContainer(blobContainer(), filesToDelete); + l.onResponse(null); + } catch (Exception e) { + logger.warn(() -> format("%s Failed to delete some blobs during snapshot delete", snapshotIds), e); + throw e; + } + })); + } - private Iterator resolveFilesToDelete( - RepositoryData oldRepositoryData, - Collection snapshotIds, - Collection deleteResults - ) { - final String basePath = basePath().buildAsString(); - final int basePathLen = basePath.length(); - final Map> indexMetaGenerations = oldRepositoryData.indexMetaDataToRemoveAfterRemovingSnapshots( - snapshotIds - ); - return Stream.concat(deleteResults.stream().flatMap(shardResult -> { - final String shardPath = shardPath(shardResult.indexId, shardResult.shardId).buildAsString(); - return shardResult.blobsToDelete.stream().map(blob -> shardPath + blob); - }), indexMetaGenerations.entrySet().stream().flatMap(entry -> { - final String indexContainerPath = indexPath(entry.getKey()).buildAsString(); - return entry.getValue().stream().map(id -> indexContainerPath + INDEX_METADATA_FORMAT.blobName(id)); - })).map(absolutePath -> { - assert absolutePath.startsWith(basePath); - return absolutePath.substring(basePathLen); - }).iterator(); - } + private Iterator resolveFilesToDelete(Collection deleteResults) { + final String basePath = basePath().buildAsString(); + final int basePathLen = basePath.length(); + final Map> indexMetaGenerations = originalRepositoryData + .indexMetaDataToRemoveAfterRemovingSnapshots(snapshotIds); + return Stream.concat(deleteResults.stream().flatMap(shardResult -> { + final String shardPath = shardPath(shardResult.indexId, shardResult.shardId).buildAsString(); + return shardResult.blobsToDelete.stream().map(blob -> shardPath + blob); + }), indexMetaGenerations.entrySet().stream().flatMap(entry -> { + final String indexContainerPath = indexPath(entry.getKey()).buildAsString(); + return entry.getValue().stream().map(id -> indexContainerPath + INDEX_METADATA_FORMAT.blobName(id)); + })).map(absolutePath -> { + assert absolutePath.startsWith(basePath); + return absolutePath.substring(basePathLen); + }).iterator(); + } + + /** + * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing + * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository. + * + * @param newRepositoryData new repository data that was just written + * @param listener listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation + */ + private void cleanupUnlinkedRootAndIndicesBlobs(RepositoryData newRepositoryData, ActionListener listener) { + final var blobsDeleted = new AtomicLong(); + final var bytesDeleted = new AtomicLong(); + try ( + var listeners = new RefCountingListener(listener.map(ignored -> DeleteResult.of(blobsDeleted.get(), bytesDeleted.get()))) + ) { - /** - * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing - * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository and with - * parameters {@code foundIndices}, {@code rootBlobs} - * - * @param deletedSnapshots if this method is called as part of a delete operation, the snapshot ids just deleted or empty if called as - * part of a repository cleanup - * @param foundIndices all indices blob containers found in the repository before {@code newRepoData} was written - * @param rootBlobs all blobs found directly under the repository root - * @param newRepoData new repository data that was just written - * @param listener listener to invoke with the combined {@link DeleteResult} of all blobs removed in this operation - */ - private void cleanupStaleBlobs( - Collection deletedSnapshots, - Map foundIndices, - Map rootBlobs, - RepositoryData newRepoData, - ActionListener listener - ) { - final var blobsDeleted = new AtomicLong(); - final var bytesDeleted = new AtomicLong(); - try (var listeners = new RefCountingListener(listener.map(ignored -> DeleteResult.of(blobsDeleted.get(), bytesDeleted.get())))) { - - final List staleRootBlobs = staleRootBlobs(newRepoData, rootBlobs.keySet()); - if (staleRootBlobs.isEmpty() == false) { - staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { - try (ref) { - logStaleRootLevelBlobs(newRepoData.getGenId() - 1, deletedSnapshots, staleRootBlobs); - deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); - for (final var staleRootBlob : staleRootBlobs) { - bytesDeleted.addAndGet(rootBlobs.get(staleRootBlob).length()); + final List staleRootBlobs = staleRootBlobs(newRepositoryData, originalRootBlobs.keySet()); + if (staleRootBlobs.isEmpty() == false) { + staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { + try (ref) { + logStaleRootLevelBlobs(newRepositoryData.getGenId() - 1, snapshotIds, staleRootBlobs); + deleteFromContainer(blobContainer(), staleRootBlobs.iterator()); + for (final var staleRootBlob : staleRootBlobs) { + bytesDeleted.addAndGet(originalRootBlobs.get(staleRootBlob).length()); + } + blobsDeleted.addAndGet(staleRootBlobs.size()); + } catch (Exception e) { + logger.warn( + () -> format( + "[%s] The following blobs are no longer part of any snapshot [%s] but failed to remove them", + metadata.name(), + staleRootBlobs + ), + e + ); } - blobsDeleted.addAndGet(staleRootBlobs.size()); - } catch (Exception e) { - logger.warn( - () -> format( - "[%s] The following blobs are no longer part of any snapshot [%s] but failed to remove them", - metadata.name(), - staleRootBlobs - ), - e - ); - } - })); - } - - final var survivingIndexIds = newRepoData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet()); - for (final var indexEntry : foundIndices.entrySet()) { - final var indexSnId = indexEntry.getKey(); - if (survivingIndexIds.contains(indexSnId)) { - continue; + })); } - staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { - try (ref) { - logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexSnId); - final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); - blobsDeleted.addAndGet(deleteResult.blobsDeleted()); - bytesDeleted.addAndGet(deleteResult.bytesDeleted()); - logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexSnId); - } catch (IOException e) { - logger.warn(() -> format(""" - [%s] index %s is no longer part of any snapshot in the repository, \ - but failed to clean up its index folder""", metadata.name(), indexSnId), e); + + final var survivingIndexIds = newRepositoryData.getIndices() + .values() + .stream() + .map(IndexId::getId) + .collect(Collectors.toSet()); + for (final var indexEntry : originalIndexContainers.entrySet()) { + final var indexId = indexEntry.getKey(); + if (survivingIndexIds.contains(indexId)) { + continue; } - })); + staleBlobDeleteRunner.enqueueTask(listeners.acquire(ref -> { + try (ref) { + logger.debug("[{}] Found stale index [{}]. Cleaning it up", metadata.name(), indexId); + final var deleteResult = indexEntry.getValue().delete(OperationPurpose.SNAPSHOT); + blobsDeleted.addAndGet(deleteResult.blobsDeleted()); + bytesDeleted.addAndGet(deleteResult.bytesDeleted()); + logger.debug("[{}] Cleaned up stale index [{}]", metadata.name(), indexId); + } catch (IOException e) { + logger.warn(() -> format(""" + [%s] index %s is no longer part of any snapshot in the repository, \ + but failed to clean up its index folder""", metadata.name(), indexId), e); + } + })); + } } - } - // If we did the cleanup of stale indices purely using a throttled executor then there would be no backpressure to prevent us from - // falling arbitrarily far behind. But nor do we want to dedicate all the SNAPSHOT threads to stale index cleanups because that - // would slow down other snapshot operations in situations that do not need backpressure. - // - // The solution is to dedicate one SNAPSHOT thread to doing the cleanups eagerly, alongside the throttled executor which spreads - // the rest of the work across the other threads if they are free. If the eager cleanup loop doesn't finish before the next one - // starts then we dedicate another SNAPSHOT thread to the deletions, and so on, until eventually either we catch up or the SNAPSHOT - // pool is fully occupied with blob deletions, which pushes back on other snapshot operations. + // If we did the cleanup of stale indices purely using a throttled executor then there would be no backpressure to prevent us + // from falling arbitrarily far behind. But nor do we want to dedicate all the SNAPSHOT threads to stale index cleanups because + // that would slow down other snapshot operations in situations that do not need backpressure. + // + // The solution is to dedicate one SNAPSHOT thread to doing the cleanups eagerly, alongside the throttled executor which spreads + // the rest of the work across the other threads if they are free. If the eager cleanup loop doesn't finish before the next one + // starts then we dedicate another SNAPSHOT thread to the deletions, and so on, until eventually either we catch up or the + // SNAPSHOT pool is fully occupied with blob deletions, which pushes back on other snapshot operations. - staleBlobDeleteRunner.runSyncTasksEagerly(threadPool.executor(ThreadPool.Names.SNAPSHOT)); - } + staleBlobDeleteRunner.runSyncTasksEagerly(snapshotExecutor); + } - /** - * Runs cleanup actions on the repository. Increments the repository state id by one before executing any modifications on the - * repository. - * TODO: Add shard level cleanups - * TODO: Add unreferenced index metadata cleanup - *
    - *
  • Deleting stale indices
    • - *
  • Deleting unreferenced root level blobs
    • - *
- * @param repositoryStateId Current repository state id - * @param repositoryMetaVersion version of the updated repository metadata to write - * @param listener Listener to complete when done - */ - public void cleanup(long repositoryStateId, IndexVersion repositoryMetaVersion, ActionListener listener) { - try { - if (isReadOnly()) { - throw new RepositoryException(metadata.name(), "cannot run cleanup on readonly repository"); - } - Map rootBlobs = blobContainer().listBlobs(OperationPurpose.SNAPSHOT); - final RepositoryData repositoryData = safeRepositoryData(repositoryStateId, rootBlobs); - final Map foundIndices = blobStore().blobContainer(indicesPath()).children(OperationPurpose.SNAPSHOT); - final Set survivingIndexIds = repositoryData.getIndices() - .values() + // Finds all blobs directly under the repository root path that are not referenced by the current RepositoryData + private static List staleRootBlobs(RepositoryData newRepositoryData, Set originalRootBlobNames) { + final Set allSnapshotIds = newRepositoryData.getSnapshotIds() .stream() - .map(IndexId::getId) + .map(SnapshotId::getUUID) .collect(Collectors.toSet()); - final List staleRootBlobs = staleRootBlobs(repositoryData, rootBlobs.keySet()); - if (survivingIndexIds.equals(foundIndices.keySet()) && staleRootBlobs.isEmpty()) { - // Nothing to clean up we return - listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); - } else { - // write new index-N blob to ensure concurrent operations will fail - writeIndexGen( - repositoryData, - repositoryStateId, - repositoryMetaVersion, - Function.identity(), - listener.delegateFailureAndWrap( - (l, v) -> cleanupStaleBlobs( - Collections.emptyList(), - foundIndices, - rootBlobs, - repositoryData, - l.map(RepositoryCleanupResult::new) - ) - ) - ); - } - } catch (Exception e) { - listener.onFailure(e); - } - } - - // Finds all blobs directly under the repository root path that are not referenced by the current RepositoryData - private static List staleRootBlobs(RepositoryData repositoryData, Set rootBlobNames) { - final Set allSnapshotIds = repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toSet()); - return rootBlobNames.stream().filter(blob -> { - if (FsBlobContainer.isTempBlobName(blob)) { - return true; - } - if (blob.endsWith(".dat")) { - final String foundUUID; - if (blob.startsWith(SNAPSHOT_PREFIX)) { - foundUUID = blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); - assert SNAPSHOT_FORMAT.blobName(foundUUID).equals(blob); - } else if (blob.startsWith(METADATA_PREFIX)) { - foundUUID = blob.substring(METADATA_PREFIX.length(), blob.length() - ".dat".length()); - assert GLOBAL_METADATA_FORMAT.blobName(foundUUID).equals(blob); - } else { - return false; + return originalRootBlobNames.stream().filter(blob -> { + if (FsBlobContainer.isTempBlobName(blob)) { + return true; } - return allSnapshotIds.contains(foundUUID) == false; - } else if (blob.startsWith(INDEX_FILE_PREFIX)) { - // TODO: Include the current generation here once we remove keeping index-(N-1) around from #writeIndexGen - try { - return repositoryData.getGenId() > Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); - } catch (NumberFormatException nfe) { - // odd case of an extra file with the index- prefix that we can't identify - return false; + if (blob.endsWith(".dat")) { + final String foundUUID; + if (blob.startsWith(SNAPSHOT_PREFIX)) { + foundUUID = blob.substring(SNAPSHOT_PREFIX.length(), blob.length() - ".dat".length()); + assert SNAPSHOT_FORMAT.blobName(foundUUID).equals(blob); + } else if (blob.startsWith(METADATA_PREFIX)) { + foundUUID = blob.substring(METADATA_PREFIX.length(), blob.length() - ".dat".length()); + assert GLOBAL_METADATA_FORMAT.blobName(foundUUID).equals(blob); + } else { + return false; + } + return allSnapshotIds.contains(foundUUID) == false; + } else if (blob.startsWith(INDEX_FILE_PREFIX)) { + // TODO: Include the current generation here once we remove keeping index-(N-1) around from #writeIndexGen + try { + return newRepositoryData.getGenId() > Long.parseLong(blob.substring(INDEX_FILE_PREFIX.length())); + } catch (NumberFormatException nfe) { + // odd case of an extra file with the index- prefix that we can't identify + return false; + } } - } - return false; - }).toList(); - } - - private void logStaleRootLevelBlobs(long previousGeneration, Collection deletedSnapshots, List blobsToDelete) { - if (logger.isInfoEnabled()) { - // If we're running root level cleanup as part of a snapshot delete we should not log the snapshot- and global metadata - // blobs associated with the just deleted snapshots as they are expected to exist and not stale. Otherwise every snapshot - // delete would also log a confusing INFO message about "stale blobs". - final Set blobNamesToIgnore = deletedSnapshots.stream() - .flatMap( - snapshotId -> Stream.of( - GLOBAL_METADATA_FORMAT.blobName(snapshotId.getUUID()), - SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()), - INDEX_FILE_PREFIX + previousGeneration + return false; + }).toList(); + } + + private void logStaleRootLevelBlobs( + long newestStaleRepositoryDataGeneration, + Collection snapshotIds, + List blobsToDelete + ) { + if (logger.isInfoEnabled()) { + // If we're running root level cleanup as part of a snapshot delete we should not log the snapshot- and global metadata + // blobs associated with the just deleted snapshots as they are expected to exist and not stale. Otherwise every snapshot + // delete would also log a confusing INFO message about "stale blobs". + final Set blobNamesToIgnore = snapshotIds.stream() + .flatMap( + snapshotId -> Stream.of( + GLOBAL_METADATA_FORMAT.blobName(snapshotId.getUUID()), + SNAPSHOT_FORMAT.blobName(snapshotId.getUUID()), + INDEX_FILE_PREFIX + newestStaleRepositoryDataGeneration + ) ) - ) - .collect(Collectors.toSet()); - final List blobsToLog = blobsToDelete.stream().filter(b -> blobNamesToIgnore.contains(b) == false).toList(); - if (blobsToLog.isEmpty() == false) { - logger.info("[{}] Found stale root level blobs {}. Cleaning them up", metadata.name(), blobsToLog); + .collect(Collectors.toSet()); + final List blobsToLog = blobsToDelete.stream().filter(b -> blobNamesToIgnore.contains(b) == false).toList(); + if (blobsToLog.isEmpty() == false) { + logger.info("[{}] Found stale root level blobs {}. Cleaning them up", metadata.name(), blobsToLog); + } } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index b22d4269c7891..6712d1c40b4ee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -69,7 +69,6 @@ public void testSynthesizeIdSimple() throws Exception { prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100580") public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); @@ -203,6 +202,7 @@ private void prepareIndexReader( IndexWriterConfig config = LuceneTestCase.newIndexWriterConfig(random(), new MockAnalyzer(random())); if (noMergePolicy) { config.setMergePolicy(NoMergePolicy.INSTANCE); + config.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 433ebc467483d..f683cb60c87c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -240,10 +238,4 @@ public void testSyntheticSourceInTimeSeries() throws IOException { assertTrue(mapper.sourceMapper().isSynthetic()); assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString()); } - - public void testSyntheticSourceInTimeSeriesBwc() throws IOException { - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); - assertTrue(sourceMapper.isSynthetic()); - assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 2899dab6ff303..6d562f88a0100 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -20,7 +20,9 @@ import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.codec.PerFieldMapperCodec; @@ -231,6 +233,26 @@ public void testDims() { } } + public void testMergeDims() throws IOException { + XContentBuilder mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "dense_vector"); + b.endObject(); + }); + MapperService mapperService = createMapperService(mapping); + + mapping = mapping(b -> { + b.startObject("field"); + b.field("type", "dense_vector").field("dims", 4).field("similarity", "cosine").field("index", true); + b.endObject(); + }); + merge(mapperService, mapping); + assertEquals( + XContentHelper.convertToMap(BytesReference.bytes(mapping), false, mapping.contentType()).v2(), + XContentHelper.convertToMap(mapperService.documentMapper().mappingSource().uncompressed(), false, mapping.contentType()).v2() + ); + } + public void testDefaults() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3))); @@ -391,6 +413,40 @@ public void testCosineWithZeroByteVector() throws Exception { ); } + public void testMaxInnerProductWithValidNorm() throws Exception { + DocumentMapper mapper = createDocumentMapper( + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", 3) + .field("index", true) + .field("similarity", VectorSimilarity.MAX_INNER_PRODUCT) + ) + ); + float[] vector = { -12.1f, 2.7f, -4 }; + // Shouldn't throw + mapper.parse(source(b -> b.array("field", vector))); + } + + public void testWithExtremeFloatVector() throws Exception { + for (VectorSimilarity vs : List.of(VectorSimilarity.COSINE, VectorSimilarity.DOT_PRODUCT, VectorSimilarity.COSINE)) { + DocumentMapper mapper = createDocumentMapper( + fieldMapping(b -> b.field("type", "dense_vector").field("dims", 3).field("index", true).field("similarity", vs)) + ); + float[] vector = { 0.07247924f, -4.310546E-11f, -1.7255947E30f }; + DocumentParsingException e = expectThrows( + DocumentParsingException.class, + () -> mapper.parse(source(b -> b.array("field", vector))) + ); + assertNotNull(e.getCause()); + assertThat( + e.getCause().getMessage(), + containsString( + "NaN or Infinite magnitude detected, this usually means the vector values are too extreme to fit within a float." + ) + ); + } + } + public void testInvalidParameters() { MapperParsingException e = expectThrows( MapperParsingException.class, diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 9df1dc24c2793..6d671a258c26a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 17f2303eb84c8..ab9d80b801863 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1509,7 +1509,7 @@ public MatchingDirectoryReader(DirectoryReader in, Query query) throws IOExcepti @Override public LeafReader wrap(LeafReader leaf) { try { - final IndexSearcher searcher = newSearcher(leaf, false, true, false); + final IndexSearcher searcher = new IndexSearcher(leaf); searcher.setQueryCache(null); final Weight weight = searcher.createWeight(query, ScoreMode.COMPLETE_NO_SCORES, 1.0f); final Scorer scorer = weight.scorer(leaf.getContext()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 540ef4cf1027b..9ccfbd2e25ca6 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -114,6 +114,7 @@ import org.elasticsearch.xcontent.XContentParser.Token; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -2036,7 +2037,17 @@ protected static boolean isTurkishLocale() { || Locale.getDefault().getLanguage().equals(new Locale("az").getLanguage()); } - public static void fail(Throwable t, String msg, Object... args) { + public static T fail(Throwable t, String msg, Object... args) { throw new AssertionError(org.elasticsearch.common.Strings.format(msg, args), t); } + + public static T fail(Throwable t) { + return fail(t, "unexpected"); + } + + @SuppressWarnings("unchecked") + public static T asInstanceOf(Class clazz, Object o) { + assertThat(o, Matchers.instanceOf(clazz)); + return (T) o; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index f6046cd41f25c..dcd7e106b2e81 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -193,6 +193,11 @@ static RoleDescriptor kibanaSystem(String name) { .build(), // Fleet telemetry queries Agent Logs indices in kibana task runner RoleDescriptor.IndicesPrivileges.builder().indices("logs-elastic_agent*").privileges("read").build(), + // Fleet publishes Agent metrics in kibana task runner + RoleDescriptor.IndicesPrivileges.builder() + .indices("metrics-fleet_server*") + .privileges("auto_configure", "read", "write", "delete") + .build(), // Legacy "Alerts as data" used in Security Solution. // Kibana user creates these indices; reads / writes to them. RoleDescriptor.IndicesPrivileges.builder().indices(ReservedRolesStore.ALERTS_LEGACY_INDEX).privileges("all").build(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 543360fc24d89..09ff29f768dce 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -258,7 +258,6 @@ protected JobUpdate doParseInstance(XContentParser parser) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98626") public void testMergeWithJob() { List detectorUpdates = new ArrayList<>(); List detectionRules1 = Collections.singletonList( diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index 166f41fa063ca..5bd20ce51a57d 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -35,6 +35,7 @@ import java.util.function.Consumer; import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices; +import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -77,8 +78,19 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { ) ) .build(); - int indexedDocs = DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); // DSL runs every second and it has to tail forcemerge the index (2 seconds) and mark it as read-only (2s) before it starts diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index cf5e79982d836..c38ed182abc64 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -32,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.getBackingIndices; +import static org.elasticsearch.xpack.downsample.DataStreamLifecycleDriver.putTSDBIndexTemplate; import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends ESIntegTestCase { @@ -68,7 +69,15 @@ public void testDownsampling() throws Exception { ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -85,6 +94,9 @@ public void testDownsampling() throws Exception { witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); @@ -127,7 +139,15 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception ) ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -144,7 +164,9 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); - + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { @@ -182,7 +204,15 @@ public void testUpdateDownsampleRound() throws Exception { ) .build(); - DataStreamLifecycleDriver.setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, DOC_COUNT); + DataStreamLifecycleDriver.setupTSDBDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); List backingIndices = getBackingIndices(client(), dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); @@ -199,7 +229,9 @@ public void testUpdateDownsampleRound() throws Exception { witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); } }); - + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java index be71c546a9d4c..d704f3bf93c54 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDriver.java @@ -37,6 +37,8 @@ import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -62,10 +64,17 @@ public class DataStreamLifecycleDriver { public static final String FIELD_DIMENSION_2 = "dimension_long"; public static final String FIELD_METRIC_COUNTER = "counter"; - public static int setupDataStreamAndIngestDocs(Client client, String dataStreamName, DataStreamLifecycle lifecycle, int docCount) - throws IOException { - putTSDBIndexTemplate(client, dataStreamName + "*", lifecycle); - return indexDocuments(client, dataStreamName, docCount); + public static int setupTSDBDataStreamAndIngestDocs( + Client client, + String dataStreamName, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle, + int docCount, + String firstDocTimestamp + ) throws IOException { + putTSDBIndexTemplate(client, dataStreamName + "*", startTime, endTime, lifecycle); + return indexDocuments(client, dataStreamName, docCount, firstDocTimestamp); } public static List getBackingIndices(Client client, String dataStreamName) { @@ -76,10 +85,24 @@ public static List getBackingIndices(Client client, String dataStreamNam return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); } - private static void putTSDBIndexTemplate(Client client, String pattern, DataStreamLifecycle lifecycle) throws IOException { + public static void putTSDBIndexTemplate( + Client client, + String pattern, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle + ) throws IOException { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); + if (Strings.hasText(startTime)) { + settings.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime); + } + + if (Strings.hasText(endTime)) { + settings.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime); + } + XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); mapping.startObject(FIELD_TIMESTAMP).field("type", "date").endObject(); @@ -129,9 +152,10 @@ private static void putComposableIndexTemplate( client.execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } - private static int indexDocuments(Client client, String dataStreamName, int docCount) { + private static int indexDocuments(Client client, String dataStreamName, int docCount, String firstDocTimestamp) { final Supplier sourceSupplier = () -> { - final String ts = randomDateForInterval(new DateHistogramInterval("1s"), System.currentTimeMillis()); + long startTime = LocalDateTime.parse(firstDocTimestamp).atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + final String ts = randomDateForInterval(new DateHistogramInterval("1s"), startTime); double counterValue = DATE_FORMATTER.parseMillis(ts); final List dimensionValues = new ArrayList<>(5); for (int j = 0; j < randomIntBetween(1, 5); j++) { diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java index 84b55a5fa8009..cf234e31f1f7c 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleClusterDisruptionIT.java @@ -209,6 +209,7 @@ public boolean validateClusterForming() { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100653") public void testDownsampleIndexWithRollingRestart() throws Exception { try (InternalTestCluster cluster = internalCluster()) { final List masterNodes = cluster.startMasterOnlyNodes(1); diff --git a/x-pack/plugin/eql/qa/correctness/build.gradle b/x-pack/plugin/eql/qa/correctness/build.gradle index 4a72f66c238e3..0008c30f260d6 100644 --- a/x-pack/plugin/eql/qa/correctness/build.gradle +++ b/x-pack/plugin/eql/qa/correctness/build.gradle @@ -14,7 +14,7 @@ dependencies { } File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file') - .orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime()) + .orElse(providers.systemProperty('eql.test.credentials.file')) .map { s -> new File(s)} .getOrNull() diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index adf1282c21fb0..9a66bf00fa71f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -93,8 +93,7 @@ public BooleanBlock expand() { public static long ramBytesEstimated(boolean[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index f46615307f767..9e6631b6807c6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -96,8 +96,7 @@ public BytesRefBlock expand() { public static long ramBytesEstimated(BytesRefArray values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index b0d77dd71271e..f9e1fe0c6e199 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -93,8 +93,7 @@ public DoubleBlock expand() { public static long ramBytesEstimated(double[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 97791a03c6044..95344bd8367c0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -93,8 +93,7 @@ public IntBlock expand() { public static long ramBytesEstimated(int[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index dddc5296e471e..a45abb1ed9248 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -93,8 +93,7 @@ public LongBlock expand() { public static long ramBytesEstimated(long[] values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index 8abf0678593ec..44819359e8e44 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -203,6 +203,10 @@ public boolean equals(Object obj) { return shards.equals(other.shards) && segments.equals(other.segments) && docs.equals(other.docs); } + private static long ramBytesOrZero(int[] array) { + return array == null ? 0 : RamUsageEstimator.shallowSizeOf(array); + } + public static long ramBytesEstimated( IntVector shards, IntVector segments, @@ -211,7 +215,7 @@ public static long ramBytesEstimated( int[] shardSegmentDocMapBackwards ) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(shards) + RamUsageEstimator.sizeOf(segments) + RamUsageEstimator.sizeOf(docs) - + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapForwards) + RamUsageEstimator.shallowSizeOf(shardSegmentDocMapBackwards); + + ramBytesOrZero(shardSegmentDocMapForwards) + ramBytesOrZero(shardSegmentDocMapBackwards); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 1f9fb93bc65c6..6a8185b43ecab 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -114,8 +114,7 @@ $endif$ public static long ramBytesEstimated($if(BytesRef)$BytesRefArray$else$$type$[]$endif$ values, int[] firstValueIndexes, BitSet nullsMask) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values) + BlockRamUsageEstimator.sizeOf(firstValueIndexes) - + BlockRamUsageEstimator.sizeOfBitSet(nullsMask) + RamUsageEstimator.shallowSizeOfInstance(MvOrdering.class); - // TODO mvordering is shared + + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java index e5ce5436990b7..29a539b1e068e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValueSources.java @@ -69,6 +69,18 @@ public static List sources( sources.add(new ValueSourceInfo(new NullValueSourceType(), new NullValueSource(), elementType, ctx.getIndexReader())); continue; // the field does not exist in this context } + if (asUnsupportedSource) { + sources.add( + new ValueSourceInfo( + new UnsupportedValueSourceType(fieldType.typeName()), + new UnsupportedValueSource(null), + elementType, + ctx.getIndexReader() + ) + ); + HeaderWarning.addWarning("Field [{}] cannot be retrieved, it is unsupported or not indexed; returning null", fieldName); + continue; + } if (fieldType.hasDocValues() == false) { // MatchOnlyTextFieldMapper class lives in the mapper-extras module. We use string equality @@ -99,19 +111,7 @@ public static List sources( var fieldContext = new FieldContext(fieldName, fieldData, fieldType); var vsType = fieldData.getValuesSourceType(); var vs = vsType.getField(fieldContext, null); - - if (asUnsupportedSource) { - sources.add( - new ValueSourceInfo( - new UnsupportedValueSourceType(fieldType.typeName()), - new UnsupportedValueSource(vs), - elementType, - ctx.getIndexReader() - ) - ); - } else { - sources.add(new ValueSourceInfo(vsType, vs, elementType, ctx.getIndexReader())); - } + sources.add(new ValueSourceInfo(vsType, vs, elementType, ctx.getIndexReader())); } return sources; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java index f6156507dffa2..c322520d8853b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MvExpandOperator.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasables; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -31,11 +32,12 @@ * 2 | 2 | "foo" * */ -public class MvExpandOperator extends AbstractPageMappingOperator { - public record Factory(int channel) implements OperatorFactory { +public class MvExpandOperator implements Operator { + + public record Factory(int channel, int blockSize) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new MvExpandOperator(channel); + return new MvExpandOperator(channel, blockSize); } @Override @@ -46,49 +48,158 @@ public String describe() { private final int channel; + private final int pageSize; + private int noops; - public MvExpandOperator(int channel) { + private Page prev; + private boolean prevCompleted = false; + private boolean finished = false; + + private Block expandingBlock; + private Block expandedBlock; + + private int nextPositionToProcess = 0; + private int nextMvToProcess = 0; + private int nextItemOnExpanded = 0; + + /** + * Count of pages that have been processed by this operator. + */ + private int pagesIn; + private int pagesOut; + + public MvExpandOperator(int channel, int pageSize) { this.channel = channel; + this.pageSize = pageSize; + assert pageSize > 0; } @Override - protected Page process(Page page) { - Block expandingBlock = page.getBlock(channel); - Block expandedBlock = expandingBlock.expand(); + public final Page getOutput() { + if (prev == null) { + return null; + } + pagesOut++; + if (prev.getPositionCount() == 0 || expandingBlock.mayHaveMultivaluedFields() == false) { + noops++; + Page result = prev; + prev = null; + return result; + } + + try { + return process(); + } finally { + if (prevCompleted && prev != null) { + prev.releaseBlocks(); + prev = null; + } + } + } + + protected Page process() { if (expandedBlock == expandingBlock) { noops++; - return page; + prevCompleted = true; + return prev; } - if (page.getBlockCount() == 1) { + if (prev.getBlockCount() == 1) { assert channel == 0; + prevCompleted = true; return new Page(expandedBlock); } - int[] duplicateFilter = buildDuplicateExpandingFilter(expandingBlock, expandedBlock.getPositionCount()); + int[] duplicateFilter = nextDuplicateExpandingFilter(); - Block[] result = new Block[page.getBlockCount()]; + Block[] result = new Block[prev.getBlockCount()]; + int[] expandedMask = new int[duplicateFilter.length]; + for (int i = 0; i < expandedMask.length; i++) { + expandedMask[i] = i + nextItemOnExpanded; + } + nextItemOnExpanded += expandedMask.length; for (int b = 0; b < result.length; b++) { - result[b] = b == channel ? expandedBlock : page.getBlock(b).filter(duplicateFilter); + result[b] = b == channel ? expandedBlock.filter(expandedMask) : prev.getBlock(b).filter(duplicateFilter); + } + if (nextItemOnExpanded == expandedBlock.getPositionCount()) { + nextItemOnExpanded = 0; } return new Page(result); } - private int[] buildDuplicateExpandingFilter(Block expandingBlock, int newPositions) { - int[] duplicateFilter = new int[newPositions]; + private int[] nextDuplicateExpandingFilter() { + int[] duplicateFilter = new int[Math.min(pageSize, expandedBlock.getPositionCount() - nextPositionToProcess)]; int n = 0; - for (int p = 0; p < expandingBlock.getPositionCount(); p++) { - int count = expandingBlock.getValueCount(p); + while (true) { + int count = expandingBlock.getValueCount(nextPositionToProcess); int positions = count == 0 ? 1 : count; - Arrays.fill(duplicateFilter, n, n + positions, p); - n += positions; + int toAdd = Math.min(pageSize - n, positions - nextMvToProcess); + Arrays.fill(duplicateFilter, n, n + toAdd, nextPositionToProcess); + n += toAdd; + + if (n == pageSize) { + if (nextMvToProcess + toAdd == positions) { + // finished expanding this position, let's move on to next position (that will be expanded with next call) + nextMvToProcess = 0; + nextPositionToProcess++; + if (nextPositionToProcess == expandingBlock.getPositionCount()) { + nextPositionToProcess = 0; + prevCompleted = true; + } + } else { + // there are still items to expand in current position, but the duplicate filter is full, so we'll deal with them at + // next call + nextMvToProcess = nextMvToProcess + toAdd; + } + return duplicateFilter; + } + + nextMvToProcess = 0; + nextPositionToProcess++; + if (nextPositionToProcess == expandingBlock.getPositionCount()) { + nextPositionToProcess = 0; + nextMvToProcess = 0; + prevCompleted = true; + return n < pageSize ? Arrays.copyOfRange(duplicateFilter, 0, n) : duplicateFilter; + } } - return duplicateFilter; } @Override - protected AbstractPageMappingOperator.Status status(int pagesProcessed) { - return new Status(pagesProcessed, noops); + public final boolean needsInput() { + return prev == null && finished == false; + } + + @Override + public final void addInput(Page page) { + assert prev == null : "has pending input page"; + prev = page; + this.expandingBlock = prev.getBlock(channel); + this.expandedBlock = expandingBlock.expand(); + pagesIn++; + prevCompleted = false; + } + + @Override + public final void finish() { + finished = true; + } + + @Override + public final boolean isFinished() { + return finished && prev == null; + } + + @Override + public final Status status() { + return new Status(pagesIn, pagesOut, noops); + } + + @Override + public void close() { + if (prev != null) { + Releasables.closeExpectNoException(() -> prev.releaseBlocks()); + } } @Override @@ -96,35 +207,42 @@ public String toString() { return "MvExpandOperator[channel=" + channel + "]"; } - public static final class Status extends AbstractPageMappingOperator.Status { + public static final class Status implements Operator.Status { + + private final int pagesIn; + private final int pagesOut; + private final int noops; + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Operator.Status.class, "mv_expand", Status::new ); - private final int noops; - - Status(int pagesProcessed, int noops) { - super(pagesProcessed); + Status(int pagesIn, int pagesOut, int noops) { + this.pagesIn = pagesIn; + this.pagesOut = pagesOut; this.noops = noops; } Status(StreamInput in) throws IOException { - super(in); + pagesIn = in.readVInt(); + pagesOut = in.readVInt(); noops = in.readVInt(); } @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + out.writeVInt(pagesIn); + out.writeVInt(pagesOut); out.writeVInt(noops); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field("pages_processed", pagesProcessed()); + builder.field("pages_in", pagesIn); + builder.field("pages_out", pagesOut); builder.field("noops", noops); return builder.endObject(); } @@ -147,12 +265,20 @@ public boolean equals(Object o) { return false; } Status status = (Status) o; - return noops == status.noops && pagesProcessed() == status.pagesProcessed(); + return noops == status.noops && pagesIn == status.pagesIn && pagesOut == status.pagesOut; + } + + public int pagesIn() { + return pagesIn; + } + + public int pagesOut() { + return pagesOut; } @Override public int hashCode() { - return Objects.hash(noops, pagesProcessed()); + return Objects.hash(noops, pagesIn, pagesOut); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 4dab7faa2a074..7c930118903cf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -105,12 +105,6 @@ public OrdinalsGroupingOperator( DriverContext driverContext ) { Objects.requireNonNull(aggregatorFactories); - boolean bytesValues = sources.get(0).source() instanceof ValuesSource.Bytes; - for (int i = 1; i < sources.size(); i++) { - if (sources.get(i).source() instanceof ValuesSource.Bytes != bytesValues) { - throw new IllegalStateException("ValuesSources are mismatched"); - } - } this.sources = sources; this.docChannel = docChannel; this.groupingField = groupingField; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java index 930ced04636f8..df6c09ea1ff97 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeBuffer.java @@ -41,13 +41,12 @@ final class ExchangeBuffer { } void addPage(Page page) { + queue.add(page); + if (queueSize.incrementAndGet() == 1) { + notifyNotEmpty(); + } if (noMoreInputs) { - page.releaseBlocks(); - } else { - queue.add(page); - if (queueSize.incrementAndGet() == 1) { - notifyNotEmpty(); - } + discardPages(); } } @@ -115,13 +114,17 @@ SubscribableListener waitForReading() { } } + private void discardPages() { + Page p; + while ((p = pollPage()) != null) { + p.releaseBlocks(); + } + } + void finish(boolean drainingPages) { noMoreInputs = true; if (drainingPages) { - Page p; - while ((p = pollPage()) != null) { - p.releaseBlocks(); - } + discardPages(); } notifyNotEmpty(); if (drainingPages || queueSize.get() == 0) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java index 8634d87e2932f..6ccde6b76ce13 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultSortableTopNEncoder.java @@ -23,7 +23,7 @@ public BytesRef decodeBytesRef(BytesRef bytes, BytesRef scratch) { @Override public String toString() { - return "DefaultUnsortable"; + return "DefaultSortable"; } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java index 2d8f2666ff2f2..f1fb7cb7736c5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java @@ -41,6 +41,11 @@ public interface TopNEncoder { */ VersionTopNEncoder VERSION = new VersionTopNEncoder(); + /** + * Placeholder encoder for unsupported data types. + */ + UnsupportedTypesTopNEncoder UNSUPPORTED = new UnsupportedTypesTopNEncoder(); + void encodeLong(long value, BreakingBytesRefBuilder bytesRefBuilder); long decodeLong(BytesRef bytes); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java new file mode 100644 index 0000000000000..d80d70970409e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/UnsupportedTypesTopNEncoder.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.topn; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; + +/** + * TopNEncoder for data types that are unsupported. This is just a placeholder class, reaching the encode/decode methods here is a bug. + * + * While this class is needed to build the TopNOperator value and key extractors infrastructure, encoding/decoding is needed + * when actually sorting on a field (which shouldn't be possible for unsupported data types) using key extractors, or when encoding/decoding + * unsupported data types fields values (which should always be "null" by convention) using value extractors. + */ +class UnsupportedTypesTopNEncoder extends SortableTopNEncoder { + @Override + public int encodeBytesRef(BytesRef value, BreakingBytesRefBuilder bytesRefBuilder) { + throw new UnsupportedOperationException("Encountered a bug; trying to encode an unsupported data type value for TopN"); + } + + @Override + public BytesRef decodeBytesRef(BytesRef bytes, BytesRef scratch) { + throw new UnsupportedOperationException("Encountered a bug; trying to decode an unsupported data type value for TopN"); + } + + @Override + public String toString() { + return "UnsupportedTypesTopNEncoder"; + } + + @Override + public TopNEncoder toSortable() { + return this; + } + + @Override + public TopNEncoder toUnsortable() { + return this; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java index bb1cd019273ed..d62fd75abbcdd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockAccountingTests.java @@ -45,7 +45,7 @@ public void testBooleanVector() { Vector emptyPlusOne = new BooleanArrayVector(new boolean[] { randomBoolean() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1))); - boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; + boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new BooleanArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + randomData.length))); @@ -61,7 +61,7 @@ public void testIntVector() { Vector emptyPlusOne = new IntArrayVector(new int[] { randomInt() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Integer.BYTES))); - int[] randomData = new int[randomIntBetween(1, 1024)]; + int[] randomData = new int[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new IntArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + (long) Integer.BYTES * randomData.length))); @@ -77,7 +77,7 @@ public void testLongVector() { Vector emptyPlusOne = new LongArrayVector(new long[] { randomLong() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Long.BYTES)); - long[] randomData = new long[randomIntBetween(1, 1024)]; + long[] randomData = new long[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new LongArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length)); @@ -93,7 +93,7 @@ public void testDoubleVector() { Vector emptyPlusOne = new DoubleArrayVector(new double[] { randomDouble() }, 1); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + Double.BYTES)); - double[] randomData = new double[randomIntBetween(1, 1024)]; + double[] randomData = new double[randomIntBetween(2, 1024)]; Vector emptyPlusSome = new DoubleArrayVector(randomData, randomData.length); assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length)); @@ -130,13 +130,11 @@ public void testBooleanBlock() { Block emptyPlusOne = new BooleanArrayBlock(new boolean[] { randomBoolean() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + 1) + alignObjectSize(Integer.BYTES))); - boolean[] randomData = new boolean[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + boolean[] randomData = new boolean[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new BooleanArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is(alignObjectSize(empty.ramBytesUsed() + randomData.length) + alignObjectSize(valueIndices.length * Integer.BYTES)) - ); + long expected = empty.ramBytesUsed() + ramBytesForBooleanArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -148,7 +146,6 @@ public void testBooleanBlockWithNullFirstValues() { assertThat(empty.ramBytesUsed(), lessThanOrEqualTo(expectedEmptyUsed)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100586") public void testIntBlock() { Block empty = new IntArrayBlock(new int[] {}, 0, new int[] {}, null, Block.MvOrdering.UNORDERED); long expectedEmptyUsed = RamUsageTester.ramUsed(empty, RAM_USAGE_ACCUMULATOR); @@ -157,10 +154,11 @@ public void testIntBlock() { Block emptyPlusOne = new IntArrayBlock(new int[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize(Integer.BYTES) + alignObjectSize(Integer.BYTES))); - int[] randomData = new int[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + int[] randomData = new int[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new IntArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat(emptyPlusSome.ramBytesUsed(), is(empty.ramBytesUsed() + alignObjectSize((long) Integer.BYTES * randomData.length) * 2)); + long expected = empty.ramBytesUsed() + ramBytesForIntArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -180,17 +178,11 @@ public void testLongBlock() { Block emptyPlusOne = new LongArrayBlock(new long[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Long.BYTES) + alignObjectSize(Integer.BYTES))); - long[] randomData = new long[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + long[] randomData = new long[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new LongArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is( - alignObjectSize(empty.ramBytesUsed() + (long) Long.BYTES * randomData.length) + alignObjectSize( - (long) valueIndices.length * Integer.BYTES - ) - ) - ); + long expected = empty.ramBytesUsed() + ramBytesForLongArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -210,17 +202,11 @@ public void testDoubleBlock() { Block emptyPlusOne = new DoubleArrayBlock(new double[] { randomInt() }, 1, new int[] { 0 }, null, Block.MvOrdering.UNORDERED); assertThat(emptyPlusOne.ramBytesUsed(), is(alignObjectSize(empty.ramBytesUsed() + Double.BYTES) + alignObjectSize(Integer.BYTES))); - double[] randomData = new double[randomIntBetween(1, 1024)]; - int[] valueIndices = IntStream.range(0, randomData.length).toArray(); + double[] randomData = new double[randomIntBetween(2, 1024)]; + int[] valueIndices = IntStream.range(0, randomData.length + 1).toArray(); Block emptyPlusSome = new DoubleArrayBlock(randomData, randomData.length, valueIndices, null, Block.MvOrdering.UNORDERED); - assertThat( - emptyPlusSome.ramBytesUsed(), - is( - alignObjectSize(empty.ramBytesUsed() + (long) Double.BYTES * randomData.length) + alignObjectSize( - valueIndices.length * Integer.BYTES - ) - ) - ); + long expected = empty.ramBytesUsed() + ramBytesForDoubleArray(randomData) + ramBytesForIntArray(valueIndices); + assertThat(emptyPlusSome.ramBytesUsed(), is(expected)); Block filterBlock = emptyPlusSome.filter(1); assertThat(filterBlock.ramBytesUsed(), lessThan(emptyPlusOne.ramBytesUsed())); @@ -252,11 +238,29 @@ public long accumulateObject(Object o, long shallowSize, Map fiel } else { queue.add(entry.getValue()); } + } else if (o instanceof AbstractArrayBlock && entry.getValue() instanceof Block.MvOrdering) { + // skip; MvOrdering is an enum, so instances are shared } else { queue.add(entry.getValue()); } } return shallowSize; } - }; + } + + static long ramBytesForBooleanArray(boolean[] arr) { + return alignObjectSize((long) Byte.BYTES * arr.length); + } + + static long ramBytesForIntArray(int[] arr) { + return alignObjectSize((long) Integer.BYTES * arr.length); + } + + static long ramBytesForLongArray(long[] arr) { + return alignObjectSize((long) Long.BYTES * arr.length); + } + + static long ramBytesForDoubleArray(double[] arr) { + return alignObjectSize((long) Long.BYTES * arr.length); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java index 350425840a598..e2eff15fcb769 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/DocVectorTests.java @@ -150,6 +150,17 @@ public void testCannotDoubleRelease() { assertThat(e.getMessage(), containsString("can't build page out of released blocks")); } + public void testRamBytesUsedWithout() { + DocVector docs = new DocVector( + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + IntBlock.newConstantBlockWith(0, 1).asVector(), + false + ); + assertThat(docs.singleSegmentNonDecreasing(), is(false)); + docs.ramBytesUsed(); // ensure non-singleSegmentNonDecreasing handles nulls in ramByteUsed + } + IntVector intRange(int startInclusive, int endExclusive) { return IntVector.range(startInclusive, endExclusive, BlockFactory.getNonBreakingInstance()); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java index 28721be14f548..f43159b7ce9bd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FilteredBlockTests.java @@ -316,6 +316,75 @@ public void testFilterToStringMultiValue() { } } + /** Tests filtering on the last position of a block with multi-values. */ + public void testFilterOnLastPositionWithMultiValues() { + { + var builder = blockFactory.newBooleanBlockBuilder(0); + builder.beginPositionEntry().appendBoolean(true).appendBoolean(false).endPositionEntry(); + builder.beginPositionEntry().appendBoolean(false).appendBoolean(true).endPositionEntry(); + BooleanBlock block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getBoolean(filter.getFirstValueIndex(0)), is(false)); + assertThat(filter.getBoolean(filter.getFirstValueIndex(0) + 1), is(true)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newIntBlockBuilder(6); + builder.beginPositionEntry().appendInt(0).appendInt(10).endPositionEntry(); + builder.beginPositionEntry().appendInt(20).appendInt(50).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getInt(filter.getFirstValueIndex(0)), is(20)); + assertThat(filter.getInt(filter.getFirstValueIndex(0) + 1), is(50)); + assertThat(filter.getValueCount(0), is(2)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newLongBlockBuilder(6); + builder.beginPositionEntry().appendLong(0).appendLong(10).endPositionEntry(); + builder.beginPositionEntry().appendLong(20).appendLong(50).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getLong(filter.getFirstValueIndex(0)), is(20L)); + assertThat(filter.getLong(filter.getFirstValueIndex(0) + 1), is(50L)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newDoubleBlockBuilder(6); + builder.beginPositionEntry().appendDouble(0).appendDouble(10).endPositionEntry(); + builder.beginPositionEntry().appendDouble(0.002).appendDouble(10e8).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getDouble(filter.getFirstValueIndex(0)), is(0.002)); + assertThat(filter.getDouble(filter.getFirstValueIndex(0) + 1), is(10e8)); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + { + var builder = blockFactory.newBytesRefBlockBuilder(6); + builder.beginPositionEntry().appendBytesRef(new BytesRef("cat")).appendBytesRef(new BytesRef("dog")).endPositionEntry(); + builder.beginPositionEntry().appendBytesRef(new BytesRef("pig")).appendBytesRef(new BytesRef("chicken")).endPositionEntry(); + var block = builder.build(); + var filter = block.filter(1); + assertThat(filter.getPositionCount(), is(1)); + assertThat(filter.getValueCount(0), is(2)); + assertThat(filter.getBytesRef(filter.getFirstValueIndex(0), new BytesRef()), equalTo(new BytesRef("pig"))); + assertThat(filter.getBytesRef(filter.getFirstValueIndex(0) + 1, new BytesRef()), equalTo(new BytesRef("chicken"))); + Releasables.close(builder, block); + releaseAndAssertBreaker(filter); + } + } + static int randomPosition(int positionCount) { return positionCount == 1 ? 0 : randomIntBetween(0, positionCount - 1); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java index fe281bbf16131..9527388a0d3cf 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorStatusTests.java @@ -16,12 +16,12 @@ public class MvExpandOperatorStatusTests extends AbstractWireSerializingTestCase { public static MvExpandOperator.Status simple() { - return new MvExpandOperator.Status(10, 9); + return new MvExpandOperator.Status(10, 15, 9); } public static String simpleToJson() { return """ - {"pages_processed":10,"noops":9}"""; + {"pages_in":10,"pages_out":15,"noops":9}"""; } public void testToXContent() { @@ -35,20 +35,28 @@ protected Writeable.Reader instanceReader() { @Override public MvExpandOperator.Status createTestInstance() { - return new MvExpandOperator.Status(randomNonNegativeInt(), randomNonNegativeInt()); + return new MvExpandOperator.Status(randomNonNegativeInt(), randomNonNegativeInt(), randomNonNegativeInt()); } @Override protected MvExpandOperator.Status mutateInstance(MvExpandOperator.Status instance) { - switch (between(0, 1)) { + switch (between(0, 2)) { case 0: return new MvExpandOperator.Status( - randomValueOtherThan(instance.pagesProcessed(), ESTestCase::randomNonNegativeInt), + randomValueOtherThan(instance.pagesIn(), ESTestCase::randomNonNegativeInt), + instance.pagesOut(), instance.noops() ); case 1: return new MvExpandOperator.Status( - instance.pagesProcessed(), + instance.pagesIn(), + randomValueOtherThan(instance.pagesOut(), ESTestCase::randomNonNegativeInt), + instance.noops() + ); + case 2: + return new MvExpandOperator.Status( + instance.pagesIn(), + instance.pagesOut(), randomValueOtherThan(instance.noops(), ESTestCase::randomNonNegativeInt) ); default: diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java index 69c965fc91323..f99685609ff78 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MvExpandOperatorTests.java @@ -9,17 +9,19 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; +import java.util.Iterator; import java.util.List; import static org.elasticsearch.compute.data.BasicBlockTests.randomBlock; import static org.elasticsearch.compute.data.BasicBlockTests.valuesAtPositions; +import static org.elasticsearch.compute.data.BlockTestUtils.deepCopyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -47,7 +49,7 @@ protected Page createPage(int positionOffset, int length) { @Override protected Operator.OperatorFactory simple(BigArrays bigArrays) { - return new MvExpandOperator.Factory(0); + return new MvExpandOperator.Factory(0, randomIntBetween(1, 1000)); } @Override @@ -60,47 +62,143 @@ protected String expectedToStringOfSimple() { return expectedDescriptionOfSimple(); } - @Override - protected void assertSimpleOutput(List input, List results) { - assertThat(results, hasSize(results.size())); - for (int i = 0; i < results.size(); i++) { - IntBlock origExpanded = input.get(i).getBlock(0); - IntBlock resultExpanded = results.get(i).getBlock(0); - int np = 0; - for (int op = 0; op < origExpanded.getPositionCount(); op++) { - if (origExpanded.isNull(op)) { - assertThat(resultExpanded.isNull(np), equalTo(true)); - assertThat(resultExpanded.getValueCount(np++), equalTo(0)); - continue; - } - List oValues = BasicBlockTests.valuesAtPositions(origExpanded, op, op + 1).get(0); - for (Object ov : oValues) { - assertThat(resultExpanded.isNull(np), equalTo(false)); - assertThat(resultExpanded.getValueCount(np), equalTo(1)); - assertThat(BasicBlockTests.valuesAtPositions(resultExpanded, np, ++np).get(0), equalTo(List.of(ov))); + class BlockListIterator implements Iterator { + private final Iterator pagesIterator; + private final int channel; + private Block currentBlock; + private int nextPosition; + + BlockListIterator(List pages, int channel) { + this.pagesIterator = pages.iterator(); + this.channel = channel; + this.currentBlock = pagesIterator.next().getBlock(channel); + this.nextPosition = 0; + } + + @Override + public boolean hasNext() { + if (currentBlock == null) { + return false; + } + + return currentBlock.getValueCount(nextPosition) == 0 + || nextPosition < currentBlock.getPositionCount() + || pagesIterator.hasNext(); + } + + @Override + public Object next() { + if (currentBlock != null && currentBlock.getValueCount(nextPosition) == 0) { + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); } + return null; } + List items = valuesAtPositions(currentBlock, nextPosition, nextPosition + 1).get(0); + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); + } + return items.size() == 1 ? items.get(0) : items; + } - IntBlock origDuplicated = input.get(i).getBlock(1); - IntBlock resultDuplicated = results.get(i).getBlock(1); - np = 0; - for (int op = 0; op < origDuplicated.getPositionCount(); op++) { - int copies = origExpanded.isNull(op) ? 1 : origExpanded.getValueCount(op); - for (int c = 0; c < copies; c++) { - if (origDuplicated.isNull(op)) { - assertThat(resultDuplicated.isNull(np), equalTo(true)); - assertThat(resultDuplicated.getValueCount(np++), equalTo(0)); - continue; - } - assertThat(resultDuplicated.isNull(np), equalTo(false)); - assertThat(resultDuplicated.getValueCount(np), equalTo(origDuplicated.getValueCount(op))); - assertThat( - BasicBlockTests.valuesAtPositions(resultDuplicated, np, ++np).get(0), - equalTo(BasicBlockTests.valuesAtPositions(origDuplicated, op, op + 1).get(0)) - ); + private void loadNextBlock() { + if (pagesIterator.hasNext() == false) { + currentBlock = null; + return; + } + this.currentBlock = pagesIterator.next().getBlock(channel); + nextPosition = 0; + } + } + + class BlockListIteratorExpander implements Iterator { + private final Iterator pagesIterator; + private final int channel; + private Block currentBlock; + private int nextPosition; + private int nextInPosition; + + BlockListIteratorExpander(List pages, int channel) { + this.pagesIterator = pages.iterator(); + this.channel = channel; + this.currentBlock = pagesIterator.next().getBlock(channel); + this.nextPosition = 0; + this.nextInPosition = 0; + } + + @Override + public boolean hasNext() { + if (currentBlock == null) { + return false; + } + + return currentBlock.getValueCount(nextPosition) == 0 + || nextInPosition < currentBlock.getValueCount(nextPosition) + || nextPosition < currentBlock.getPositionCount() + || pagesIterator.hasNext(); + } + + @Override + public Object next() { + if (currentBlock != null && currentBlock.getValueCount(nextPosition) == 0) { + nextPosition++; + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); } + return null; + } + List items = valuesAtPositions(currentBlock, nextPosition, nextPosition + 1).get(0); + Object result = items == null ? null : items.get(nextInPosition++); + if (nextInPosition == currentBlock.getValueCount(nextPosition)) { + nextPosition++; + nextInPosition = 0; + } + if (currentBlock.getPositionCount() == nextPosition) { + loadNextBlock(); + } + return result; + } + + private void loadNextBlock() { + if (pagesIterator.hasNext() == false) { + currentBlock = null; + return; + } + this.currentBlock = pagesIterator.next().getBlock(channel); + nextPosition = 0; + nextInPosition = 0; + } + } + + @Override + protected void assertSimpleOutput(List input, List results) { + assertThat(results, hasSize(results.size())); + + var inputIter = new BlockListIteratorExpander(input, 0); + var resultIter = new BlockListIteratorExpander(results, 0); + + while (inputIter.hasNext()) { + assertThat(resultIter.hasNext(), equalTo(true)); + assertThat(resultIter.next(), equalTo(inputIter.next())); + } + assertThat(resultIter.hasNext(), equalTo(false)); + + var originalMvIter = new BlockListIterator(input, 0); + var inputIter2 = new BlockListIterator(input, 1); + var resultIter2 = new BlockListIterator(results, 1); + + while (originalMvIter.hasNext()) { + Object originalMv = originalMvIter.next(); + int originalMvSize = originalMv instanceof List l ? l.size() : 1; + assertThat(resultIter2.hasNext(), equalTo(true)); + Object inputValue = inputIter2.next(); + for (int j = 0; j < originalMvSize; j++) { + assertThat(resultIter2.next(), equalTo(inputValue)); } } + assertThat(resultIter2.hasNext(), equalTo(false)); } @Override @@ -110,7 +208,7 @@ protected ByteSizeValue smallEnoughToCircuitBreak() { } public void testNoopStatus() { - MvExpandOperator op = new MvExpandOperator(0); + MvExpandOperator op = new MvExpandOperator(0, randomIntBetween(1, 1000)); List result = drive( op, List.of(new Page(IntVector.newVectorBuilder(2).appendInt(1).appendInt(2).build().asBlock())).iterator(), @@ -118,26 +216,45 @@ public void testNoopStatus() { ); assertThat(result, hasSize(1)); assertThat(valuesAtPositions(result.get(0).getBlock(0), 0, 2), equalTo(List.of(List.of(1), List.of(2)))); - MvExpandOperator.Status status = (MvExpandOperator.Status) op.status(); - assertThat(status.pagesProcessed(), equalTo(1)); + MvExpandOperator.Status status = op.status(); + assertThat(status.pagesIn(), equalTo(1)); + assertThat(status.pagesOut(), equalTo(1)); assertThat(status.noops(), equalTo(1)); } public void testExpandStatus() { - MvExpandOperator op = new MvExpandOperator(0); + MvExpandOperator op = new MvExpandOperator(0, randomIntBetween(1, 1)); var builder = IntBlock.newBlockBuilder(2).beginPositionEntry().appendInt(1).appendInt(2).endPositionEntry(); List result = drive(op, List.of(new Page(builder.build())).iterator(), driverContext()); assertThat(result, hasSize(1)); assertThat(valuesAtPositions(result.get(0).getBlock(0), 0, 2), equalTo(List.of(List.of(1), List.of(2)))); - MvExpandOperator.Status status = (MvExpandOperator.Status) op.status(); - assertThat(status.pagesProcessed(), equalTo(1)); + MvExpandOperator.Status status = op.status(); + assertThat(status.pagesIn(), equalTo(1)); + assertThat(status.pagesOut(), equalTo(1)); assertThat(status.noops(), equalTo(0)); } - // TODO: remove this once possible - // https://github.com/elastic/elasticsearch/issues/99826 - @Override - protected boolean canLeak() { - return true; + public void testExpandWithBytesRefs() { + DriverContext context = driverContext(); + List input = CannedSourceOperator.collectPages(new AbstractBlockSourceOperator(context.blockFactory(), 8 * 1024) { + private int idx; + + @Override + protected int remaining() { + return 10000 - idx; + } + + @Override + protected Page createPage(int positionOffset, int length) { + idx += length; + return new Page( + randomBlock(context.blockFactory(), ElementType.BYTES_REF, length, true, 1, 10, 0, 0).block(), + randomBlock(context.blockFactory(), ElementType.INT, length, false, 1, 10, 0, 0).block() + ); + } + }); + List origInput = deepCopyOf(input, BlockFactory.getNonBreakingInstance()); + List results = drive(new MvExpandOperator(0, randomIntBetween(1, 1000)), input.iterator(), context); + assertSimpleOutput(origInput, results); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java index 63f601669636c..5d881f03bd07f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/OperatorTestCase.java @@ -212,7 +212,7 @@ protected final void assertSimple(DriverContext context, int size) { unreleasedInputs++; } } - if ((canLeak() == false) && unreleasedInputs > 0) { + if (unreleasedInputs > 0) { throw new AssertionError("[" + unreleasedInputs + "] unreleased input blocks"); } } @@ -308,12 +308,6 @@ protected void start(Driver driver, ActionListener driverListener) { } } - // TODO: Remove this once all operators do not leak anymore - // https://github.com/elastic/elasticsearch/issues/99826 - protected boolean canLeak() { - return false; - } - public static void assertDriverContext(DriverContext driverContext) { assertTrue(driverContext.isFinished()); assertThat(driverContext.getSnapshot().releasables(), empty()); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java new file mode 100644 index 0000000000000..4c975c6c07834 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/exchange/ExchangeBufferTests.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.exchange; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.MockBlockFactory; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.test.ESTestCase; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.Matchers.equalTo; + +public class ExchangeBufferTests extends ESTestCase { + + public void testDrainPages() throws Exception { + ExchangeBuffer buffer = new ExchangeBuffer(randomIntBetween(10, 1000)); + var blockFactory = blockFactory(); + CountDownLatch latch = new CountDownLatch(1); + Thread[] producers = new Thread[between(1, 4)]; + AtomicBoolean stopped = new AtomicBoolean(); + AtomicInteger addedPages = new AtomicInteger(); + for (int t = 0; t < producers.length; t++) { + producers[t] = new Thread(() -> { + try { + latch.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + while (stopped.get() == false && addedPages.incrementAndGet() < 10_000) { + buffer.addPage(randomPage(blockFactory)); + } + }); + producers[t].start(); + } + latch.countDown(); + try { + int minPage = between(10, 100); + int receivedPage = 0; + while (receivedPage < minPage) { + Page p = buffer.pollPage(); + if (p != null) { + p.releaseBlocks(); + ++receivedPage; + } + } + } finally { + buffer.finish(true); + stopped.set(true); + } + for (Thread t : producers) { + t.join(); + } + assertThat(buffer.size(), equalTo(0)); + blockFactory.ensureAllBlocksAreReleased(); + } + + private static MockBlockFactory blockFactory() { + BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); + CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); + return new MockBlockFactory(breaker, bigArrays); + } + + private static Page randomPage(BlockFactory blockFactory) { + Block block = BasicBlockTests.randomBlock( + blockFactory, + randomFrom(ElementType.LONG, ElementType.BYTES_REF, ElementType.BOOLEAN), + randomIntBetween(1, 100), + randomBoolean(), + 0, + between(1, 2), + 0, + between(1, 2) + ).block(); + return new Page(block); + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml index 14ae1ff98d8ad..895a1718b2cbc 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml @@ -106,6 +106,8 @@ setup: --- load everything: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test' @@ -156,6 +158,8 @@ filter on counter: --- from doc with aggregate_metric_double: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test2' @@ -183,6 +187,8 @@ stats on aggregate_metric_double: --- from index pattern unsupported counter: - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'FROM test*' diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml index 44af9559598ab..ad0c7b516fde1 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml @@ -263,3 +263,96 @@ unsupported: - match: { columns.0.name: shape } - match: { columns.0.type: unsupported } - length: { values: 0 } + +--- +unsupported with sort: + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + esql.query: + body: + query: 'from test | sort some_doc.bar' + + - match: { columns.0.name: aggregate_metric_double } + - match: { columns.0.type: unsupported } + - match: { columns.1.name: binary } + - match: { columns.1.type: unsupported } + - match: { columns.2.name: completion } + - match: { columns.2.type: unsupported } + - match: { columns.3.name: date_nanos } + - match: { columns.3.type: unsupported } + - match: { columns.4.name: date_range } + - match: { columns.4.type: unsupported } + - match: { columns.5.name: dense_vector } + - match: { columns.5.type: unsupported } + - match: { columns.6.name: double_range } + - match: { columns.6.type: unsupported } + - match: { columns.7.name: float_range } + - match: { columns.7.type: unsupported } + - match: { columns.8.name: geo_point } + - match: { columns.8.type: unsupported } + - match: { columns.9.name: geo_point_alias } + - match: { columns.9.type: unsupported } + - match: { columns.10.name: histogram } + - match: { columns.10.type: unsupported } + - match: { columns.11.name: integer_range } + - match: { columns.11.type: unsupported } + - match: { columns.12.name: ip_range } + - match: { columns.12.type: unsupported } + - match: { columns.13.name: long_range } + - match: { columns.13.type: unsupported } + - match: { columns.14.name: match_only_text } + - match: { columns.14.type: text } + - match: { columns.15.name: name } + - match: { columns.15.type: keyword } + - match: { columns.16.name: rank_feature } + - match: { columns.16.type: unsupported } + - match: { columns.17.name: rank_features } + - match: { columns.17.type: unsupported } + - match: { columns.18.name: search_as_you_type } + - match: { columns.18.type: unsupported } + - match: { columns.19.name: search_as_you_type._2gram } + - match: { columns.19.type: unsupported } + - match: { columns.20.name: search_as_you_type._3gram } + - match: { columns.20.type: unsupported } + - match: { columns.21.name: search_as_you_type._index_prefix } + - match: { columns.21.type: unsupported } + - match: { columns.22.name: shape } + - match: { columns.22.type: unsupported } + - match: { columns.23.name: some_doc.bar } + - match: { columns.23.type: long } + - match: { columns.24.name: some_doc.foo } + - match: { columns.24.type: keyword } + - match: { columns.25.name: text } + - match: { columns.25.type: text } + - match: { columns.26.name: token_count } + - match: { columns.26.type: integer } + + - length: { values: 1 } + - match: { values.0.0: null } + - match: { values.0.1: null } + - match: { values.0.2: null } + - match: { values.0.3: null } + - match: { values.0.4: null } + - match: { values.0.5: null } + - match: { values.0.6: null } + - match: { values.0.7: null } + - match: { values.0.8: null } + - match: { values.0.9: null } + - match: { values.0.10: null } + - match: { values.0.11: null } + - match: { values.0.12: null } + - match: { values.0.13: null } + - match: { values.0.14: "foo bar baz" } + - match: { values.0.15: Alice } + - match: { values.0.16: null } + - match: { values.0.17: null } + - match: { values.0.18: null } + - match: { values.0.19: null } + - match: { values.0.20: null } + - match: { values.0.21: null } + - match: { values.0.22: null } + - match: { values.0.23: 12 } + - match: { values.0.24: xy } + - match: { values.0.25: "foo bar" } + - match: { values.0.26: 3 } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml index 280a32aa10cd3..ff327b2592c88 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml @@ -267,6 +267,8 @@ disjoint_mappings: --- same_name_different_type: + - skip: + features: allowed_warnings_regex - do: indices.create: index: test1 @@ -307,6 +309,8 @@ same_name_different_type: - { "message": 2 } - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" esql.query: body: query: 'from test1,test2 ' diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 7cc11c6fab5b3..ae27e8f56f9f7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -24,3 +24,77 @@ a:integer | b:keyword | j:keyword 3 | b | "a" 3 | b | "b" ; + + +explosion +row +a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +c = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +d = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +e = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +f = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +g = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +x = 10000000000000 +| mv_expand a | mv_expand b | mv_expand c | mv_expand d | mv_expand e | mv_expand f | mv_expand g +| limit 10; + +a:integer | b:integer | c:integer | d:integer | e:integer | f:integer | g:integer | x:long +1 | 1 | 1 | 1 | 1 | 1 | 1 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 2 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 3 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 4 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 5 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 6 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 7 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 8 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 9 | 10000000000000 +1 | 1 | 1 | 1 | 1 | 1 | 10 | 10000000000000 +; + + +explosionStats +row +a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +b = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +c = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +d = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +e = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], +x = 10000000000000 +| mv_expand a | mv_expand b | mv_expand c | mv_expand d | mv_expand e +| stats sum_a = sum(a) by b +| sort b; + +//12555000 = sum(1..30) * 30 * 30 * 30 +sum_a:long | b:integer +12555000 | 1 +12555000 | 2 +12555000 | 3 +12555000 | 4 +12555000 | 5 +12555000 | 6 +12555000 | 7 +12555000 | 8 +12555000 | 9 +12555000 | 10 +12555000 | 11 +12555000 | 12 +12555000 | 13 +12555000 | 14 +12555000 | 15 +12555000 | 16 +12555000 | 17 +12555000 | 18 +12555000 | 19 +12555000 | 20 +12555000 | 21 +12555000 | 22 +12555000 | 23 +12555000 | 24 +12555000 | 25 +12555000 | 26 +12555000 | 27 +12555000 | 28 +12555000 | 29 +12555000 | 30 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index fd4fe13b9c1b1..2712ef8d2f59b 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -35,6 +35,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -574,7 +575,6 @@ public void testStringLength() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testFilterWithNullAndEvalFromIndex() { // append entry, with an absent count, to the index client().prepareBulk().add(new IndexRequest("test").id("no_count").source("data", 12, "data_d", 2d, "color", "red")).get(); @@ -862,7 +862,6 @@ public void testFromStatsLimit() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/99826") public void testFromLimit() { try (EsqlQueryResponse results = run("from test | keep data | limit 2")) { logger.info(results); @@ -1188,6 +1187,39 @@ public void testGroupingMultiValueByOrdinals() { } } + public void testUnsupportedTypesOrdinalGrouping() { + assertAcked( + client().admin().indices().prepareCreate("index-1").setMapping("f1", "type=keyword", "f2", "type=keyword", "v", "type=long") + ); + assertAcked( + client().admin().indices().prepareCreate("index-2").setMapping("f1", "type=object", "f2", "type=keyword", "v", "type=long") + ); + Map groups = new HashMap<>(); + int numDocs = randomIntBetween(10, 20); + for (int i = 0; i < numDocs; i++) { + String k = randomFrom("a", "b", "c"); + long v = randomIntBetween(1, 10); + groups.merge(k, v, Long::sum); + groups.merge(null, v, Long::sum); // null group + client().prepareIndex("index-1").setSource("f1", k, "v", v).get(); + client().prepareIndex("index-2").setSource("f2", k, "v", v).get(); + } + client().admin().indices().prepareRefresh("index-1", "index-2").get(); + for (String field : List.of("f1", "f2")) { + try (var resp = run("from index-1,index-2 | stats sum(v) by " + field)) { + Iterator> values = resp.values(); + Map actual = new HashMap<>(); + while (values.hasNext()) { + Iterator row = values.next(); + Long v = (Long) row.next(); + String k = (String) row.next(); + actual.put(k, v); + } + assertThat(actual, equalTo(groups)); + } + } + } + private void createNestedMappingIndex(String indexName) throws IOException { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 8732321e8d068..818d58e91a91c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -219,7 +219,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) { ) : plan.policyName(); - var matchField = plan.matchField() == null || plan.matchField() instanceof EmptyAttribute + var matchField = policy != null && (plan.matchField() == null || plan.matchField() instanceof EmptyAttribute) ? new UnresolvedAttribute(plan.source(), policy.getMatchField()) : plan.matchField(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 59c6e2782b014..40f81d0247b33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -55,6 +55,7 @@ import static org.elasticsearch.xpack.esql.stats.FeatureMetric.SORT; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.WHERE; +import static org.elasticsearch.xpack.ql.analyzer.VerifierChecks.checkFilterConditionType; import static org.elasticsearch.xpack.ql.common.Failure.fail; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -121,87 +122,128 @@ else if (p.resolved()) { // Concrete verifications plan.forEachDown(p -> { - if (p instanceof Aggregate agg) { - agg.aggregates().forEach(e -> { - var exp = e instanceof Alias ? ((Alias) e).child() : e; - if (exp instanceof AggregateFunction aggFunc) { - Expression field = aggFunc.field(); - - // TODO: allow an expression? - if ((field instanceof FieldAttribute - || field instanceof MetadataAttribute - || field instanceof ReferenceAttribute - || field instanceof Literal) == false) { - failures.add( - fail( - e, - "aggregate function's field must be an attribute or literal; found [" - + field.sourceText() - + "] of type [" - + field.nodeName() - + "]" - ) - ); - } - } else if (agg.groupings().contains(exp) == false) { // TODO: allow an expression? + // if the children are unresolved, so will this node; counting it will only add noise + if (p.childrenResolved() == false) { + return; + } + checkFilterConditionType(p, failures); + checkAggregate(p, failures); + checkRegexExtractOnlyOnStrings(p, failures); + + checkRow(p, failures); + checkEvalFields(p, failures); + + checkOperationsOnUnsignedLong(p, failures); + checkBinaryComparison(p, failures); + }); + + // gather metrics + if (failures.isEmpty()) { + gatherMetrics(plan); + } + + return failures; + } + + private static void checkAggregate(LogicalPlan p, Set failures) { + if (p instanceof Aggregate agg) { + agg.aggregates().forEach(e -> { + var exp = e instanceof Alias ? ((Alias) e).child() : e; + if (exp instanceof AggregateFunction aggFunc) { + Expression field = aggFunc.field(); + + // TODO: allow an expression? + if ((field instanceof FieldAttribute + || field instanceof MetadataAttribute + || field instanceof ReferenceAttribute + || field instanceof Literal) == false) { failures.add( fail( - exp, - "expected an aggregate function or group but got [" - + exp.sourceText() + e, + "aggregate function's field must be an attribute or literal; found [" + + field.sourceText() + "] of type [" - + exp.nodeName() + + field.nodeName() + "]" ) ); } - }); - } else if (p instanceof RegexExtract re) { - Expression expr = re.input(); - DataType type = expr.dataType(); - if (EsqlDataTypes.isString(type) == false) { + } else if (agg.groupings().contains(exp) == false) { // TODO: allow an expression? failures.add( fail( - expr, - "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]", - re.getClass().getSimpleName(), - expr.sourceText(), - type + exp, + "expected an aggregate function or group but got [" + exp.sourceText() + "] of type [" + exp.nodeName() + "]" ) ); } - } else if (p instanceof Row row) { - failures.addAll(validateRow(row)); - } else if (p instanceof Eval eval) { - failures.addAll(validateEval(eval)); + }); + } + } + + private static void checkRegexExtractOnlyOnStrings(LogicalPlan p, Set failures) { + if (p instanceof RegexExtract re) { + Expression expr = re.input(); + DataType type = expr.dataType(); + if (EsqlDataTypes.isString(type) == false) { + failures.add( + fail( + expr, + "{} only supports KEYWORD or TEXT values, found expression [{}] type [{}]", + re.getClass().getSimpleName(), + expr.sourceText(), + type + ) + ); } + } + } - p.forEachExpression(BinaryOperator.class, bo -> { - Failure f = validateUnsignedLongOperator(bo); - if (f != null) { - failures.add(f); + private static void checkRow(LogicalPlan p, Set failures) { + if (p instanceof Row row) { + row.fields().forEach(a -> { + if (EsqlDataTypes.isRepresentable(a.dataType()) == false) { + failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText())); } }); - p.forEachExpression(BinaryComparison.class, bc -> { - Failure f = validateBinaryComparison(bc); - if (f != null) { - failures.add(f); - } - }); - p.forEachExpression(Neg.class, neg -> { - Failure f = validateUnsignedLongNegation(neg); - if (f != null) { - failures.add(f); + } + } + + private static void checkEvalFields(LogicalPlan p, Set failures) { + if (p instanceof Eval eval) { + eval.fields().forEach(field -> { + DataType dataType = field.dataType(); + if (EsqlDataTypes.isRepresentable(dataType) == false) { + failures.add( + fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText()) + ); } }); - }); - - // gather metrics - if (failures.isEmpty()) { - gatherMetrics(plan); } + } - return failures; + private static void checkOperationsOnUnsignedLong(LogicalPlan p, Set failures) { + p.forEachExpression(e -> { + Failure f = null; + + if (e instanceof BinaryOperator bo) { + f = validateUnsignedLongOperator(bo); + } else if (e instanceof Neg neg) { + f = validateUnsignedLongNegation(neg); + } + + if (f != null) { + failures.add(f); + } + }); + } + + private static void checkBinaryComparison(LogicalPlan p, Set failures) { + p.forEachExpression(BinaryComparison.class, bc -> { + Failure f = validateBinaryComparison(bc); + if (f != null) { + failures.add(f); + } + }); } private void gatherMetrics(LogicalPlan plan) { @@ -228,29 +270,6 @@ private void gatherMetrics(LogicalPlan plan) { } } - private static Collection validateRow(Row row) { - List failures = new ArrayList<>(row.fields().size()); - row.fields().forEach(a -> { - if (EsqlDataTypes.isRepresentable(a.dataType()) == false) { - failures.add(fail(a, "cannot use [{}] directly in a row assignment", a.child().sourceText())); - } - }); - return failures; - } - - private static Collection validateEval(Eval eval) { - List failures = new ArrayList<>(eval.fields().size()); - eval.fields().forEach(field -> { - DataType dataType = field.dataType(); - if (EsqlDataTypes.isRepresentable(dataType) == false) { - failures.add( - fail(field, "EVAL does not support type [{}] in expression [{}]", dataType.typeName(), field.child().sourceText()) - ); - } - }); - return failures; - } - /** * Limit QL's comparisons to types we support. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 1c26de4a599f5..bdc1c948f2055 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -380,6 +380,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case "version" -> TopNEncoder.VERSION; case "boolean", "null", "byte", "short", "integer", "long", "double", "float", "half_float", "datetime", "date_period", "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc" -> TopNEncoder.DEFAULT_SORTABLE; + // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point + case "unsupported" -> TopNEncoder.UNSUPPORTED; default -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type()); }; } @@ -582,7 +584,8 @@ private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContex private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(mvExpandExec.child(), context); - return source.with(new MvExpandOperator.Factory(source.layout.get(mvExpandExec.target().id()).channel()), source.layout); + int blockSize = 5000;// TODO estimate row size and use context.pageSize() + return source.with(new MvExpandOperator.Factory(source.layout.get(mvExpandExec.target().id()).channel(), blockSize), source.layout); } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 1ee90256b95dd..6cbc1f93bcdf1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1256,6 +1256,14 @@ public void testNonExistingEnrichPolicy() { assertThat(e.getMessage(), containsString("unresolved enrich policy [foo]")); } + public void testNonExistingEnrichNoMatchField() { + var e = expectThrows(VerificationException.class, () -> analyze(""" + from test + | enrich foo + """)); + assertThat(e.getMessage(), containsString("unresolved enrich policy [foo]")); + } + public void testNonExistingEnrichPolicyWithSimilarName() { var e = expectThrows(VerificationException.class, () -> analyze(""" from test diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 10f134432a0a2..cd1c9d8fbe830 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -292,6 +292,14 @@ public void testPeriodAndDurationInEval() { } } + public void testFilterNonBoolField() { + assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where emp_no")); + } + + public void testFilterDateConstant() { + assertEquals("1:19: Condition expression needs to be boolean, found [DATE_PERIOD]", error("from test | where 1 year")); + } + private String error(String query) { return error(query, defaultAnalyzer); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index cdff3f0b5f2ca..3a6479215f479 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -420,7 +420,7 @@ protected void assertSimpleWithNulls(List data, Block value, int nullBlo assertTrue("argument " + nullBlock + " is null", value.isNull(0)); } - public void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { + public final void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { assumeTrue("nothing to do if a type error", testCase.getExpectedTypeError() == null); assumeTrue("All test data types must be representable in order to build fields", testCase.allTypesAreRepresentable()); int count = 10_000; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 15d37acbccfcb..8db6b1bbd0c93 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.function.Supplier; @@ -54,12 +53,6 @@ public static Iterable parameters() { return parameterSuppliersFromTypedData(builder.suppliers()); } - @Override - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100559") - public void testEvaluateInManyThreads() throws ExecutionException, InterruptedException { - super.testEvaluateInManyThreads(); - } - @Override protected void assertSimpleWithNulls(List data, Block value, int nullBlock) { for (int i = 0; i < data.size(); i++) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 25439d0bfc930..cc84a5c53c81c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -50,7 +50,7 @@ import org.elasticsearch.xpack.inference.action.TransportGetInferenceModelAction; import org.elasticsearch.xpack.inference.action.TransportInferenceAction; import org.elasticsearch.xpack.inference.action.TransportPutInferenceModelAction; -import org.elasticsearch.xpack.inference.external.http.HttpClient; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.HttpSettings; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceModelAction; @@ -62,13 +62,16 @@ import java.util.Collection; import java.util.List; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class InferencePlugin extends Plugin implements ActionPlugin, InferenceServicePlugin, SystemIndexPlugin { public static final String NAME = "inference"; public static final String UTILITY_THREAD_POOL_NAME = "inference_utility"; + public static final String HTTP_CLIENT_SENDER_THREAD_POOL_NAME = "inference_http_client_sender"; private final Settings settings; - private final SetOnce httpClient = new SetOnce<>(); + private final SetOnce httpClientManager = new SetOnce<>(); public InferencePlugin(Settings settings) { this.settings = settings; @@ -119,8 +122,7 @@ public Collection createComponents( AllocationService allocationService, IndicesService indicesService ) { - var httpSettings = new HttpSettings(settings, clusterService); - httpClient.set(HttpClient.create(httpSettings, threadPool)); + httpClientManager.set(HttpClientManager.create(settings, threadPool, clusterService)); ModelRegistry modelRegistry = new ModelRegistry(client); return List.of(modelRegistry); @@ -154,22 +156,35 @@ public Collection getSystemIndexDescriptors(Settings sett } @Override - public List> getExecutorBuilders(Settings unused) { - ScalingExecutorBuilder utility = new ScalingExecutorBuilder( - UTILITY_THREAD_POOL_NAME, - 0, - 1, - TimeValue.timeValueMinutes(10), - false, - "xpack.inference.utility_thread_pool" + public List> getExecutorBuilders(Settings settingsToUse) { + return List.of( + new ScalingExecutorBuilder( + UTILITY_THREAD_POOL_NAME, + 0, + 1, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ), + /* + * This executor is specifically for enqueuing requests to be sent. The underlying + * connection pool used by the http client will block if there are no available connections to lease. + * See here for more info: https://hc.apache.org/httpcomponents-client-4.5.x/current/tutorial/html/connmgmt.html + */ + new ScalingExecutorBuilder( + HTTP_CLIENT_SENDER_THREAD_POOL_NAME, + 0, + 1, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.http_client_sender_thread_pool" + ) ); - - return List.of(utility); } @Override public List> getSettings() { - return HttpSettings.getSettings(); + return Stream.concat(HttpSettings.getSettings().stream(), HttpClientManager.getSettings().stream()).collect(Collectors.toList()); } @Override @@ -194,8 +209,8 @@ public List getInferenceServiceNamedWriteables() { @Override public void close() { - if (httpClient.get() != null) { - IOUtils.closeWhileHandlingException(httpClient.get()); + if (httpClientManager.get() != null) { + IOUtils.closeWhileHandlingException(httpClientManager.get()); } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java index 5e3ceac875921..5622ac51ba187 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpClient.java @@ -13,9 +13,6 @@ import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; -import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; -import org.apache.http.nio.reactor.ConnectingIOReactor; -import org.apache.http.nio.reactor.IOReactorException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -29,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.HTTP_CLIENT_SENDER_THREAD_POOL_NAME; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; public class HttpClient implements Closeable { @@ -41,45 +39,19 @@ enum Status { } private final CloseableHttpAsyncClient client; - private final IdleConnectionEvictor connectionEvictor; private final AtomicReference status = new AtomicReference<>(Status.CREATED); private final ThreadPool threadPool; private final HttpSettings settings; - public static HttpClient create(HttpSettings settings, ThreadPool threadPool) { - PoolingNHttpClientConnectionManager connectionManager = createConnectionManager(); - IdleConnectionEvictor connectionEvictor = new IdleConnectionEvictor( - threadPool, - connectionManager, - settings.getEvictionInterval(), - settings.getEvictionMaxIdle() - ); + public static HttpClient create(HttpSettings settings, ThreadPool threadPool, PoolingNHttpClientConnectionManager connectionManager) { + CloseableHttpAsyncClient client = createAsyncClient(connectionManager); - int maxConnections = settings.getMaxConnections(); - CloseableHttpAsyncClient client = createAsyncClient(connectionManager, maxConnections); - - return new HttpClient(settings, client, connectionEvictor, threadPool); - } - - private static PoolingNHttpClientConnectionManager createConnectionManager() { - ConnectingIOReactor ioReactor; - try { - ioReactor = new DefaultConnectingIOReactor(); - } catch (IOReactorException e) { - var message = "Failed to initialize the inference http client"; - logger.error(message, e); - throw new ElasticsearchException(message, e); - } - - return new PoolingNHttpClientConnectionManager(ioReactor); + return new HttpClient(settings, client, threadPool); } - private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager, int maxConnections) { + private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConnectionManager connectionManager) { HttpAsyncClientBuilder clientBuilder = HttpAsyncClientBuilder.create(); - clientBuilder.setConnectionManager(connectionManager); - clientBuilder.setMaxConnPerRoute(maxConnections); - clientBuilder.setMaxConnTotal(maxConnections); // The apache client will be shared across all connections because it can be expensive to create it // so we don't want to support cookies to avoid accidental authentication for unauthorized users clientBuilder.disableCookieManagement(); @@ -88,24 +60,32 @@ private static CloseableHttpAsyncClient createAsyncClient(PoolingNHttpClientConn } // Default for testing - HttpClient(HttpSettings settings, CloseableHttpAsyncClient asyncClient, IdleConnectionEvictor evictor, ThreadPool threadPool) { + HttpClient(HttpSettings settings, CloseableHttpAsyncClient asyncClient, ThreadPool threadPool) { this.settings = settings; this.threadPool = threadPool; this.client = asyncClient; - this.connectionEvictor = evictor; } public void start() { if (status.compareAndSet(Status.CREATED, Status.STARTED)) { client.start(); - connectionEvictor.start(); } } - public void send(HttpUriRequest request, ActionListener listener) throws IOException { + public void send(HttpUriRequest request, ActionListener listener) { // The caller must call start() first before attempting to send a request assert status.get() == Status.STARTED; + threadPool.executor(HTTP_CLIENT_SENDER_THREAD_POOL_NAME).execute(() -> { + try { + doPrivilegedSend(request, listener); + } catch (IOException e) { + listener.onFailure(new ElasticsearchException(format("Failed to send request [%s]", request.getRequestLine()), e)); + } + }); + } + + private void doPrivilegedSend(HttpUriRequest request, ActionListener listener) throws IOException { SocketAccess.doPrivileged(() -> client.execute(request, new FutureCallback<>() { @Override public void completed(HttpResponse response) { @@ -144,6 +124,5 @@ private void failUsingUtilityThread(Exception exception, ActionListener MAX_CONNECTIONS = Setting.intSetting( + "xpack.inference.http.max_connections", + // TODO pick a reasonable values here + 20, + 1, + 1000, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private static final TimeValue DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME = TimeValue.timeValueSeconds(10); + static final Setting CONNECTION_EVICTION_THREAD_INTERVAL_SETTING = Setting.timeSetting( + "xpack.inference.http.connection_eviction_interval", + DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private static final TimeValue DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME; + static final Setting CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting( + "xpack.inference.http.connection_eviction_max_idle_time", + DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private final ThreadPool threadPool; + private final PoolingNHttpClientConnectionManager connectionManager; + private EvictorSettings evictorSettings; + private IdleConnectionEvictor connectionEvictor; + private final HttpClient httpClient; + + public static HttpClientManager create(Settings settings, ThreadPool threadPool, ClusterService clusterService) { + PoolingNHttpClientConnectionManager connectionManager = createConnectionManager(); + return new HttpClientManager(settings, connectionManager, threadPool, clusterService); + } + + // Default for testing + HttpClientManager( + Settings settings, + PoolingNHttpClientConnectionManager connectionManager, + ThreadPool threadPool, + ClusterService clusterService + ) { + this.threadPool = threadPool; + + this.connectionManager = connectionManager; + setMaxConnections(MAX_CONNECTIONS.get(settings)); + + this.httpClient = HttpClient.create(new HttpSettings(settings, clusterService), threadPool, connectionManager); + + evictorSettings = new EvictorSettings(settings); + connectionEvictor = createConnectionEvictor(); + + this.addSettingsUpdateConsumers(clusterService); + } + + private static PoolingNHttpClientConnectionManager createConnectionManager() { + ConnectingIOReactor ioReactor; + try { + ioReactor = new DefaultConnectingIOReactor(); + } catch (IOReactorException e) { + var message = "Failed to initialize the inference http client manager"; + logger.error(message, e); + throw new ElasticsearchException(message, e); + } + + return new PoolingNHttpClientConnectionManager(ioReactor); + } + + private void addSettingsUpdateConsumers(ClusterService clusterService) { + clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONNECTIONS, this::setMaxConnections); + clusterService.getClusterSettings() + .addSettingsUpdateConsumer(CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, this::setEvictionInterval); + clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, this::setEvictionMaxIdle); + } + + private IdleConnectionEvictor createConnectionEvictor() { + return new IdleConnectionEvictor(threadPool, connectionManager, evictorSettings.evictionInterval, evictorSettings.evictionMaxIdle); + } + + public static List> getSettings() { + return List.of(MAX_CONNECTIONS, CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING); + } + + public void start() { + httpClient.start(); + connectionEvictor.start(); + } + + public HttpClient getHttpClient() { + return httpClient; + } + + @Override + public void close() throws IOException { + httpClient.close(); + connectionEvictor.stop(); + } + + private void setMaxConnections(int maxConnections) { + connectionManager.setMaxTotal(maxConnections); + connectionManager.setDefaultMaxPerRoute(maxConnections); + } + + // default for testing + void setEvictionInterval(TimeValue evictionInterval) { + evictorSettings = new EvictorSettings(evictionInterval, evictorSettings.evictionMaxIdle); + + connectionEvictor.stop(); + connectionEvictor = createConnectionEvictor(); + connectionEvictor.start(); + } + + void setEvictionMaxIdle(TimeValue evictionMaxIdle) { + evictorSettings = new EvictorSettings(evictorSettings.evictionInterval, evictionMaxIdle); + + connectionEvictor.stop(); + connectionEvictor = createConnectionEvictor(); + connectionEvictor.start(); + } + + private static class EvictorSettings { + private final TimeValue evictionInterval; + private final TimeValue evictionMaxIdle; + + EvictorSettings(Settings settings) { + this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings); + this.evictionMaxIdle = CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING.get(settings); + } + + EvictorSettings(TimeValue evictionInterval, TimeValue evictionMaxIdle) { + this.evictionInterval = evictionInterval; + this.evictionMaxIdle = evictionMaxIdle; + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java index 420f7822df06c..07d998dff956e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpSettings.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.TimeValue; import java.util.List; @@ -26,89 +25,24 @@ public class HttpSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); - static final Setting MAX_CONNECTIONS = Setting.intSetting( - "xpack.inference.http.max_connections", - 500, - 1, - // TODO pick a reasonable value here - 1000, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private static final TimeValue DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME = TimeValue.timeValueSeconds(10); - - static final Setting CONNECTION_EVICTION_THREAD_INTERVAL_SETTING = Setting.timeSetting( - "xpack.inference.http.connection_eviction_interval", - DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); - - private static final TimeValue DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = DEFAULT_CONNECTION_EVICTION_THREAD_INTERVAL_TIME; - static final Setting CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING = Setting.timeSetting( - "xpack.inference.http.connection_eviction_max_idle_time", - DEFAULT_CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, - Setting.Property.NodeScope, - Setting.Property.Dynamic - ); private volatile ByteSizeValue maxResponseSize; - private volatile int maxConnections; - private volatile TimeValue evictionInterval; - private volatile TimeValue evictionMaxIdle; public HttpSettings(Settings settings, ClusterService clusterService) { this.maxResponseSize = MAX_HTTP_RESPONSE_SIZE.get(settings); - this.maxConnections = MAX_CONNECTIONS.get(settings); - this.evictionInterval = CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.get(settings); - this.evictionMaxIdle = CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING.get(settings); clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_HTTP_RESPONSE_SIZE, this::setMaxResponseSize); - clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONNECTIONS, this::setMaxConnections); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer(CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, this::setEvictionInterval); - clusterService.getClusterSettings().addSettingsUpdateConsumer(CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING, this::setEvictionMaxIdle); } public ByteSizeValue getMaxResponseSize() { return maxResponseSize; } - public int getMaxConnections() { - return maxConnections; - } - - public TimeValue getEvictionInterval() { - return evictionInterval; - } - - public TimeValue getEvictionMaxIdle() { - return evictionMaxIdle; - } - private void setMaxResponseSize(ByteSizeValue maxResponseSize) { this.maxResponseSize = maxResponseSize; } - private void setMaxConnections(int maxConnections) { - this.maxConnections = maxConnections; - } - - private void setEvictionInterval(TimeValue evictionInterval) { - this.evictionInterval = evictionInterval; - } - - private void setEvictionMaxIdle(TimeValue evictionMaxIdle) { - this.evictionMaxIdle = evictionMaxIdle; - } - public static List> getSettings() { - return List.of( - MAX_HTTP_RESPONSE_SIZE, - MAX_CONNECTIONS, - CONNECTION_EVICTION_THREAD_INTERVAL_SETTING, - CONNECTION_EVICTION_MAX_IDLE_TIME_SETTING - ); + return List.of(MAX_HTTP_RESPONSE_SIZE); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java index 3ea0bc04848e0..f326661adc6f4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/IdleConnectionEvictor.java @@ -16,6 +16,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; @@ -36,7 +37,7 @@ public class IdleConnectionEvictor { private final NHttpClientConnectionManager connectionManager; private final TimeValue sleepTime; private final TimeValue maxIdleTime; - private Scheduler.Cancellable cancellableTask; + private final AtomicReference cancellableTask = new AtomicReference<>(); public IdleConnectionEvictor( ThreadPool threadPool, @@ -51,13 +52,13 @@ public IdleConnectionEvictor( } public synchronized void start() { - if (cancellableTask == null) { + if (cancellableTask.get() == null) { startInternal(); } } private void startInternal() { - cancellableTask = threadPool.scheduleWithFixedDelay(() -> { + Scheduler.Cancellable task = threadPool.scheduleWithFixedDelay(() -> { try { connectionManager.closeExpiredConnections(); if (maxIdleTime != null) { @@ -67,13 +68,17 @@ private void startInternal() { logger.warn("HTTP connection eviction failed", e); } }, sleepTime, threadPool.executor(UTILITY_THREAD_POOL_NAME)); + + cancellableTask.set(task); } public void stop() { - cancellableTask.cancel(); + if (cancellableTask.get() != null) { + cancellableTask.get().cancel(); + } } public boolean isRunning() { - return cancellableTask != null && cancellableTask.isCancelled() == false; + return cancellableTask.get() != null && cancellableTask.get().isCancelled() == false; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java index 7dffbc693ca51..d1f27302f85f1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettings.java @@ -87,10 +87,21 @@ public ElserMlNodeServiceSettings(int numAllocations, int numThreads, String var public ElserMlNodeServiceSettings(StreamInput in) throws IOException { numAllocations = in.readVInt(); numThreads = in.readVInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + if (transportVersionIsCompatibleWithElserModelVersion(in.getTransportVersion())) { modelVariant = in.readString(); } else { - modelVariant = ElserMlNodeService.ELSER_V1_MODEL; + modelVariant = ElserMlNodeService.ELSER_V2_MODEL; + } + } + + static boolean transportVersionIsCompatibleWithElserModelVersion(TransportVersion transportVersion) { + var nextNonPatchVersion = TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME; + + if (transportVersion.onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + return true; + } else { + return transportVersion.onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH) + && transportVersion.before(nextNonPatchVersion); } } @@ -130,7 +141,7 @@ public TransportVersion getMinimalSupportedVersion() { public void writeTo(StreamOutput out) throws IOException { out.writeVInt(numAllocations); out.writeVInt(numThreads); - if (out.getTransportVersion().onOrAfter(TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED)) { + if (transportVersionIsCompatibleWithElserModelVersion(out.getTransportVersion())) { out.writeString(modelVariant); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java new file mode 100644 index 0000000000000..a9bdee95de5fc --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientManagerTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http; + +import org.apache.http.HttpHeaders; +import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.junit.After; +import org.junit.Before; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createHttpPost; +import static org.elasticsearch.xpack.inference.external.http.HttpClientTests.createThreadPool; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class HttpClientManagerTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(getTestName()); + } + + @After + public void shutdown() { + terminate(threadPool); + webServer.close(); + } + + public void testSend_MockServerReceivesRequest() throws Exception { + int responseCode = randomIntBetween(200, 203); + String body = randomAlphaOfLengthBetween(2, 8096); + webServer.enqueue(new MockResponse().setResponseCode(responseCode).setBody(body)); + + String paramKey = randomAlphaOfLength(3); + String paramValue = randomAlphaOfLength(3); + var httpPost = createHttpPost(webServer.getPort(), paramKey, paramValue); + + var manager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty()); + try (var httpClient = manager.getHttpClient()) { + httpClient.start(); + + PlainActionFuture listener = new PlainActionFuture<>(); + httpClient.send(httpPost, listener); + + var result = listener.actionGet(TIMEOUT); + + assertThat(result.response().getStatusLine().getStatusCode(), equalTo(responseCode)); + assertThat(new String(result.body(), StandardCharsets.UTF_8), is(body)); + assertThat(webServer.requests(), hasSize(1)); + assertThat(webServer.requests().get(0).getUri().getPath(), equalTo(httpPost.getURI().getPath())); + assertThat(webServer.requests().get(0).getUri().getQuery(), equalTo(paramKey + "=" + paramValue)); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); + } + } + + public void testStartsANewEvictor_WithNewEvictionInterval() { + var threadPool = mock(ThreadPool.class); + var manager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty()); + + var evictionInterval = TimeValue.timeValueSeconds(1); + manager.setEvictionInterval(evictionInterval); + verify(threadPool).scheduleWithFixedDelay(any(Runnable.class), eq(evictionInterval), any()); + } + + public void testStartsANewEvictor_WithNewEvictionMaxIdle() throws InterruptedException { + var mockConnectionManager = mock(PoolingNHttpClientConnectionManager.class); + + Settings settings = Settings.builder() + .put(HttpClientManager.CONNECTION_EVICTION_THREAD_INTERVAL_SETTING.getKey(), TimeValue.timeValueNanos(1)) + .build(); + var manager = new HttpClientManager(settings, mockConnectionManager, threadPool, mockClusterService(settings)); + + CountDownLatch runLatch = new CountDownLatch(1); + doAnswer(invocation -> { + manager.close(); + runLatch.countDown(); + return Void.TYPE; + }).when(mockConnectionManager).closeIdleConnections(anyLong(), any()); + + var evictionMaxIdle = TimeValue.timeValueSeconds(1); + manager.setEvictionMaxIdle(evictionMaxIdle); + runLatch.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); + + verify(mockConnectionManager, times(1)).closeIdleConnections(eq(evictionMaxIdle.millis()), eq(TimeUnit.MILLISECONDS)); + } + + private static ClusterService mockClusterServiceEmpty() { + return mockClusterService(Settings.EMPTY); + } + + private static ClusterService mockClusterService(Settings settings) { + var clusterService = mock(ClusterService.class); + + var registeredSettings = Stream.concat(HttpClientManager.getSettings().stream(), HttpSettings.getSettings().stream()) + .collect(Collectors.toSet()); + + var cSettings = new ClusterSettings(settings, registeredSettings); + when(clusterService.getClusterSettings()).thenReturn(cSettings); + + return clusterService; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java index 42c8422af3982..b0b0a34aabf97 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/HttpClientTests.java @@ -45,6 +45,7 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.HTTP_CLIENT_SENDER_THREAD_POOL_NAME; import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -64,17 +65,7 @@ public class HttpClientTests extends ESTestCase { @Before public void init() throws Exception { webServer.start(); - threadPool = new TestThreadPool( - getTestName(), - new ScalingExecutorBuilder( - UTILITY_THREAD_POOL_NAME, - 1, - 4, - TimeValue.timeValueMinutes(10), - false, - "xpack.inference.utility_thread_pool" - ) - ); + threadPool = createThreadPool(getTestName()); } @After @@ -92,7 +83,7 @@ public void testSend_MockServerReceivesRequest() throws Exception { String paramValue = randomAlphaOfLength(3); var httpPost = createHttpPost(webServer.getPort(), paramKey, paramValue); - try (var httpClient = HttpClient.create(emptyHttpSettings(), threadPool)) { + try (var httpClient = HttpClient.create(emptyHttpSettings(), threadPool, createConnectionManager())) { httpClient.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -119,10 +110,9 @@ public void testSend_FailedCallsOnFailure() throws Exception { return mock(Future.class); }).when(asyncClient).execute(any(), any()); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -143,10 +133,9 @@ public void testSend_CancelledCallsOnFailure() throws Exception { return mock(Future.class); }).when(asyncClient).execute(any(), any()); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -162,10 +151,9 @@ public void testStart_MultipleCallsOnlyStartTheClientOnce() throws Exception { var asyncClient = mock(CloseableHttpAsyncClient.class); when(asyncClient.execute(any(), any())).thenReturn(mock(Future.class)); - var evictor = createEvictor(threadPool); var httpPost = createHttpPost(webServer.getPort(), "a", "b"); - try (var client = new HttpClient(emptyHttpSettings(), asyncClient, evictor, threadPool)) { + try (var client = new HttpClient(emptyHttpSettings(), asyncClient, threadPool)) { client.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -188,7 +176,7 @@ public void testSend_FailsWhenMaxBytesReadIsExceeded() throws Exception { Settings settings = Settings.builder().put(HttpSettings.MAX_HTTP_RESPONSE_SIZE.getKey(), ByteSizeValue.ONE).build(); var httpSettings = createHttpSettings(settings); - try (var httpClient = HttpClient.create(httpSettings, threadPool)) { + try (var httpClient = HttpClient.create(httpSettings, threadPool, createConnectionManager())) { httpClient.start(); PlainActionFuture listener = new PlainActionFuture<>(); @@ -199,7 +187,7 @@ public void testSend_FailsWhenMaxBytesReadIsExceeded() throws Exception { } } - private static HttpPost createHttpPost(int port, String paramKey, String paramValue) throws URISyntaxException { + public static HttpPost createHttpPost(int port, String paramKey, String paramValue) throws URISyntaxException { URI uri = new URIBuilder().setScheme("http") .setHost("localhost") .setPort(port) @@ -219,16 +207,33 @@ private static HttpPost createHttpPost(int port, String paramKey, String paramVa return httpPost; } - private static IdleConnectionEvictor createEvictor(ThreadPool threadPool) throws IOReactorException { - var manager = createConnectionManager(); - return new IdleConnectionEvictor(threadPool, manager, new TimeValue(10, TimeUnit.SECONDS), new TimeValue(10, TimeUnit.SECONDS)); + public static ThreadPool createThreadPool(String name) { + return new TestThreadPool( + name, + new ScalingExecutorBuilder( + UTILITY_THREAD_POOL_NAME, + 1, + 4, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ), + new ScalingExecutorBuilder( + HTTP_CLIENT_SENDER_THREAD_POOL_NAME, + 1, + 4, + TimeValue.timeValueMinutes(10), + false, + "xpack.inference.utility_thread_pool" + ) + ); } private static PoolingNHttpClientConnectionManager createConnectionManager() throws IOReactorException { return new PoolingNHttpClientConnectionManager(new DefaultConnectingIOReactor()); } - private static HttpSettings emptyHttpSettings() { + public static HttpSettings emptyHttpSettings() { return createHttpSettings(Settings.EMPTY); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java index 35d5c0b8e9603..8b6f3f1a56ba6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeServiceSettingsTests.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.inference.services.elser; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -85,6 +87,52 @@ public void testFromMapMissingOptions() { assertThat(e.getMessage(), containsString("[service_settings] does not contain the required setting [num_allocations]")); } + public void testTransportVersionIsCompatibleWithElserModelVersion() { + assertTrue( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED + ) + ); + assertTrue( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH + ) + ); + + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion(TransportVersions.ML_PACKAGE_LOADER_PLATFORM_ADDED) + ); + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME + ) + ); + assertFalse( + ElserMlNodeServiceSettings.transportVersionIsCompatibleWithElserModelVersion( + TransportVersions.UNIVERSAL_PROFILING_LICENSE_ADDED + ) + ); + } + + public void testBwcWrite() throws IOException { + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED); + assertEquals(settings, copy); + } + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.PLUGIN_DESCRIPTOR_OPTIONAL_CLASSNAME); + assertNotEquals(settings, copy); + assertEquals(".elser_model_2", copy.getModelVariant()); + } + { + var settings = new ElserMlNodeServiceSettings(1, 1, ".elser_model_1"); + var copy = copyInstance(settings, TransportVersions.ELSER_SERVICE_MODEL_VERSION_ADDED_PATCH); + assertEquals(settings, copy); + } + } + public void testFromMapInvalidSettings() { var settingsMap = new HashMap( Map.of(ElserMlNodeServiceSettings.NUM_ALLOCATIONS, 0, ElserMlNodeServiceSettings.NUM_THREADS, -1) diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java index 5405852173a62..4b0783dda84cc 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java @@ -206,7 +206,7 @@ public void testManyDistinctOverFields() throws Exception { int user = 0; while (timestamp < now) { List data = new ArrayList<>(); - for (int i = 0; i < 10000; i++) { + for (int i = 0; i < 20000; i++) { // It's important that the values used here are either always represented in less than 16 UTF-8 bytes or // always represented in more than 22 UTF-8 bytes. Otherwise platform differences in when the small string // optimisation is used will make the results of this test very different for the different platforms. diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 467378f4cd738..ba43cf82d1458 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -596,6 +596,11 @@ private boolean canUseWatcher() { @Override public void onCleanUpIndices(TimeValue retention) { + if (stateInitialized.get() == false) { + // ^ this is once the cluster state is recovered. Don't try to interact with the cluster service until that happens + logger.debug("exporter not yet initialized"); + return; + } ClusterState clusterState = clusterService.state(); if (clusterService.localNode() == null || clusterState == null diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java index 3b0d301099d72..a30975be1055d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java @@ -7,17 +7,25 @@ package org.elasticsearch.xpack.monitoring.exporter.local; +import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.monitoring.cleaner.CleanerService; import org.elasticsearch.xpack.monitoring.exporter.Exporter; import org.elasticsearch.xpack.monitoring.exporter.MonitoringMigrationCoordinator; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; public class LocalExporterTests extends ESTestCase { @@ -37,4 +45,29 @@ public void testLocalExporterRemovesListenersOnClose() { verify(licenseState).removeListener(exporter); } + public void testLocalExporterDoesNotInteractWithClusterServiceUntilStateIsRecovered() { + final ClusterService clusterService = mock(ClusterService.class); + final XPackLicenseState licenseState = mock(XPackLicenseState.class); + final Exporter.Config config = new Exporter.Config("name", "type", Settings.EMPTY, clusterService, licenseState); + final CleanerService cleanerService = mock(CleanerService.class); + final MonitoringMigrationCoordinator migrationCoordinator = new MonitoringMigrationCoordinator(); + try (Client client = new NoOpClient(getTestName())) { + final LocalExporter exporter = new LocalExporter(config, client, migrationCoordinator, cleanerService); + + final TimeValue retention = TimeValue.timeValueDays(randomIntBetween(1, 90)); + exporter.onCleanUpIndices(retention); + + verify(clusterService).addListener(same(exporter)); + verifyNoMoreInteractions(clusterService); + + final ClusterState oldState = ClusterState.EMPTY_STATE; + final ClusterState newState = ClusterStateCreationUtils.stateWithNoShard(); + exporter.clusterChanged(new ClusterChangedEvent(getTestName(), newState, oldState)); + verify(clusterService).localNode(); + + exporter.onCleanUpIndices(retention); + verify(clusterService).state(); + verify(clusterService, times(2)).localNode(); + } + } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 8311d0f613175..cdddd0a5e5fe0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -56,6 +56,10 @@ import org.elasticsearch.xpack.wildcard.Wildcard; import java.io.IOException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -130,18 +134,15 @@ public void testDownsamplingAuthorized() throws Exception { ) .build(); - setupDataStreamAndIngestDocs(client(), dataStreamName, lifecycle, 10_000); - waitAndAssertDownsamplingCompleted(dataStreamName); - } - - @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") - public void testSystemDataStreamConfigurationWithDownsampling() throws Exception { - String dataStreamName = SystemDataStreamWithDownsamplingConfigurationPlugin.SYSTEM_DATA_STREAM_NAME; - indexDocuments(client(), dataStreamName, 10_000); - waitAndAssertDownsamplingCompleted(dataStreamName); - } - - private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Exception { + setupDataStreamAndIngestDocs( + client(), + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + 10_000, + "1990-09-09T18:00:00" + ); List backingIndices = getDataStreamBackingIndices(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0).getName(); String firstRoundDownsamplingIndex = "downsample-5m-" + firstGenerationBackingIndex; @@ -158,6 +159,9 @@ private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Ex } }); + // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with + // testing so DSL doesn't have to wait for the end_time to lapse) + putTSDBIndexTemplate(client(), dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { @@ -188,6 +192,52 @@ private void waitAndAssertDownsamplingCompleted(String dataStreamName) throws Ex }, 30, TimeUnit.SECONDS); } + @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") + public void testSystemDataStreamConfigurationWithDownsampling() throws Exception { + String dataStreamName = SystemDataStreamWithDownsamplingConfigurationPlugin.SYSTEM_DATA_STREAM_NAME; + indexDocuments(client(), dataStreamName, 10_000, Instant.now().toEpochMilli()); + List backingIndices = getDataStreamBackingIndices(dataStreamName); + String firstGenerationBackingIndex = backingIndices.get(0).getName(); + String secondRoundDownsamplingIndex = "downsample-10m-" + firstGenerationBackingIndex; + + Set witnessedDownsamplingIndices = new HashSet<>(); + clusterService().addListener(event -> { + if (event.indicesCreated().contains(secondRoundDownsamplingIndex)) { + witnessedDownsamplingIndices.add(secondRoundDownsamplingIndex); + } + }); + + DataStreamLifecycleService masterDataStreamLifecycleService = internalCluster().getCurrentMasterNodeInstance( + DataStreamLifecycleService.class + ); + try { + // we can't update the index template backing a system data stream, so we run DSL "in the future" + // this means that only one round of downsampling will execute due to an optimisation we have in DSL to execute the last + // matching round + masterDataStreamLifecycleService.setNowSupplier(() -> Instant.now().plus(50, ChronoUnit.DAYS).toEpochMilli()); + client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); + + assertBusy(() -> { + assertNoAuthzErrors(); + assertThat(witnessedDownsamplingIndices.contains(secondRoundDownsamplingIndex), is(true)); + }, 30, TimeUnit.SECONDS); + + assertBusy(() -> { + assertNoAuthzErrors(); + List dsBackingIndices = getDataStreamBackingIndices(dataStreamName); + + assertThat(dsBackingIndices.size(), is(2)); + String writeIndex = dsBackingIndices.get(1).getName(); + assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); + // the last downsampling round must remain in the data stream + assertThat(dsBackingIndices.get(0).getName(), is(secondRoundDownsamplingIndex)); + }, 30, TimeUnit.SECONDS); + } finally { + // restore a real nowSupplier so other tests running against this cluster succeed + masterDataStreamLifecycleService.setNowSupplier(() -> Instant.now().toEpochMilli()); + } + } + private Map collectErrorsFromStoreAsMap() { Iterable lifecycleServices = internalCluster().getInstances(DataStreamLifecycleService.class); Map indicesAndErrors = new HashMap<>(); @@ -221,15 +271,36 @@ private void assertNoAuthzErrors() { } } - private void setupDataStreamAndIngestDocs(Client client, String dataStreamName, DataStreamLifecycle lifecycle, int docCount) - throws IOException { - putTSDBIndexTemplate(client, dataStreamName + "*", lifecycle); - indexDocuments(client, dataStreamName, docCount); + private void setupDataStreamAndIngestDocs( + Client client, + String dataStreamName, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle, + int docCount, + String firstDocTimestamp + ) throws IOException { + putTSDBIndexTemplate(client, dataStreamName + "*", startTime, endTime, lifecycle); + long startTimestamp = LocalDateTime.parse(firstDocTimestamp).atZone(ZoneId.of("UTC")).toInstant().toEpochMilli(); + indexDocuments(client, dataStreamName, docCount, startTimestamp); } - private void putTSDBIndexTemplate(Client client, String pattern, DataStreamLifecycle lifecycle) throws IOException { + private void putTSDBIndexTemplate( + Client client, + String pattern, + @Nullable String startTime, + @Nullable String endTime, + DataStreamLifecycle lifecycle + ) throws IOException { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); + if (Strings.hasText(startTime)) { + settings.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime); + } + + if (Strings.hasText(endTime)) { + settings.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime); + } CompressedXContent mapping = getTSDBMappings(); putComposableIndexTemplate(client, "id1", mapping, List.of(pattern), settings.build(), null, lifecycle); } @@ -275,9 +346,9 @@ private void putComposableIndexTemplate( client.execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } - private void indexDocuments(Client client, String dataStreamName, int docCount) { + private void indexDocuments(Client client, String dataStreamName, int docCount, long startTime) { final Supplier sourceSupplier = () -> { - final String ts = randomDateForInterval(new DateHistogramInterval("1s"), System.currentTimeMillis()); + final String ts = randomDateForInterval(new DateHistogramInterval("1s"), startTime); double counterValue = DATE_FORMATTER.parseMillis(ts); final List dimensionValues = new ArrayList<>(5); for (int j = 0; j < randomIntBetween(1, 5); j++) { @@ -336,27 +407,26 @@ private void bulkIndex(Client client, String dataStreamName, Supplier getSystemDataStreamDescriptors() { - DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder() - .downsampling( - new DataStreamLifecycle.Downsampling( - List.of( - new DataStreamLifecycle.Downsampling.Round( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.Downsampling.Round( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + public static final DataStreamLifecycle LIFECYCLE = DataStreamLifecycle.newBuilder() + .downsampling( + new DataStreamLifecycle.Downsampling( + List.of( + new DataStreamLifecycle.Downsampling.Round( + TimeValue.timeValueMillis(0), + new DownsampleConfig(new DateHistogramInterval("5m")) + ), + new DataStreamLifecycle.Downsampling.Round( + TimeValue.timeValueSeconds(10), + new DownsampleConfig(new DateHistogramInterval("10m")) ) ) ) - .build(); + ) + .build(); + static final String SYSTEM_DATA_STREAM_NAME = ".fleet-actions-results"; + @Override + public Collection getSystemDataStreamDescriptors() { Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of(FIELD_DIMENSION_1)); @@ -368,7 +438,7 @@ public Collection getSystemDataStreamDescriptors() { SystemDataStreamDescriptor.Type.EXTERNAL, new ComposableIndexTemplate( List.of(SYSTEM_DATA_STREAM_NAME), - new Template(settings.build(), getTSDBMappings(), null, lifecycle), + new Template(settings.build(), getTSDBMappings(), null, LIFECYCLE), null, null, null, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 55929a1c1b83e..13fb4246a5b3a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -340,7 +340,7 @@ public GeoShapeWithDocValuesFieldMapper( } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { // TODO: Make common with the index method ShapeFieldMapper if (geometry == null) { return; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 378b78111ab19..f5cc7280aa8bb 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -150,7 +150,7 @@ public PointFieldMapper( } @Override - protected void index(DocumentParserContext context, CartesianPoint point) throws IOException { + protected void index(DocumentParserContext context, CartesianPoint point) { if (fieldType().isIndexed()) { context.doc().add(new XYPointField(fieldType().name(), (float) point.getX(), (float) point.getY())); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 127a4fd1050cd..838fd56cfc11a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -204,7 +204,7 @@ public ShapeFieldMapper( } @Override - protected void index(DocumentParserContext context, Geometry geometry) throws IOException { + protected void index(DocumentParserContext context, Geometry geometry) { // TODO: Make common with the index method GeoShapeWithDocValuesFieldMapper if (geometry == null) { return; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml index abf367043d9c8..0c765e39656c7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/110_field_caps.yml @@ -9,8 +9,6 @@ setup: body: settings: index: - number_of_replicas: 0 - number_of_shards: 2 mode: time_series routing_path: [ metricset, k8s.pod.uid ] time_series: @@ -35,8 +33,6 @@ setup: body: settings: index: - number_of_replicas: 0 - number_of_shards: 2 mode: time_series routing_path: [ metricset, k8s.pod.uid ] time_series: @@ -57,10 +53,6 @@ setup: indices.create: index: test_non_time_series body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml index 05a2c640e68ef..2325a078764fc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/90_tsdb_mappings.yml @@ -7,10 +7,6 @@ aggregate_double_metric with time series mappings: indices.create: index: test_index body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": @@ -51,10 +47,6 @@ aggregate_double_metric with wrong time series mappings: indices.create: index: tsdb_index body: - settings: - index: - number_of_replicas: 0 - number_of_shards: 2 mappings: properties: "@timestamp": @@ -95,7 +87,6 @@ aggregate_double_metric with wrong time series mappings: index: tsdb-fieldcap body: settings: - number_of_replicas: 0 mode: time_series routing_path: [field1] time_series: diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java index 32cdcee280d6e..6ba0f572a2f9f 100644 --- a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java +++ b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TestFeatureResetIT.java @@ -114,7 +114,8 @@ public void testTransformFeatureReset() throws Exception { ); // assert transform indices are gone - assertThat(ESRestTestCase.entityAsMap(adminClient().performRequest(new Request("GET", ".transform-*"))), is(anEmptyMap())); + Map transformIndices = ESRestTestCase.entityAsMap(adminClient().performRequest(new Request("GET", ".transform-*"))); + assertThat("Indices were: " + transformIndices, transformIndices, is(anEmptyMap())); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index c1964448c2662..81a719e24f633 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -426,7 +426,7 @@ public void cleanUpFeature( client.admin() .cluster() .prepareListTasks() - .setActions(TransformField.TASK_NAME) + .setActions(TransformField.TASK_NAME + "*") .setWaitForCompletion(true) .execute(ActionListener.wrap(listTransformTasks -> { listTransformTasks.rethrowFailures("Waiting for transform tasks"); diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 79a2be51197e6..aeb3dad547946 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -73,7 +73,6 @@ public void waitForMlTemplates() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/93521") public void testMlIndicesBecomeHidden() throws Exception { if (isRunningAgainstOldCluster()) { // trigger ML indices creation