Merge branch 'main' into notif_lock_fail

.github/CODEOWNERS

@@ -1 +1 @@
-*   @bowenlan-amzn @getsaurabh02 @lezzago @praveensameneni @xluo-aws @gaobinlong @Hailong-am @amsiglan @sbcd90 @eirsep @AWSHurneyt
+*   @bowenlan-amzn @getsaurabh02 @lezzago @praveensameneni @xluo-aws @gaobinlong @Hailong-am @amsiglan @sbcd90 @eirsep @AWSHurneyt @jowg-amazon

.github/workflows/docker-security-test-workflow.yml

@@ -2,13 +2,13 @@ name: Docker Security Test Workflow
 on:
   pull_request:
     branches:
-      - "*"
+      - "**"
   push:
     branches:
-      - "*"
+      - "**"
 
 jobs:
-  test:
+  docker-test:
     # This job runs on Linux
     runs-on: ubuntu-latest
     steps:
@@ -73,7 +73,7 @@ jobs:
           if [ $security -gt 0 ]
           then
             echo "Security plugin is available"
-            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin
+            ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin --tests '*SecurityBehaviorIT'
           else
             echo "Security plugin is NOT available skipping this run as tests without security have already been run"
           fi
@@ -83,3 +83,15 @@ jobs:
         with:
           name: logs
           path: build/testclusters/integTest-*/logs/*
+      - name: Collect docker logs on failure
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          dest: './logs'
+      - name: Tar logs
+        run: tar cvzf ./logs.tgz ./logs
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs.tgz
+          path: ./logs.tgz

.github/workflows/multi-node-test-workflow.yml

@@ -3,13 +3,24 @@ name: Multi node test workflow
 on:
   pull_request:
     branches:
-      - "*"
+      - "**"
   push:
     branches:
-      - "*"
+      - "**"
 
 jobs:
-  test:
+  multi-node-test:
+    env:
+      TEST_FILTER: ${{ matrix.test_filter }}
+    strategy:
+      fail-fast: false
+      matrix:
+        feature: [ism, non-ism]
+        include:
+          - feature: ism
+            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+          - feature: non-ism
+            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
     # This job runs on Linux
     runs-on: ubuntu-latest
     steps:
@@ -23,7 +34,7 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3
+        run: ./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

.github/workflows/security-test-workflow.yml

@@ -3,13 +3,13 @@ name: Security test workflow
 on:
   pull_request:
     branches:
-      - "*"
+      - "**"
   push:
     branches:
-      - "*"
+      - "**"
 
 jobs:
-  test:
+  security-test:
     # This job runs on Linux
     runs-on: ubuntu-latest
     steps:
@@ -24,7 +24,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Run integration tests
         run: | 
-          ./gradlew integTest -Dsecurity=true -Dhttps=true
+          ./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

.github/workflows/test-and-build-workflow.yml

@@ -2,30 +2,36 @@ name: Test and Build Workflow
 on:
   pull_request:
     branches:
-      - "*"
+      - "**"
   push:
     branches:
-      - "*"
+      - "**"
 
 jobs:
-  build:
+  test-and-build:
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
+      TEST_FILTER: ${{ matrix.test_filter }}
     strategy:
       # This setting says that all jobs should finish, even if one fails
       fail-fast: false
       # This starts three jobs, setting these environment variables uniquely for the different jobs
       matrix:
         java: [11, 17]
         os: [ubuntu-latest, windows-latest, macos-latest]
+        feature: [ism, non-ism]
         include:
           - os: windows-latest
             os_build_args: -x integTest -x jacocoTestReport
             working_directory: X:\
             os_java_options: -Xmx4096M
           - os: macos-latest
             os_build_args: -x integTest -x jacocoTestReport
+          - feature: ism
+            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+          - feature: non-ism
+            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
     runs-on: ${{ matrix.os }}
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
@@ -44,7 +50,7 @@ jobs:
         run: subst 'X:' .
       - name: Build with Gradle
         working-directory: ${{ env.WORKING_DIR }}
-        run: ./gradlew build ${{ env.BUILD_ARGS }}
+        run: ./gradlew build ${{ env.BUILD_ARGS }} ${{ env.TEST_FILTER }}
         env:
           _JAVA_OPTIONS: ${{ matrix.os_java_options }}
       - name: Upload failed logs

DEVELOPER_GUIDE.md

@@ -48,7 +48,7 @@ However, to build the `index management` plugin project, we also use the OpenSea
 
 ### Building from the command line
 
-1. `./gradlew build` builds and tests project.
+1. `./gradlew build` builds and tests project. 
 2. `./gradlew run` launches a single node cluster with the index management (and job-scheduler) plugin installed.
 3. `./gradlew run -PnumNodes=3` launches a multi-node cluster with the index management (and job-scheduler) plugin installed.
 4. `./gradlew integTest` launches a single node cluster with the index management (and job-scheduler) plugin installed and runs all integ tests.

MAINTAINERS.md

@@ -5,7 +5,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 ## Current Maintainers
 
 | Maintainer            | GitHub ID                                             | Affiliation |
-| --------------------- | ----------------------------------------------------- | ----------- |
+|-----------------------|-------------------------------------------------------| ----------- |
 | Ashish Agrawal        | [lezzago](https://github.com/lezzago)                 | Amazon      |
 | Bowen Lan             | [bowenlan-amzn](https://github.com/bowenlan-amzn)     | Amazon      |
 | Praveen Sameneni      | [praveensameneni](https://github.com/praveensameneni) | Amazon      |
@@ -17,3 +17,4 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Subhobrata Dey        | [sbcd90](https://github.com/sbcd90)                   | Amazon      |
 | Surya Sashank Nistala | [eirsep](https://github.com/eirsep)                   | Amazon      |
 | Thomas Hurney         | [AWSHurneyt](https://github.com/AWSHurneyt)           | Amazon      |
+| Joanne Wang           | [jowg-amazon](https://github.com/jowg-amazon)         | Amazon      |

build.gradle

@@ -574,6 +574,21 @@ integTest {
     if (usingRemoteCluster) {
         exclude 'org/opensearch/indexmanagement/controlcenter/notification/filter/NotificationActionListenerIT.class'
     }
+
+    if (project.hasProperty('includeTests')) {
+        String includeTests = project.property('includeTests')
+        println("Include tests ${includeTests}")
+        filter {
+            includeTestsMatching includeTests
+        }
+    }
+    if (project.hasProperty('excludeTests')) {
+        String excludeTests = project.property('excludeTests')
+        println("Exclude tests ${excludeTests}")
+        filter {
+            excludeTestsMatching excludeTests
+        }
+    }
 }
 
 task integTestRemote(type: RestIntegTestTask) {
@@ -613,7 +628,7 @@ task integTestRemote(type: RestIntegTestTask) {
 
 // === Set up BWC tests ===
 
-String bwcVersionShort = "2.10.0"
+String bwcVersionShort = "2.11.0"
 String bwcVersion = bwcVersionShort + ".0"
 String baseName = "indexmanagementBwcCluster"
 

release-notes/opensearch-index-management.release-notes-2.10.0.0.md

@@ -0,0 +1,32 @@
+## Version 2.10.0.0 2023-09-07
+
+Compatible with OpenSearch 2.10.0
+
+### Maintenance
+* Increment version to 2.10.0-SNAPSHOT. ([#852](https://github.com/opensearch-project/index-management/pull/852))
+
+### Features
+* Support copy alias in rollover. ([#892](https://github.com/opensearch-project/index-management/pull/892))
+* make control center index as system index. ([#919](https://github.com/opensearch-project/index-management/pull/919))
+
+### Bug fixes
+* Fix debug log for missing ISM config index. ([#846](https://github.com/opensearch-project/index-management/pull/846))
+* Handle NPE in isRollupIndex. ([#855](https://github.com/opensearch-project/index-management/pull/855))
+* fix for max & min aggregations when no metric property exist. ([#870](https://github.com/opensearch-project/index-management/pull/870))
+* fix intelliJ IDEA gradle sync error ([#916](https://github.com/opensearch-project/index-management/pull/916))
+
+### Infrastructure
+* Add auto github release workflow. ([#691](https://github.com/opensearch-project/index-management/pull/691))
+* Fixed the publish maven workflow to execute after pushes to release branches. ([#837](https://github.com/opensearch-project/index-management/pull/837))
+* Upgrade the backport workflow. ([#862](https://github.com/opensearch-project/index-management/pull/862))
+* Updates demo certs used in integ tests. ([#921](https://github.com/opensearch-project/index-management/pull/921))
+
+### Refactoring
+* [Backport 2.x] Fix after core #8157. ([#886](https://github.com/opensearch-project/index-management/pull/886))
+* Fix breaking change by core refactor. ([#888](https://github.com/opensearch-project/index-management/pull/888))
+* Handle core breaking change. ([#895](https://github.com/opensearch-project/index-management/pull/895))
+* Set preference to _primary when searching control-center index. ([#911](https://github.com/opensearch-project/index-management/pull/911))
+* Add primary first preference to all search requests. ([#912](https://github.com/opensearch-project/index-management/pull/912))
+
+### Documentation
+* Added 2.10 release notes. ([#925](https://github.com/opensearch-project/index-management/pull/925))

spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/ActionProperties.kt

@@ -23,7 +23,8 @@ data class ActionProperties(
     val snapshotName: String? = null,
     val rollupId: String? = null,
     val hasRollupFailed: Boolean? = null,
-    val shrinkActionProperties: ShrinkActionProperties? = null
+    val shrinkActionProperties: ShrinkActionProperties? = null,
+    val transformActionProperties: TransformActionProperties? = null
 ) : Writeable, ToXContentFragment {
 
     override fun writeTo(out: StreamOutput) {
@@ -32,6 +33,7 @@ data class ActionProperties(
         out.writeOptionalString(rollupId)
         out.writeOptionalBoolean(hasRollupFailed)
         out.writeOptionalWriteable(shrinkActionProperties)
+        out.writeOptionalWriteable(transformActionProperties)
     }
 
     override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
@@ -40,6 +42,7 @@ data class ActionProperties(
         if (rollupId != null) builder.field(Properties.ROLLUP_ID.key, rollupId)
         if (hasRollupFailed != null) builder.field(Properties.HAS_ROLLUP_FAILED.key, hasRollupFailed)
         if (shrinkActionProperties != null) builder.addObject(ShrinkActionProperties.SHRINK_ACTION_PROPERTIES, shrinkActionProperties, params)
+        if (transformActionProperties != null) builder.addObject(TransformActionProperties.TRANSFORM_ACTION_PROPERTIES, transformActionProperties, params)
         return builder
     }
 
@@ -52,7 +55,8 @@ data class ActionProperties(
             val rollupId: String? = si.readOptionalString()
             val hasRollupFailed: Boolean? = si.readOptionalBoolean()
             val shrinkActionProperties: ShrinkActionProperties? = si.readOptionalWriteable { ShrinkActionProperties.fromStreamInput(it) }
-            return ActionProperties(maxNumSegments, snapshotName, rollupId, hasRollupFailed, shrinkActionProperties)
+            val transformActionProperties: TransformActionProperties? = si.readOptionalWriteable { TransformActionProperties.fromStreamInput(it) }
+            return ActionProperties(maxNumSegments, snapshotName, rollupId, hasRollupFailed, shrinkActionProperties, transformActionProperties)
         }
 
         fun parse(xcp: XContentParser): ActionProperties {
@@ -61,6 +65,7 @@ data class ActionProperties(
             var rollupId: String? = null
             var hasRollupFailed: Boolean? = null
             var shrinkActionProperties: ShrinkActionProperties? = null
+            var transformActionProperties: TransformActionProperties? = null
 
             ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp)
             while (xcp.nextToken() != Token.END_OBJECT) {
@@ -75,10 +80,13 @@ data class ActionProperties(
                     ShrinkActionProperties.SHRINK_ACTION_PROPERTIES -> {
                         shrinkActionProperties = if (xcp.currentToken() == Token.VALUE_NULL) null else ShrinkActionProperties.parse(xcp)
                     }
+                    TransformActionProperties.TRANSFORM_ACTION_PROPERTIES -> {
+                        transformActionProperties = if (xcp.currentToken() == Token.VALUE_NULL) null else TransformActionProperties.parse(xcp)
+                    }
                 }
             }
 
-            return ActionProperties(maxNumSegments, snapshotName, rollupId, hasRollupFailed, shrinkActionProperties)
+            return ActionProperties(maxNumSegments, snapshotName, rollupId, hasRollupFailed, shrinkActionProperties, transformActionProperties)
         }
     }
 

spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/TransformActionProperties.kt

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.indexmanagement.spi.indexstatemanagement.model
+
+import org.opensearch.core.common.io.stream.StreamInput
+import org.opensearch.core.common.io.stream.StreamOutput
+import org.opensearch.core.common.io.stream.Writeable
+import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken
+import org.opensearch.core.xcontent.ToXContent
+import org.opensearch.core.xcontent.ToXContentFragment
+import org.opensearch.core.xcontent.XContentBuilder
+import org.opensearch.core.xcontent.XContentParser
+
+data class TransformActionProperties(
+    val transformId: String?
+) : Writeable, ToXContentFragment {
+
+    override fun writeTo(out: StreamOutput) {
+        out.writeOptionalString(transformId)
+    }
+
+    override fun toXContent(builder: XContentBuilder, params: ToXContent.Params?): XContentBuilder {
+        if (transformId != null) builder.field(Properties.TRANSFORM_ID.key, transformId)
+        return builder
+    }
+
+    companion object {
+        const val TRANSFORM_ACTION_PROPERTIES = "transform_action_properties"
+
+        fun fromStreamInput(sin: StreamInput): TransformActionProperties {
+            val transformId: String? = sin.readOptionalString()
+            return TransformActionProperties(transformId)
+        }
+
+        fun parse(xcp: XContentParser): TransformActionProperties {
+            var transformId: String? = null
+
+            ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp)
+            while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
+                val fieldName = xcp.currentName()
+                xcp.nextToken()
+
+                when (fieldName) {
+                    Properties.TRANSFORM_ID.key -> transformId = xcp.text()
+                }
+            }
+
+            return TransformActionProperties(transformId)
+        }
+    }
+
+    enum class Properties(val key: String) {
+        TRANSFORM_ID("transform_id")
+    }
+}