Merge branch 'main' into load_constants

.buildkite/pipelines/intake.template.yml

@@ -14,23 +14,31 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part2
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part3
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:

.buildkite/pipelines/intake.yml

@@ -33,14 +33,22 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.15", "8.10.5", "8.11.0", "8.12.0"]
+            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic-packaging.yml

@@ -20,6 +20,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9
@@ -1072,6 +1073,22 @@ steps:
         env:
           BWC_VERSION: 7.17.15
 
+      - label: "{{matrix.image}} / 7.17.16 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.16
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.16
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1680,8 +1697,24 @@ steps:
         env:
           BWC_VERSION: 8.10.4
 
-      - label: "{{matrix.image}} / 8.10.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.5
+      - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.0
+
+      - label: "{{matrix.image}} / 8.11.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1694,10 +1727,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.10.5
+          BWC_VERSION: 8.11.1
 
-      - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
+      - label: "{{matrix.image}} / 8.11.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -1710,7 +1743,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.0
+          BWC_VERSION: 8.11.2
 
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0

.buildkite/pipelines/periodic-platform-support.yml

@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic.template.yml

@@ -54,7 +54,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
@@ -82,7 +82,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
@@ -180,3 +180,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
+    if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+\$/
+  - label: Check branch consistency
+    command: .ci/scripts/run-gradle.sh branchConsistency
+    timeout_in_minutes: 15
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-2
+  - label: Check branch protection rules
+    command: .buildkite/scripts/branch-protection.sh
+    timeout_in_minutes: 5

.buildkite/pipelines/periodic.yml

@@ -652,6 +652,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.15
+      - label: 7.17.16 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.16#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.16
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1032,26 +1042,36 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.4
-      - label: 8.10.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.5#bwcTest
+      - label: 8.11.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: custom-32-98304
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.10.5
-      - label: 8.11.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
+          BWC_VERSION: 8.11.0
+      - label: 8.11.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: custom-32-98304
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.11.0
+          BWC_VERSION: 8.11.1
+      - label: 8.11.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.2
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300
@@ -1241,3 +1261,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
+    if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+\$/
+  - label: Check branch consistency
+    command: .ci/scripts/run-gradle.sh branchConsistency
+    timeout_in_minutes: 15
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-2
+  - label: Check branch protection rules
+    command: .buildkite/scripts/branch-protection.sh
+    timeout_in_minutes: 5

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -3,9 +3,9 @@ config:
 steps:
   - group: packaging-tests-unix
     steps:
-      - label: "{{matrix.image}} / {{matrix.PACKAGING_TASK}} / packaging-tests-unix"
-        key: "packaging-tests-unix"
-        command: ./.ci/scripts/packaging-test.sh $$PACKAGING_TASK
+      - label: "{{matrix.image}} / docker / packaging-tests-unix"
+        key: "packaging-tests-unix-docker"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -22,18 +22,71 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-16-32768
+      - label: "{{matrix.image}} / packages / packaging-tests-unix"
+        key: "packaging-tests-unix-packages"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.packages
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rocky-9
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-16-32768
+      - label: "{{matrix.image}} / archives / packaging-tests-unix"
+        key: "packaging-tests-unix-archives"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.archives
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8
-            PACKAGING_TASK:
-              - destructiveDistroTest.docker
-              - destructiveDistroTest.packages
-              - destructiveDistroTest.archives
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: custom-16-32768
-        env:
-          PACKAGING_TASK: "{{matrix.PACKAGING_TASK}}"

.buildkite/pipelines/pull-request/part-4-fips.yml

@@ -0,0 +1,11 @@
+config:
+  allow-labels: "Team:Security"
+steps:
+  - label: part-4-fips
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-windows.yml

@@ -0,0 +1,14 @@
+config:
+  allow-labels: "test-windows"
+steps:
+  - label: part-4-windows
+    command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-windows-2022
+      machineType: custom-32-98304
+      diskType: pd-ssd
+      diskSizeGb: 350
+    env:
+      GRADLE_TASK: checkPart4

.buildkite/pipelines/pull-request/part-4.yml

@@ -0,0 +1,11 @@
+config:
+  skip-target-branches: "7.17"
+steps:
+  - label: part-4
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk