Merge branch 'elastic:main' into main

.buildkite/hooks/pre-command

@@ -92,7 +92,7 @@ fi
 
 # Initialize the build scan and gobld annotations with empty/open <details> tags
 # This ensures that they are collapsible when they get appended to
-if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* ]]; then
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
   cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
 <details>
 

.buildkite/pipelines/intake.template.yml

@@ -14,23 +14,23 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part2
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part3
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:

.buildkite/pipelines/periodic-packaging.template.yml

@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic-packaging.yml

@@ -20,6 +20,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic-platform-support.yml

@@ -19,6 +19,7 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9

.buildkite/pipelines/periodic.template.yml

@@ -54,7 +54,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
@@ -82,14 +82,14 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
@@ -180,3 +180,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
+    if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+\$/
+  - label: Check branch consistency
+    command: .ci/scripts/run-gradle.sh branchConsistency
+    timeout_in_minutes: 15
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-2
+  - label: Check branch protection rules
+    command: .buildkite/scripts/branch-protection.sh
+    timeout_in_minutes: 5

.buildkite/pipelines/periodic.yml

@@ -1150,7 +1150,7 @@ steps:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
@@ -1241,3 +1241,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
+    if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+\$/
+  - label: Check branch consistency
+    command: .ci/scripts/run-gradle.sh branchConsistency
+    timeout_in_minutes: 15
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-2
+  - label: Check branch protection rules
+    command: .buildkite/scripts/branch-protection.sh
+    timeout_in_minutes: 5

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -3,9 +3,9 @@ config:
 steps:
   - group: packaging-tests-unix
     steps:
-      - label: "{{matrix.image}} / {{matrix.PACKAGING_TASK}} / packaging-tests-unix"
-        key: "packaging-tests-unix"
-        command: ./.ci/scripts/packaging-test.sh $$PACKAGING_TASK
+      - label: "{{matrix.image}} / docker / packaging-tests-unix"
+        key: "packaging-tests-unix-docker"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -22,18 +22,71 @@ steps:
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
+              - rocky-9
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-16-32768
+      - label: "{{matrix.image}} / packages / packaging-tests-unix"
+        key: "packaging-tests-unix-packages"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.packages
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rocky-9
+              - rhel-7
+              - rhel-8
+              - rhel-9
+              - almalinux-8
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          diskSizeGb: 350
+          machineType: custom-16-32768
+      - label: "{{matrix.image}} / archives / packaging-tests-unix"
+        key: "packaging-tests-unix-archives"
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.archives
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - centos-7
+              - debian-10
+              - debian-11
+              - opensuse-leap-15
+              - oraclelinux-7
+              - oraclelinux-8
+              - sles-12
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rocky-9
               - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8
-            PACKAGING_TASK:
-              - destructiveDistroTest.docker
-              - destructiveDistroTest.packages
-              - destructiveDistroTest.archives
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
           diskSizeGb: 350
           machineType: custom-16-32768
-        env:
-          PACKAGING_TASK: "{{matrix.PACKAGING_TASK}}"

.buildkite/pull-requests.json

@@ -12,9 +12,6 @@
       "build_on_commit": true,
       "build_on_comment": true,
       "trigger_comment_regex": "run\\W+elasticsearch-ci.+",
-      "labels": [
-        "buildkite-opt-in"
-      ],
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
     },

.buildkite/scripts/branch-protection.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -euo pipefail
+
+STATUS=$(curl -s "https://api.github.com/repos/elastic/elasticsearch/branches/$BUILDKITE_BRANCH" | jq '.protected')
+echo "Branch $BUILDKITE_BRANCH protection status is: $STATUS"
+if [[ "$STATUS" == "false" ]]; then
+  echo "Development branch $BUILDKITE_BRANCH is not set as protected in GitHub but should be."
+  exit 1
+fi

.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml

@@ -2,12 +2,12 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+branch-consistency
     display-name: "elastic / elasticsearch # %BRANCH% - branch consistency"
-    description: Testing of the Elasticsearch %BRANCH% branch version consistency.
-    triggers:
-      - timed: "H 7 * * *"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
+    triggers: []
     builders:
       - inject:
-          properties-file: '.ci/java-versions.properties'
+          properties-file: ".ci/java-versions.properties"
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
       - shell: |

.ci/jobs.t/elastic+elasticsearch+branch-protection.yml

@@ -2,10 +2,10 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+branch-protection
     display-name: "elastic / elasticsearch # %BRANCH% - branch protection"
-    description: Elasticsearch %BRANCH% branch protection.
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     node: master
-    triggers:
-      - timed: "H 7 * * *"
+    triggers: []
     scm: []
     parameters: []
     builders:

.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc-snapshots.yml

@@ -3,7 +3,6 @@ jjbb-template: matrix-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc-snapshots
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc"
-  - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks.
   - matrix-yaml-file: ".ci/snapshotBwcVersions"
   - matrix-variable: BWC_VERSION
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$BWC_VERSION#bwcTest"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml

@@ -3,5 +3,4 @@ jjbb-template: generic-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 1"
-  - job-description: Elasticsearch %BRANCH% branch intake check part 1.
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml

@@ -3,5 +3,4 @@ jjbb-template: generic-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 2"
-  - job-description: Elasticsearch %BRANCH% branch intake check part 2.
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+part3.yml

@@ -3,5 +3,4 @@ jjbb-template: generic-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part3
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 3"
-  - job-description: Elasticsearch %BRANCH% branch intake check part 3.
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml

@@ -3,5 +3,4 @@ jjbb-template: generic-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake rest compatibility"
-  - job-description: Elasticsearch %BRANCH% branch intake REST compatibility checks.
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml

@@ -3,5 +3,4 @@ jjbb-template: generic-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake sanity check"
-  - job-description: Elasticsearch %BRANCH% branch intake sanity check.
   - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml

@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
     display-name: "elastic / elasticsearch # %BRANCH% - update last good commit"
-    description: Elasticsearch %BRANCH% branch update last good commit in build-stats.
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     node: light
     properties: []
     builders:

.ci/jobs.t/elastic+elasticsearch+intake.yml

@@ -2,24 +2,12 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+intake
     display-name: "elastic / elasticsearch # %BRANCH% - intake"
-    description: "Testing of the Elasticsearch %BRANCH% branch on every push.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: multijob
     node: master
     vault: []
-    triggers:
-      # We use this trigger instead of the provided "github" webhook trigger because it's more robust.
-      # Here we only trigger builds for pushes to the corresponding branch, rather than a push to any branch of the
-      # configured git repository. This avoids duplicate builds being triggered when pushes to multiple branches are
-      # done in quick succession.
-      - generic-webhook-trigger:
-          post-content-params:
-            - type: JSONPath
-              key: ref
-              value: '$.ref'
-          regex-filter-text: '$ref'
-          regex-filter-expression: "^refs/heads/%BRANCH%$"
-          cause: Push to GitHub (refs/heads/%BRANCH%)
-          silent-response: true
+    triggers: []
     scm:
       - git:
           wipe-workspace: false

.ci/jobs.t/elastic+elasticsearch+periodic+bwc.yml

@@ -3,7 +3,6 @@ jjbb-template: matrix-gradle-unix.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+periodic+bwc
   - job-display-name: "elastic / elasticsearch # %BRANCH% - backwards compatibility matrix"
-  - job-description: "Testing of the Elasticsearch %BRANCH% branch backwards compatibility matrix.\n"
   - matrix-yaml-file: ".ci/bwcVersions"
   - matrix-variable: BWC_VERSION
   - gradle-args: "-Dbwc.checkout.align=true v$BWC_VERSION#bwcTest"

.ci/jobs.t/elastic+elasticsearch+periodic+concurrent-search-tests.yml

@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+concurrent-search-tests
     display-name: "elastic / elasticsearch # %BRANCH% - concurrent search tests"
-    description: "Testing concurrent search enabled for the Elasticsearch %BRANCH% branch.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     node: "general-purpose && docker"
     builders:
       - inject:

.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml

@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+ear
     display-name: "elastic / elasticsearch # %BRANCH% - encryption at rest"
-    description: "The Elasticsearch %BRANCH% branch encryption at rest compatibility tests.\n\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     node: packaging-large
     builders:
       - inject: