Merge branch '8.x' of https://github.com/elastic/beats into mergify/b…

…p/8.x/pr-40766
elastic · Oct 1, 2024 · 472bfe1 · 472bfe1
2 parents 4224f72 + ffd7f25
commit 472bfe1
Show file tree

Hide file tree

Showing 375 changed files with 29,575 additions and 21,085 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -19,6 +19,8 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -40,14 +42,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "auditbeat: check/update"
 
+      - label: "Auditbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "auditbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests

diff --git a/.buildkite/aws-tests-pipeline.yml b/.buildkite/aws-tests-pipeline.yml
@@ -19,7 +19,7 @@ env:
 steps:
   - label: ":ubuntu: x-pack/filebeat: AWS Tests"
     key: "x-pack-filebeat-extended-cloud-test"
-    skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467"
+    skip: "skipping:  elastic/ingest-dev#3467"
     # Related issue: https://github.com/elastic/ingest-dev/issues/3467
     env:
       MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"

diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -8,11 +8,24 @@ env:
   # Other deps
   ASDF_KIND_VERSION: "0.20.0"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
 steps:
   - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
 
     steps:
+    - label: "Deploy/K8S: Run pre-commit"
+      command: "pre-commit run --all-files"
+      agents:
+        image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+        cpu: "2000m"
+        memory: "4Gi"
+        useCustomGlobalHooks: true
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s: pre-commit"
+
     - label: "Checks"
       command: |
         set -euo pipefail
@@ -26,10 +39,10 @@ steps:
         - github_commit_status:
             context: "deploy/k8s checks"
 
-    - label: "K8S Test/K8S version: v1.30.2"
-      key: "k8s-test-130"
+    - label: "K8S Test/K8S version: v1.31.0"
+      key: "k8s-test-131"
       env:
-        K8S_VERSION: "v1.30.2"
+        K8S_VERSION: "v1.31.0"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -46,12 +59,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.30.2"
+            context: "deploy/k8s test v1.31.0"
 
-    - label: "K8S Test/K8S version: v1.29.4"
-      key: "k8s-test-129"
+    - label: "K8S Test/K8S version: v1.30.4"
+      key: "k8s-test-130"
       env:
-        K8S_VERSION: "v1.29.4"
+        K8S_VERSION: "v1.30.4"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -68,12 +81,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.29.4"
+            context: "deploy/k8s test v1.30.4"
 
-    - label: "K8S Test/K8S version: v1.28.9"
-      key: "k8s-test-128"
+    - label: "K8S Test/K8S version: v1.29.8"
+      key: "k8s-test-129"
       env:
-        K8S_VERSION: "v1.28.9"
+        K8S_VERSION: "v1.29.8"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -90,12 +103,12 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.28.9"
+            context: "deploy/k8s test v1.29.8"
 
-    - label: "K8S Test/K8S version: v1.27.13"
-      key: "k8s-test-127"
+    - label: "K8S Test/K8S version: v1.28.13"
+      key: "k8s-test-128"
       env:
-        K8S_VERSION: "v1.27.13"
+        K8S_VERSION: "v1.28.13"
         MODULE: "${MODULE}"
       commands: |
         set -euo pipefail
@@ -112,4 +125,4 @@ steps:
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "deploy/k8s test v1.27.13"
+            context: "deploy/k8s test v1.28.13"
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -18,6 +18,8 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -32,6 +34,7 @@ env:
 steps:
   - group: "Check/Update"
     key: "filebeat-check-update"
+
     steps:
       - label: "Filebeat: Run check/update"
         command: |
@@ -42,14 +45,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "filebeat: check/update"
 
+      - label: "Filebeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "filebeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -18,9 +18,12 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
   ASDF_NODEJS_VERSION: 18.17.1
+  ASDF_PYTHON_VERSION: 3.10.9
 
   # Unit tests
   RACE_DETECTOR: "true"
@@ -40,14 +43,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "hearbeat: check/update"
 
+      - label: "Heartbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "heartbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -135,17 +148,9 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win 2022 Unit Tests"
 
+      # Heartbeat has no modules
       - label: ":ubuntu: Heartbeat: Go Integration Tests"
         command: |
-          set -euo pipefail
-          echo "~~~ Installing @elastic/synthetics"
-          npm install -g @elastic/synthetics
-
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet heartbeat
-
-          echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -162,14 +167,9 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
+      # Heartbeat has no modules yet
       - label: ":ubuntu: Heartbeat: Python Integration Tests"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet heartbeat
-
-          echo "~~~ Running tests"
           cd heartbeat
           mage pythonIntegTest
         retry:

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -6,6 +6,13 @@ set -euo pipefail
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
+
+  if [[ "${GITHUB_PR_TRIGGER_COMMENT:-}" != "" ]]; then
+    buildkite-agent annotate "**No changes.** Triggered from comment: _${GITHUB_PR_TRIGGER_COMMENT}_" --style "info" --context 'ctx-info'
+  fi
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then
     PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET

diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -9,6 +9,8 @@ env:
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -30,14 +32,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "libbeat: check/update"
 
+      - label: "Libbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "libbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -67,7 +79,7 @@ steps:
           - "libbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "libbeat: Ununtu x86_64 Unit Tests"
+              context: "libbeat: Ubuntu x86_64 Unit Tests"
 
       - label: ":ubuntu: Libbeat: Go Integration Tests"
         key: "mandatory-int-test"

diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
@@ -18,10 +18,15 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Integration Tests
   K8S_VERSION: "v1.29.0"
   ASDF_KIND_VERSION: "0.20.0"
 
+  # Module Tests
+  BEAT_PATH: "metricbeat"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -42,14 +47,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "metricbeat: check/update"
 
+      - label: "Metricbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "metricbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -80,14 +95,10 @@ steps:
           - github_commit_status:
               context: "metricbeat: Ubuntu x86_64 Unit Tests"
 
-      - label: ":ubuntu: Metricbeat: Go Integration Tests"
+      - label: ":ubuntu: Metricbeat: Go Integration Tests (Module)"
         key: "mandatory-int-test"
         command: |
           set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet metricbeat
-
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat
@@ -104,16 +115,12 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Go Integration Tests"
+              context: "metricbeat: Go Integration Tests (Module)"
 
-      - label: ":ubuntu: Metricbeat: Python Integration Tests"
+      - label: ":ubuntu: Metricbeat: Python Integration Tests (Module)"
         key: "mandatory-python-int-test"
         command: |
           set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet metricbeat
-
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat