Merge branch 'main' into mtoff/json-span-events

DataDog · Oct 18, 2024 · 07299c0 · 07299c0
2 parents 47af7c8 + 1c49cec
commit 07299c0
Show file tree

Hide file tree

Showing 694 changed files with 13,766 additions and 10,783 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v45979046-89a73a40
+      - image: gcr.io/datadoghq/agent-circleci-runner:v46542806-c7a4a6be
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent
@@ -100,7 +100,7 @@ jobs:
       - run:
           name: run unit tests
           no_output_timeout: 20m
-          command: inv -e test --rerun-fails=2 --python-runtimes 3 --coverage --race --profile --cpus 8 --build-stdlib --test-washer
+          command: inv -e test --rerun-fails=2 --coverage --race --profile --cpus 8 --build-stdlib --test-washer
           environment:
             GO_TEST_SKIP_FLAKE: "true"
 

diff --git a/.copyright-overrides.yml b/.copyright-overrides.yml
@@ -373,3 +373,5 @@ github.com/vibrantbyte/go-antpath/extend: vibrantbyte <[email protected]>|suchao
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp: Copyright 2024 gRPC authors.
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/grpc_gcp: Copyright 2024 gRPC authors.
 github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/multiendpoint: Copyright 2024 gRPC authors.
+
+github.com/NVIDIA/go-nvml/*: Copyright 2023 NVIDIA CORPORATION
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -131,15 +131,15 @@
 /.gitlab/deps_build/                                 @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/windows_python_build.yml                    @DataDog/windows-agent
 
-/.gitlab/kitchen_*                                   @DataDog/agent-delivery
-/.gitlab/kitchen_testing/windows.yml                 @DataDog/agent-delivery @DataDog/windows-agent
-/.gitlab/e2e_install_packages/windows.yml            @DataDog/agent-delivery @DataDog/windows-agent
+/.gitlab/kitchen_*                                   @DataDog/container-ecosystems @DataDog/agent-delivery
+/.gitlab/kitchen_testing/windows.yml                 @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
+/.gitlab/e2e_install_packages/windows.yml            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
 /.gitlab/common/test_infra_version.yml               @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx-loops
-/.gitlab/e2e/install_packages                        @DataDog/agent-delivery
+/.gitlab/e2e/install_packages                        @DataDog/container-ecosystems @DataDog/agent-delivery
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
@@ -221,7 +221,7 @@
 /cmd/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
 /cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
-/cmd/agent/install*.sh                  @DataDog/agent-delivery
+/cmd/agent/install*.sh                  @DataDog/container-ecosystems @DataDog/agent-delivery
 /cmd/cluster-agent/                     @DataDog/container-platform
 /cmd/cluster-agent/commands/            @DataDog/container-platform
 /cmd/cluster-agent-cloudfoundry/        @DataDog/platform-integrations
@@ -244,7 +244,7 @@
 /cmd/system-probe/modules/tcp_queue_tracer* @DataDog/container-integrations
 /cmd/system-probe/modules/traceroute*   @DataDog/network-device-monitoring @Datadog/Networks
 /cmd/system-probe/modules/ping*         @DataDog/network-device-monitoring
-/cmd/system-probe/modules/service_discover* @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/cmd/system-probe/modules/service_discover* @DataDog/universal-service-monitoring
 /cmd/system-probe/modules/language_detection* @DataDog/processes @DataDog/universal-service-monitoring
 /cmd/system-probe/runtime/              @DataDog/agent-security
 /cmd/system-probe/modules/dynamic_instrumentation* @DataDog/debugger
@@ -270,6 +270,7 @@
 /docs/cloud-workload-security/          @DataDog/documentation @DataDog/agent-security
 
 /docs/public/components/                                     @DataDog/documentation @DataDog/agent-shared-components
+/docs/public/hostname/                                       @DataDog/documentation @DataDog/agent-shared-components
 /docs/public/architecture/dogstatsd/                         @DataDog/documentation @DataDog/agent-metrics-logs
 /docs/public/guidelines/deprecated-components-documentation/ @DataDog/documentation @DataDog/agent-shared-components
 
@@ -334,6 +335,7 @@
 /comp/languagedetection/client @DataDog/container-platform
 /comp/rdnsquerier @DataDog/network-device-monitoring
 /comp/serializer/compression @DataDog/agent-metrics-logs
+/comp/snmpscan @DataDog/network-device-monitoring
 # END COMPONENTS
 
 # pkg
@@ -345,7 +347,7 @@
 /pkg/cli/                               @DataDog/agent-shared-components
 /pkg/cli/subcommands/clusterchecks      @DataDog/container-platform
 /pkg/dogstatsd/                         @DataDog/agent-metrics-logs
-/pkg/discovery/				@DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/pkg/discovery/                         @DataDog/universal-service-monitoring
 /pkg/errors/                            @DataDog/agent-shared-components
 /pkg/forwarder/                         @DataDog/agent-processing-and-routing
 /pkg/gohai                              @DataDog/agent-shared-components
@@ -370,6 +372,7 @@
 /pkg/trace/traceutil/otel*.go           @DataDog/opentelemetry
 /pkg/trace/stats/                       @DataDog/agent-apm @DataDog/opentelemetry
 /pkg/trace/telemetry/                   @DataDog/apm-trace-storage
+/pkg/trace/transform/                   @DataDog/opentelemetry
 /comp/core/autodiscovery/listeners/           @DataDog/container-platform
 /comp/core/autodiscovery/listeners/cloudfoundry*.go  @DataDog/platform-integrations
 /comp/core/autodiscovery/listeners/snmp*.go   @DataDog/network-device-monitoring
@@ -405,7 +408,7 @@
 /pkg/collector/corechecks/net/          @DataDog/platform-integrations
 /pkg/collector/corechecks/oracle        @DataDog/database-monitoring
 /pkg/collector/corechecks/sbom/         @DataDog/container-integrations
-/pkg/collector/corechecks/servicediscovery/ @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/pkg/collector/corechecks/servicediscovery/ @DataDog/universal-service-monitoring
 /pkg/collector/corechecks/snmp/         @DataDog/network-device-monitoring
 /pkg/collector/corechecks/system/                 @DataDog/platform-integrations
 /pkg/collector/corechecks/system/**/*_windows*.go @DataDog/platform-integrations @DataDog/windows-agent
@@ -569,7 +572,6 @@
 /comp/core/workloadmeta/collectors/internal/cloudfoundry @DataDog/platform-integrations
 /pkg/sbom/                              @DataDog/container-integrations @DataDog/agent-security
 /pkg/internaltelemetry                  @DataDog/windows-kernel-integrations @DataDog/fleet
-/pkg-config/                            @DataDog/agent-delivery
 /pkg/networkpath/                       @DataDog/network-device-monitoring @DataDog/Networks
 /pkg/collector/corechecks/networkpath/  @DataDog/network-device-monitoring @DataDog/Networks
 
@@ -614,11 +616,12 @@
 /test/benchmarks/                       @DataDog/agent-metrics-logs
 /test/benchmarks/kubernetes_state/      @DataDog/container-integrations
 /test/integration/                      @DataDog/container-integrations
+/test/integration/docker/otel_agent_build_tests.py @DataDog/opentelemetry
 /test/integration/serverless            @DataDog/serverless @Datadog/serverless-aws
 /test/integration/serverless_perf       @DataDog/serverless @Datadog/serverless-aws
 /test/kitchen/                          @DataDog/agent-devx-loops
-/test/kitchen/test-definitions/         @DataDog/agent-delivery
-/test/kitchen/test/integration/         @DataDog/agent-delivery
+/test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
+/test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
 /test/kitchen/kitchen-azure-security-agent-test.yml @DataDog/agent-security
 /test/kitchen/kitchen-vagrant-security-agent.yml @DataDog/agent-security
 /test/kitchen/site-cookbooks/dd-security-agent-check/ @DataDog/agent-security
@@ -637,18 +640,17 @@
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/network-device-monitoring
 /test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/network-device-monitoring
-/test/fakeintake/aggregator/servicediscoveryAggregator.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring
-/test/fakeintake/aggregator/servicediscoveryAggregator_test.go @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/test/fakeintake/aggregator/servicediscovery* @DataDog/universal-service-monitoring
 /test/new-e2e/                                @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/new-e2e/pkg/components/datadog-installer @DataDog/windows-agent
 /test/new-e2e/test-infra-definition           @DataDog/agent-devx-loops
 /test/new-e2e/system-probe                    @DataDog/ebpf-platform
 /test/new-e2e/scenarios/system-probe          @DataDog/ebpf-platform
-/test/new-e2e/tests/agent-platform            @DataDog/agent-delivery @DataDog/agent-devx-loops
+/test/new-e2e/tests/agent-platform            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/agent-devx-loops
 /test/new-e2e/tests/agent-shared-components   @DataDog/agent-shared-components
 /test/new-e2e/tests/agent-subcommands         @DataDog/agent-shared-components
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
-/test/new-e2e/tests/discovery                 @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/test/new-e2e/tests/discovery                 @DataDog/universal-service-monitoring
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/network-device-monitoring
 /test/new-e2e/tests/npm                       @DataDog/Networks
@@ -665,6 +667,7 @@
 /test/new-e2e/tests/remote-config             @DataDog/remote-config
 /test/new-e2e/tests/updater                   @DataDog/fleet @DataDog/windows-agent
 /test/new-e2e/tests/installer                 @DataDog/fleet @DataDog/windows-agent
+/test/new-e2e/tests/gpu                       @Datadog/ebpf-platform
 /test/otel/                                   @DataDog/opentelemetry
 /test/system/                                 @DataDog/agent-shared-components
 /test/system/dogstatsd/                       @DataDog/agent-metrics-logs

diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml
@@ -159,6 +159,5 @@ jobs:
         if: steps.should.outputs.should_run == 'true'
         with:
           header: serverless-binary-size
-          hide_and_recreate: true
-          hide_classify: "RESOLVED"
+          recreate: true
           path: ${{ steps.write.outputs.filename }}
diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml
@@ -30,7 +30,7 @@ jobs:
           path: go/src/github.com/DataDog/datadog-agent
 
       - name: Set up Node 20
-        uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
 

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -30,7 +30,6 @@ include:
   - .gitlab/integration_test/include.yml
   - .gitlab/internal_image_deploy/internal_image_deploy.yml
   - .gitlab/internal_kubernetes_deploy/include.yml
-  - .gitlab/junit_upload/junit_upload.yml
   - .gitlab/kitchen_cleanup/include.yml
   - .gitlab/kitchen_deploy/kitchen_deploy.yml
   - .gitlab/kitchen_testing/include.yml
@@ -96,7 +95,9 @@ stages:
   - internal_image_deploy
   - install_script_testing
   - e2e_pre_test
+  - e2e_init
   - e2e
+  - e2e_cleanup
   - e2e_k8s
   - e2e_install_packages
   - kitchen_cleanup
@@ -171,15 +172,15 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_BUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_WINBUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_ARMBUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v46542806-c7a4a6be
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v45979046-89a73a40
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v46542806-c7a4a6be
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
   DEB_GPG_KEY_ID: c0962c7d
@@ -218,6 +219,7 @@ variables:
   E2E_TESTS_AZURE_TENANT_ID: ci.datadog-agent.e2e_tests_azure_tenant_id  # agent-devx-loops
   E2E_TESTS_AZURE_SUBSCRIPTION_ID: ci.datadog-agent.e2e_tests_azure_subscription_id  # agent-devx-loops
   E2E_TESTS_GCP_CREDENTIALS: ci.datadog-agent.e2e_tests_gcp_credentials  # agent-devx-loops
+  E2E_PULUMI_CONFIG_PASSPHRASE: ci.datadog-agent.pulumi_password  # agent-devx-loops
   KITCHEN_EC2_SSH_KEY: ci.datadog-agent.aws_ec2_kitchen_ssh_key  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_ID: ci.datadog-agent.azure_kitchen_client_id  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_SECRET: ci.datadog-agent.azure_kitchen_client_secret  # agent-devx-loops
@@ -254,15 +256,21 @@ variables:
   VCPKG_BLOB_SAS_URL: ci.datadog-agent-buildimages.vcpkg_blob_sas_url  # windows-agent
   WINGET_PAT: ci.datadog-agent.winget_pat  # windows-agent
   # End aws ssm variables
+
   # Start vault variables
+  AGENT_API_KEY_ORG2: agent-api-key-org-2  # agent-devx-infra
+  AGENT_APP_KEY_ORG2: agent-ci-app-key-org-2  # agent-devx-infra
+  AGENT_GITHUB_APP: agent-github-app  # agent-devx-infra
+  AGENT_QA_E2E: agent-qa-e2e  # agent-devx-loops
+  ATLASSIAN_WRITE: atlassian-write  # agent-devx-infra
+  DOCKER_REGISTRY_RO: dockerhub-readonly  # agent-delivery
+  GITLAB_TOKEN: gitlab-token  # agent-devx-infra
+  INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2  # agent-devx-infra
+  MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
+  MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra
+  SLACK_AGENT: slack-agent-ci  # agent-devx-infra
   # End vault variables
 
-  ATLASSIAN_WRITE: atlassian-write
-  AGENT_GITHUB_APP: agent-github-app
-  MACOS_GITHUB_APP_1: macos-github-app-one
-  MACOS_GITHUB_APP_2: macos-github-app-two
-  DOCKER_REGISTRY_RO: dockerhub-readonly
-
   DD_PKG_VERSION: "latest"
 
   # Job stage attempts (see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts)
@@ -768,19 +776,6 @@ workflow:
       paths: *system_probe_change_paths
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_e2e_changes_or_manual:
-  - <<: *if_main_branch
-  - !reference [.except_mergequeue]
-  - changes:
-      paths:
-        - test/new-e2e/pkg/**/*
-        - test/new-e2e/test-infra-definition/*
-        - test/new-e2e/go.mod
-      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-    when: on_success
-  - when: manual
-    allow_failure: true
-
 # New E2E related rules
 
 .on_e2e_main_release_or_rc: # This rule is used as a base for all new-e2e rules
@@ -797,6 +792,7 @@ workflow:
     when: on_success
   - changes:
       paths:
+        - .gitlab/e2e/e2e.yml
         - test/new-e2e/pkg/**/*
         - test/new-e2e/go.mod
         - flakes.yaml
@@ -892,6 +888,7 @@ workflow:
   - changes:
       paths:
         # TODO: Add paths that should trigger tests for npm
+        - pkg/network/**/*
         - test/new-e2e/tests/npm/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
@@ -1192,3 +1189,12 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
   - when: manual
     allow_failure: true
+
+.on_gpu_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/gpu/**/*
+        - test/new-e2e/tests/gpu/**/*
+        - pkg/collector/corechecks/gpu/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml
@@ -16,6 +16,7 @@ needs-rules:
     - build_processed_btfhub_archive
     - check_already_deployed_version_7
     - cleanup_kitchen_functional_test
+    - compute_gitlab_ci_config
     - deploy_containers-cws-instrumentation-final-versioned
     - deploy_containers-cws-instrumentation-latest
     - deploy_containers-cws-instrumentation-rc-mutable
@@ -26,9 +27,12 @@ needs-rules:
     - lint_flavor_dogstatsd_linux-x64
     - lint_flavor_heroku_linux-x64
     - lint_flavor_iot_linux-x64
+    - lint_gitlab_ci
+    - lint_gitlab_ci_jobs_codeowners
     - lint_linux-arm64
     - lint_linux-x64
     - lint_macos_gitlab_amd64
+    - new-e2e-eks-cleanup-on-failure
     - periodic_kitchen_cleanup_azure
     - periodic_kitchen_cleanup_ec2
     - periodic_kitchen_cleanup_s3
@@ -105,7 +109,6 @@ job-owners:
     - trace-agent-v05-2cpus-normal_load-fixed_sps
     - trace-agent-v05-2cpus-stress_load-fixed_sps
     - trigger-flakes-finder
-    - unit_tests_arm64_junit_upload
     - unit_tests_notify
     - update_rc_build_links
     - validate_modules
diff --git a/.gitlab/.pre/cancel-prev-pipelines.yml b/.gitlab/.pre/cancel-prev-pipelines.yml
@@ -14,5 +14,5 @@ cancel-prev-pipelines:
       when: never
     - when: on_success
   script:
-    - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN
+    - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN
     - inv pipeline.auto-cancel-previous-pipelines