feat(backendconnection): custom collector image & config reloader

* use a custom built collector image for the OTel collector daemon set
  managed by the operator
* have the OTel collector process automatically reload the
  configuration when the config map changes 

Co-authored-by: Michele Mancioppi <[email protected]>
Co-authored-by: Bastian Krol <[email protected]>

.github/workflows/ci.yaml

@@ -97,18 +97,34 @@ jobs:
               - api/**
               - cmd/**
               - internal/**
+            collector:
+              - .github/actions/build-image/**
+              - .github/workflows/ci.yaml
+              - images/collector/**
+            configuration_reloader:
+              - .github/actions/build-image/**
+              - .github/workflows/ci.yaml
+              - images/configreloader/**
 
       - name: show changed files
         env:
           INSTRUMENTATION_CHANGED_FILES_FLAG: ${{ steps.changed-files.outputs.instrumentation_any_changed }}
           INSTRUMENTATION_CHANGED_FILES_LIST: ${{ steps.changed-files.outputs.instrumentation_all_changed_files }}
           OPERATOR_CONTROLLER_CHANGED_FILES_FLAG: ${{ steps.changed-files.outputs.operator_controller_any_changed }}
           OPERATOR_CONTROLLER_CHANGED_FILES_LIST: ${{ steps.changed-files.outputs.operator_controller_all_changed_files }}
+          COLLECTOR_CHANGED_FILES_FLAG: ${{ steps.changed-files.outputs.collector_any_changed }}
+          COLLECTOR_CHANGED_FILES_LIST: ${{ steps.changed-files.outputs.collector_all_changed_files }}
+          CONFIGURATION_RELOADER_CHANGED_FILES_FLAG: ${{ steps.changed-files.outputs.configuration_reloader_any_changed }}
+          CONFIGURATION_RELOADER_CHANGED_FILES_LIST: ${{ steps.changed-files.outputs.configuration_reloader_all_changed_files }}
         run: |
           echo "files for instrumentation image have changed: $INSTRUMENTATION_CHANGED_FILES_FLAG"
           echo "changed files for instrumentation image: $INSTRUMENTATION_CHANGED_FILES_LIST"
           echo "files for operator controller image have changed: $OPERATOR_CONTROLLER_CHANGED_FILES_FLAG"
           echo "changed files for operator controller image: $OPERATOR_CONTROLLER_CHANGED_FILES_LIST"
+          echo "files for collector image have changed: $COLLECTOR_CHANGED_FILES_FLAG"
+          echo "changed files for collector image: $COLLECTOR_CHANGED_FILES_LIST"
+          echo "files for configuration reloader image have changed: $CONFIGURATION_RELOADER_CHANGED_FILES_FLAG"
+          echo "changed files for configuration reloader image: $CONFIGURATION_RELOADER_CHANGED_FILES_LIST"
 
       - name: build instrumentation image
         uses: ./.github/actions/build-image
@@ -121,6 +137,26 @@ jobs:
           imageUrl: https://github.com/dash0hq/dash0-operator/tree/main/images/instrumentation
           context: images/instrumentation
 
+      - name: build collector image
+        uses: ./.github/actions/build-image
+        if: steps.changed-files.outputs.collector_any_changed == 'true' || contains(github.ref, 'refs/tags/')
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          imageName: collector
+          imageTitle: Dash0 Kubernetes Collector
+          imageDescription: the OpenTelemetry collector for the Dash0 Kubernetes operator
+          context: images/collector
+
+      - name: build configuration reloader image
+        uses: ./.github/actions/build-image
+        if: steps.changed-files.outputs.configuration_reloader_any_changed == 'true' || contains(github.ref, 'refs/tags/')
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          imageName: configuration-reloader
+          imageTitle: Dash0 Kubernetes Configuration Reloader
+          imageDescription: the configuration reloader for the Dash0 Kubernetes operator
+          context: images/configreloader
+
       - name: build operator controller image
         uses: ./.github/actions/build-image
         if: steps.changed-files.outputs.operator_controller_any_changed == 'true' || contains(github.ref, 'refs/tags/')

CONTRIBUTING.md

@@ -30,9 +30,22 @@ Run `make docker-build` to build the container image locally, this will tag the
 After that, you can deploy the operator to your cluster:
 
 * Deploy the locally built image `operator-controller:latest` to the cluster: `make deploy-via-helm`
-  (or `make deploy-via-kustomize`)
 * Alternatively, deploy with images from a remote registry:
-  `make deploy-via-helm IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller IMG_TAG=main-dev IMG_PULL_POLICY="" INSTRUMENTATION_IMG_REPOSITORY=ghcr.io/dash0hq/instrumentation INSTRUMENTATION_IMG_TAG=main-dev INSTRUMENTATION_IMG_PULL_POLICY=""`
+  ```
+  make deploy-via-helm \
+    CONTROLLER_IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller \
+    CONTROLLER_IMG_TAG=main-dev \
+    CONTROLLER_IMG_PULL_POLICY="" \
+    INSTRUMENTATION_IMG_REPOSITORY=ghcr.io/dash0hq/instrumentation \
+    INSTRUMENTATION_IMG_TAG=main-dev \
+    INSTRUMENTATION_IMG_PULL_POLICY="" \
+    COLLECTOR_IMG_REPOSITORY=ghcr.io/dash0hq/collector \
+    COLLECTOR_IMG_TAG=main-dev \
+    COLLECTOR_IMG_PULL_POLICY="" \
+    CONFIGURATION_RELOADER_IMG_REPOSITORY=configuration-reloader \
+    CONFIGURATION_RELOADER_IMG_TAG=main-dev \
+    CONFIGURATION_RELOADER_IMG_PULL_POLICY=""
+  ```
 * The custom resource definition will automatically be installed when deploying the operator. However, you can also do
   that separately via kustomize if required via `make install`.
 
@@ -45,14 +58,6 @@ admin.
 make undeploy-via-helm
 ```
 
-or
-
-```sh
-make undeploy-via-kustomize
-```
-
-When undeploying the controllor, the same tool (helm vs. kustomiz) should be used as when deploying it.
-
 This will also remove the custom resource definition. However, the custom resource definition can also be removed
 separately via `make uninstall` without removing the operator.
 
@@ -77,12 +82,18 @@ The tests can also be run with remote images, like this:
 ```
 BUILD_OPERATOR_CONTROLLER_IMAGE=false \
   BUILD_INSTRUMENTATION_IMAGE=false \
-  IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller \
-  IMG_TAG=main-dev \
-  IMG_PULL_POLICY="" \
+  CONTROLLER_IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller \
+  CONTROLLER_IMG_TAG=main-dev \
+  CONTROLLER_IMG_PULL_POLICY="" \
   INSTRUMENTATION_IMG_REPOSITORY=ghcr.io/dash0hq/instrumentation \
   INSTRUMENTATION_IMG_TAG=main-dev \
   INSTRUMENTATION_IMG_PULL_POLICY="" \
+  COLLECTOR_IMG_REPOSITORY=ghcr.io/dash0hq/collector \
+  COLLECTOR_IMG_TAG=main-dev \
+  COLLECTOR_IMG_PULL_POLICY="" \
+  CONFIGURATION_RELOADER_IMG_REPOSITORY=configuration-reloader \
+  CONFIGURATION_RELOADER_IMG_TAG=main-dev \
+  CONFIGURATION_RELOADER_IMG_PULL_POLICY="" \
   make test-e2e
 ```
 
@@ -96,18 +107,24 @@ BUILD_OPERATOR_CONTROLLER_IMAGE=false \
   BUILD_INSTRUMENTATION_IMAGE=false \
   OPERATOR_HELM_CHART=dash0-operator/dash0-operator \
   OPERATOR_HELM_CHART_URL=https://dash0hq.github.io/dash0-operator \
-  IMG_REPOSITORY="" \
-  IMG_TAG="" \
-  IMG_PULL_POLICY="" \
+  CONTROLLER_IMG_REPOSITORY="" \
+  CONTROLLER_IMG_TAG="" \
+  CONTROLLER_IMG_PULL_POLICY="" \
   INSTRUMENTATION_IMG_REPOSITORY="" \
   INSTRUMENTATION_IMG_TAG="" \
   INSTRUMENTATION_IMG_PULL_POLICY="" \
+  COLLECTOR_IMG_REPOSITORY="" \
+  COLLECTOR_IMG_TAG="" \
+  COLLECTOR_IMG_PULL_POLICY="" \
+  CONFIGURATION_RELOADER_IMG_REPOSITORY="" \
+  CONFIGURATION_RELOADER_IMG_TAG="" \
+  CONFIGURATION_RELOADER_IMG_PULL_POLICY="" \
   make test-e2e
 ```
 
-Note: Unsetting parameters like `IMG_REPOSITORY` explicitly (by setting them to an empty string) will lead to the
+Note: Unsetting parameters like `CONTROLLER_IMG_REPOSITORY` explicitly (by setting them to an empty string) will lead to the
 end-to-end test not setting those values when deploying via helm, so that the default value from the chart will be used.
-Otherwise, without `IMG_REPOSITORY=""` being present, the test suite will use `IMG_REPOSITORY=operator-controller` (the
+Otherwise, without `CONTROLLER_IMG_REPOSITORY=""` being present, the test suite will use `CONTROLLER_IMG_REPOSITORY=operator-controller` (the
 image built from local sources) as the default setting.
 
 ### Semi-Manual Test Scenarios
@@ -135,12 +152,18 @@ they deploy in their `AfterAll`/`AfterEach` hooks. The scripts in `test-resource
         ```
         BUILD_OPERATOR_CONTROLLER_IMAGE=false \
           BUILD_INSTRUMENTATION_IMAGE=false \
-          IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller \
-          IMG_TAG=main-dev \
-          IMG_PULL_POLICY="" \
+          CONTROLLER_IMG_REPOSITORY=ghcr.io/dash0hq/operator-controller \
+          CONTROLLER_IMG_TAG=main-dev \
+          CONTROLLER_IMG_PULL_POLICY="" \
           INSTRUMENTATION_IMG_REPOSITORY=ghcr.io/dash0hq/instrumentation \
           INSTRUMENTATION_IMG_TAG=main-dev \
           INSTRUMENTATION_IMG_PULL_POLICY="" \
+          COLLECTOR_IMG_REPOSITORY=ghcr.io/dash0hq/collector \
+          COLLECTOR_IMG_TAG=main-dev \
+          COLLECTOR_IMG_PULL_POLICY="" \
+          CONFIGURATION_RELOADER_IMG_REPOSITORY=configuration-reloader \
+          CONFIGURATION_RELOADER_IMG_TAG=main-dev \
+          CONFIGURATION_RELOADER_IMG_PULL_POLICY="" \
           test-resources/bin/test-roundtrip-01-aum-operator-cr.sh
         ```
       * To run the scenario with the helm chart from the official remote repository and the default images referenced in
@@ -149,18 +172,24 @@ they deploy in their `AfterAll`/`AfterEach` hooks. The scripts in `test-resource
         BUILD_OPERATOR_CONTROLLER_IMAGE=false \
           BUILD_INSTRUMENTATION_IMAGE=false \
           OPERATOR_HELM_CHART=dash0-operator/dash0-operator \
-          IMG_REPOSITORY="" \
-          IMG_TAG="" \
-          IMG_PULL_POLICY="" \
+          CONTROLLER_IMG_REPOSITORY="" \
+          CONTROLLER_IMG_TAG="" \
+          CONTROLLER_IMG_PULL_POLICY="" \
           INSTRUMENTATION_IMG_REPOSITORY="" \
           INSTRUMENTATION_IMG_TAG="" \
           INSTRUMENTATION_IMG_PULL_POLICY="" \
+          COLLECTOR_IMG_REPOSITORY="" \
+          COLLECTOR_IMG_TAG=main \
+          COLLECTOR_IMG_PULL_POLICY="" \
+          CONFIGURATION_RELOADER_IMG_REPOSITORY="" \
+          CONFIGURATION_RELOADER_IMG_TAG=main \
+          CONFIGURATION_RELOADER_IMG_PULL_POLICY="" \
           test-resources/bin/test-roundtrip-01-aum-operator-cr.sh
         ```
-        Note: Unsetting parameters like `IMG_REPOSITORY` explicitly (by setting them to an empty string) will lead to
+        Note: Unsetting parameters like `CONTROLLER_IMG_REPOSITORY` explicitly (by setting them to an empty string) will lead to
         the scenario not setting those values when deploying via helm, so that the default value from the chart will
-        actually be used. Otherwise, without `IMG_REPOSITORY=""` being present, the test script will use 
-        `IMG_REPOSITORY=operator-controller` (the image built from local sources) as the default setting.
+        actually be used. Otherwise, without `CONTROLLER_IMG_REPOSITORY=""` being present, the test script will use 
+        `CONTROLLER_IMG_REPOSITORY=operator-controller` (the image built from local sources) as the default setting.
 
 ## Make Targets
 

Makefile

@@ -56,16 +56,27 @@ OPERATOR_SDK_VERSION ?= v1.34.1
 OPERATOR_HELM_CHART ?= helm-chart/dash0-operator
 
 # image repository and tag to use for building/pushing the operator image
-IMG_REPOSITORY ?= operator-controller
-IMG_TAG ?= latest
-IMG ?= $(IMG_REPOSITORY):$(IMG_TAG)
-IMG_PULL_POLICY ?= Never
+CONTROLLER_IMG_REPOSITORY ?= operator-controller
+CONTROLLER_IMG_TAG ?= latest
+CONTROLLER_IMG ?= $(CONTROLLER_IMG_REPOSITORY):$(CONTROLLER_IMG_TAG)
+CONTROLLER_IMG_PULL_POLICY ?= Never
 
 INSTRUMENTATION_IMG_REPOSITORY ?= instrumentation
 INSTRUMENTATION_IMG_TAG ?= latest
 INSTRUMENTATION_IMG ?= $(INSTRUMENTATION_IMG_REPOSITORY):$(INSTRUMENTATION_IMG_TAG)
 INSTRUMENTATION_IMG_PULL_POLICY ?= Never
 
+COLLECTOR_IMG_REPOSITORY ?= collector
+COLLECTOR_IMG_TAG ?= latest
+COLLECTOR_IMG ?= $(COLLECTOR_IMG_REPOSITORY):$(COLLECTOR_IMG_TAG)
+COLLECTOR_IMG_PULL_POLICY ?= Never
+
+CONFIGURATION_RELOADER_IMG_REPOSITORY ?= configuration-reloader
+CONFIGURATION_RELOADER_IMG_TAG ?= latest
+CONFIGURATION_RELOADER_IMG ?= $(CONFIGURATION_RELOADER_IMG_REPOSITORY):$(CONFIGURATION_RELOADER_IMG_TAG)
+CONFIGURATION_RELOADER_IMG_PULL_POLICY ?= Never
+
+
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.28.3
 
@@ -172,24 +183,30 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+	$(CONTAINER_TOOL) build -t ${CONTROLLER_IMG} .
+	$(CONTAINER_TOOL) build -t ${COLLECTOR_IMG} images/collector
+	$(CONTAINER_TOOL) build -t ${CONFIGURATION_RELOADER_IMG} images/configreloader
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
-# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# architectures. (i.e. make docker-buildx CONTROLLER_IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
 # - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
 # - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
-# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
+# - be able to push the image to your registry (i.e. if you do not set a valid value via CONTROLLER_IMG=<myregistry/image:<tag>> then the export will fail)
 # To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
 PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
 .PHONY: docker-buildx
 docker-buildx: ## Build and push docker image for the manager for cross-platform support
 	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
 	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' images/collector/Dockerfile > images/collector/Dockerfile.cross
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' images/configreloader/Dockerfile > images/configreloader/Dockerfile.cross
 	- $(CONTAINER_TOOL) buildx create --name project-v3-builder
 	$(CONTAINER_TOOL) buildx use project-v3-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${CONTROLLER_IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${COLLECTOR_IMG} -f images/collector/Dockerfile.cross images/collector
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${CONFIGURATION_RELOADER_IMG} -f images/configreloader/Dockerfile.cross images/configreloader
 	- $(CONTAINER_TOOL) buildx rm project-v3-builder
-	rm Dockerfile.cross
+	rm Dockerfile.cross images/collector/Dockerfile.cross images/configreloader/Dockerfile.cross
 
 ##@ Deployment
 
@@ -220,22 +237,28 @@ deploy-via-helm: ## Deploy the controller via helm to the K8s cluster specified
 	fi
 
 	test-resources/bin/render-templates.sh
-	helm install \
+	helm upgrade --install \
 		--namespace dash0-system \
 		--create-namespace \
-		--set operator.image.repository=$(IMG_REPOSITORY) \
-		--set operator.image.tag=$(IMG_TAG) \
-		--set operator.image.pullPolicy=$(IMG_PULL_POLICY) \
+		--set operator.image.repository=$(CONTROLLER_IMG_REPOSITORY) \
+		--set operator.image.tag=$(CONTROLLER_IMG_TAG) \
+		--set operator.image.pullPolicy=$(CONTROLLER_IMG_PULL_POLICY) \
 		--set operator.initContainerImage.repository=$(INSTRUMENTATION_IMG_REPOSITORY) \
 		--set operator.initContainerImage.tag=$(INSTRUMENTATION_IMG_TAG) \
 		--set operator.initContainerImage.pullPolicy=$(INSTRUMENTATION_IMG_PULL_POLICY) \
+		--set operator.collectorImage.repository=$(COLLECTOR_IMG_REPOSITORY) \
+		--set operator.collectorImage.tag=$(COLLECTOR_IMG_TAG) \
+		--set operator.collectorImage.pullPolicy=$(COLLECTOR_IMG_PULL_POLICY) \
+		--set operator.configurationReloaderImage.repository=$(CONFIGURATION_RELOADER_IMG_REPOSITORY) \
+		--set operator.configurationReloaderImage.tag=$(CONFIGURATION_RELOADER_IMG_TAG) \
+		--set operator.configurationReloaderImage.pullPolicy=$(CONFIGURATION_RELOADER_IMG_PULL_POLICY) \
 		--set operator.developmentMode=true \
 		dash0-operator \
 		$(OPERATOR_HELM_CHART)
 
 .PHONY: undeploy-via-helm
 undeploy-via-helm: ## Undeploy the controller via helm from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	helm uninstall --namespace dash0-system dash0-operator
+	helm uninstall --namespace dash0-system dash0-operator --timeout 30s
 	$(KUBECTL) delete ns dash0-system
 
 ##@ Build Dependencies
@@ -295,7 +318,7 @@ endif
 .PHONY: bundle
 bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q
-	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
+	cd config/manager && $(KUSTOMIZE) edit set image controller=$(CONTROLLER_IMG)
 	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS)
 	$(OPERATOR_SDK) bundle validate ./bundle
 
@@ -305,7 +328,7 @@ bundle-build: ## Build the bundle image.
 
 .PHONY: bundle-push
 bundle-push: ## Push the bundle image.
-	$(MAKE) docker-push IMG=$(BUNDLE_IMG)
+	$(MAKE) docker-push CONTROLLER_IMG=$(BUNDLE_IMG)
 
 .PHONY: opm
 OPM = $(LOCALBIN)/opm
@@ -346,4 +369,4 @@ catalog-build: opm ## Build a catalog image.
 # Push the catalog image.
 .PHONY: catalog-push
 catalog-push: ## Push a catalog image.
-	$(MAKE) docker-push IMG=$(CATALOG_IMG)
+	$(MAKE) docker-push CONTROLLER_IMG=$(CATALOG_IMG)