Add support to create default Instrumentation for operator based auto…

…-instrumentation (#836) Add support to create default Instrumentation for operator based auto-instrumentation
signalfx · Jul 13, 2023 · 6d3ca53 · 6d3ca53
1 parent ba1f0d3
commit 6d3ca53
Show file tree

Hide file tree

Showing 14 changed files with 430 additions and 97 deletions.
diff --git a/.github/workflows/update_java_agent.yaml → .../update_instrumentation_dependencies.yaml b/.github/workflows/update_java_agent.yaml → .../update_instrumentation_dependencies.yaml
@@ -1,39 +1,42 @@
-name: Check for new java agent
+name: Check for new instrumentation versions
 
 on:
   schedule:
     - cron: "45 */4 * * *"
   workflow_dispatch:
 
 env:
-  JAVA_REPO: ghcr.io/signalfx/splunk-otel-java/splunk-otel-java
   VALUES_YAML: helm-charts/splunk-otel-collector/values.yaml
-  LATEST_API: https://api.github.com/repos/signalfx/splunk-otel-java/releases/latest
+  LATEST_API: https://api.github.com/repos/signalfx/splunk-otel-{lang}/releases/latest
 
 jobs:
-  maybe_update_java:
+  maybe_update:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        language: ['java'] # Add other languages here
     steps:
       - uses: actions/[email protected]
       - name: swizzle version
         id: swizzle_version
         run: |
-          LOCAL_VER=$(grep -A 1 $JAVA_REPO $VALUES_YAML | tail -1 | sed -e 's/.*tag: //' | tr -d '"')
-          LATEST_VER=$(curl -qs -H "Accept: application/vnd.github+json" $LATEST_API | jq -r .tag_name)
+          REPO=ghcr.io/signalfx/splunk-otel-${{ matrix.language }}/splunk-otel-${{ matrix.language }}
+          LOCAL_VER=$(grep $REPO $VALUES_YAML | awk -F: '{print $2}' | tr -d ' "')
+          LATEST_VER=$(curl -qs -H "Accept: application/vnd.github+json" $(echo $LATEST_API | sed "s/{lang}/${{ matrix.language }}/g") | jq -r .tag_name)
           echo "LATEST_VER=$LATEST_VER" >> $GITHUB_OUTPUT
-          echo "Current version is $LOCAL_VER, latest is $LATEST_VER"
+          echo "Current version of ${{ matrix.language }} is $LOCAL_VER, latest is $LATEST_VER"
           if [ "$LATEST_VER" == "$LOCAL_VER" ]; then
             echo We are already up to date. Nothing else to do.
           else
             echo 'Verifying that the image is pullable...'
             echo '(If this fails, the agent version is out of sync with ghcr version)'
-            docker pull $JAVA_REPO:$LATEST_VER
+            docker pull $REPO:$LATEST_VER
             echo 'Looks like we are good to update...'
             echo Updating to new version
             echo "NEED_UPDATE=1" >> $GITHUB_OUTPUT
-            VLINE=$(grep -n -A 1 $JAVA_REPO $VALUES_YAML | tail -1 | sed -e "s/-.*//")
-            echo sed -i "${VLINE}s/\"v.*\"/\"${LATEST_VER}\"/" $VALUES_YAML
-            sed -i "${VLINE}s/\"v.*\"/\"${LATEST_VER}\"/" $VALUES_YAML
+            VLINE=$(grep -n $REPO $VALUES_YAML | cut -f1 -d:)
+            echo sed -i "${VLINE}s|:${LOCAL_VER}|:${LATEST_VER}|" $VALUES_YAML
+            sed -i "${VLINE}s|:${LOCAL_VER}|:${LATEST_VER}|" $VALUES_YAML
             git --no-pager diff
           fi
       - name: render some templates
@@ -51,8 +54,8 @@ jobs:
         if: ${{ steps.swizzle_version.outputs.NEED_UPDATE == 1 }}
         uses: peter-evans/create-pull-request@v5
         with:
-          commit-message: update java agent version
-          title: Update Java agent version to ${{ steps.swizzle_version.outputs.LATEST_VER }}
-          body: Use the new version of the java agent
-          branch: "update-java-${{ steps.swizzle_version.outputs.LATEST_VER }}"
+          commit-message: Update ${{ matrix.language }} instrumentation version
+          title: Update ${{ matrix.language }} agent version to ${{ steps.swizzle_version.outputs.LATEST_VER }}
+          body: Use the new version of the ${{ matrix.language }} instrumentation library
+          branch: "update-${{ matrix.language }}-${{ steps.swizzle_version.outputs.LATEST_VER }}"
           base: main
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Update PodDisruptionBudgets API version to allow both `policy/v1beta1` and `policy/v1` [#835](https://github.com/signalfx/splunk-otel-collector-chart/pull/835)
 - Update clusterrole to allow collector to check for the `aws-auth` configmap in EKS clusters [#840](https://github.com/signalfx/splunk-otel-collector-chart/pull/840)
+- Add support to create default Instrumentation for operator based auto-instrumentation [#836](https://github.com/signalfx/splunk-otel-collector-chart/pull/836)
 
 ## [0.80.0] - 2023-06-27
 

diff --git a/docs/auto-instrumentation-install.md b/docs/auto-instrumentation-install.md
@@ -30,10 +30,32 @@ If a cert-manager is not available in the cluster (or other TLS certificate sour
 using `certmanager.enabled=true`. The cert-manager issues TLS certificates the operator requires. You can use the
 commands below to run these steps.
 
-_Note: In order for trace telemetry data to be properly ingested, the attribute `deployment.environment` must be on board
-the exported traces. There are two ways to set this attribute._
-- _Use the values.yaml optional `environment` configuration_
-- _Use the Instrumentation spec with the env var OTEL_RESOURCE_ATTRIBUTES_
+<details>
+<summary>Expand for extended Instrumentation configuration details</summary>
+
+- An [opentelemetry.io/v1alpha1 Instrumentation](https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md#instrumentation)
+object is used to configure the auto-instrumentation of your applications. To successfully enable instrumentation, the
+target pod must have an Instrumentation object available.
+- When `operator.enabled=true`, the helm chart deploys a
+[default](https://github.com/signalfx/splunk-otel-collector-chart/blob/main/examples/enable-operator-and-auto-instrumentation/rendered_manifests/operator/instrumentation.yaml)
+opentelemetry.io/v1alpha1 Instrumentation object.
+  - The default Instrumentation supports [AlwaysOn Profiling](https://docs.splunk.com/Observability/apm/profiling/intro-profiling.html) when `splunkObservability.profilingEnabled=true`.
+    - These environment variables will be set when auto-instrumenting applications.
+      - SPLUNK_PROFILER_ENABLED="true"
+      - SPLUNK_PROFILER_MEMORY_ENABLED="true"
+    - Example
+      - [Enable always-on profiling](../examples/enable-operator-and-auto-instrumentation/instrumentation/instrumentation-enable-profiling.yaml)
+  - Users can override the specifications of the default deployed instrumentation by setting override values under `operator.instrumentation.spec`.
+    - Examples
+      - [Add custom environment span tag](../examples/enable-operator-and-auto-instrumentation/instrumentation/instrumentation_add_custom_environment_span_tag.yaml)
+      - [Add trace sampler](../examples/enable-operator-and-auto-instrumentation/instrumentation/instrumentation-add-trace-sampler.yaml)
+      - [Enable always-on profiling partially](../examples/enable-operator-and-auto-instrumentation/instrumentation/instrumentation-enable-profiling-partially.yaml)
+- To view a deployed Instrumentation, you can use the command: `kubectl get otelinst {instrumentation_name} -o yaml`.
+- For proper ingestion of trace telemetry data, the `deployment.environment` attribute must be present in the exported traces. There are two ways to set this attribute:
+  - Use the optional `environment` configuration in `values.yaml`.
+  - Use the Instrumentation spec (`operator.instrumentation.spec.env`) with the environment variable `OTEL_RESOURCE_ATTRIBUTES`.
+
+</details>
 
 ```bash
 # Check if cert-manager is already installed, don't deploy a second cert-manager.
@@ -43,31 +65,13 @@ kubectl get pods -l app=cert-manager --all-namespaces
 helm install splunk-otel-collector -f ./my_values.yaml --set operator.enabled=true,environment=dev splunk-otel-collector-chart/splunk-otel-collector
 ```
 
-### 2. Deploy the opentelemetry.io/v1alpha1 Instrumentation
-
-The [Instrumentation](https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md#instrumentation)
-object is used to configure how your applications will be auto-instrumented. An Instrumentation object must be
-available to the target pod being auto-instrumented in order for instrumentation to be successful.
-
-For Splunk Instrumentation configurations, please see or use this [instrumentation.yaml](../examples/enable-operator-and-auto-instrumentation/instrumentation.yaml)
-in our examples directory. The Splunk Instrumentation object should be installed in the namespace target applications
-resides in. For example, for instrumenting applications in a namespace called `my-app-namespace` the Instrumentation
-object could be installed with the following command.
-
-```bash
-# Install
-kubectl apply -f instrumentation.yaml -n my-app-namespace
-# Check the current deployed values
-kubectl get otelinst -o yaml -n my-app-namespace
-```
-
-### 3. Verify all the OpenTelemetry resources (collector, operator, webhook, instrumentation) are deployed successfully
+### 2. Verify all the OpenTelemetry resources (collector, operator, webhook, instrumentation) are deployed successfully
 
 <details>
 <summary>Expand for sample output to verify against</summary>
 
 ```bash
-kubectl get pods -n default
+kubectl get pods
 # NAME                                                            READY   STATUS
 # splunk-otel-collector-agent-lfthw                               2/2     Running
 # splunk-otel-collector-cert-manager-6b9fb8b95f-2lmv4             1/1     Running
@@ -76,19 +80,19 @@ kubectl get pods -n default
 # splunk-otel-collector-k8s-cluster-receiver-856f5fbcf9-pqkwg     1/1     Running
 # splunk-otel-collector-opentelemetry-operator-56c4ddb4db-zcjgh   2/2     Running
 
-kubectl get mutatingwebhookconfiguration.admissionregistration.k8s.io -n default
+kubectl get mutatingwebhookconfiguration.admissionregistration.k8s.io
 # NAME                                      WEBHOOKS   AGE
 # splunk-otel-collector-cert-manager-webhook              1          14m
 # splunk-otel-collector-opentelemetry-operator-mutation   3          14m
 
-kubectl get otelinst -n my-app-namespace
-# NAME                          AGE   ENDPOINT
-# splunk-instrumentation        3m   http://$(SPLUNK_OTEL_AGENT):4317
+kubectl get otelinst
+# NAME                    AGE   ENDPOINT
+# splunk-otel-collector   3s    http://$(SPLUNK_OTEL_AGENT):4317
 ```
 
 </details>
 
-### 4. Instrument application by setting an annotation
+### 3. Instrument application by setting an annotation
 
 An _instrumentation.opentelemetry.io/inject-{instrumentation_library}_ annotation can be added to the following:
 - Namespace: All pods within that namespace will be instrumented.
@@ -106,7 +110,7 @@ The instrumentation annotations can have the following values:
 - "my-other-namespace/my-instrumentation" - name and namespace of Instrumentation CR instance in another namespace to use.
 - "false" - do not inject.
 
-### 5. Check out the results at [Splunk Observability APM](https://app.us1.signalfx.com/#/apm)
+### 4. Check out the results at [Splunk Observability APM](https://app.us1.signalfx.com/#/apm)
 
 The trace and metrics data should populate the APM dashboard.To better
 visualize this example as a whole, we have also included an image

diff --git a/examples/enable-operator-and-auto-instrumentation/README.md b/examples/enable-operator-and-auto-instrumentation/README.md
@@ -28,23 +28,11 @@ These helm install commands will deploy the chart to the current namespace for t
 # Check if a cert-manager is already installed by looking for cert-manager pods.
 kubectl get pods -l app=cert-manager --all-namespaces
 
-# If cert-manager is not deployed, make sure to add certmanager.enabled=true to the list of values to set
-helm install splunk-otel-collector -f ./my_values.yaml --set operator.enabled=true,environment=dev splunk-otel-collector-chart/splunk-otel-collector
+# If cert-manager is deployed, make sure to remove certmanager.enabled=true to the list of values to set
+helm install splunk-otel-collector -f ./my_values.yaml --set operator.enabled=true,certmanager.enabled=true,environment=dev splunk-otel-collector-chart/splunk-otel-collector
 ```
 
-#### 2.2 Deploy the opentelemetry.io/v1alpha1 Instrumentation
-
-The Instrumentation object is a spec to configure what instrumentation libraries to use in the target namespace
-to use for instrumentation. This Instrumentation will be used to know how to instrument the spring-clinic applications
-in the spring-petclinic namespace.
-
-```bash
-curl https://raw.githubusercontent.com/signalfx/splunk-otel-collector-chart/main/examples/enable-operator-and-auto-instrumentation/instrumentation.yaml | kubectl apply -n spring-petclinic -f -
-# Note: You can use this command if you want to use a local custom instrumentation file.
-# kubectl apply -f examples/enable-operator-and-auto-instrumentation/instrumentation.yaml -n spring-petclinic
-```
-
-#### 2.3 Verify all the OpenTelemetry resources (collector, operator, webhook, instrumentation) are deployed successfully
+#### 2.2 Verify all the OpenTelemetry resources (collector, operator, webhook, instrumentation) are deployed successfully
 
 <details>
 <summary>Expand for kubectl commands to run and output</summary>
@@ -79,7 +67,7 @@ kubectl get otelinst -n spring-petclinic
 
 </details>
 
-#### 2.4 Instrument application by setting an annotation
+#### 2.3 Instrument application by setting an annotation
 
 The required annotation can exist on pods or namespaces. Depending on the variety of applications you are instrumenting,
 you may want to use either of these options. We show how to annotate pods and namespaces below.
@@ -129,6 +117,7 @@ kubectl patch deployment customers-service -p '{"spec": {"template":{"metadata":
 kubectl patch deployment vets-service -p '{"spec": {"template":{"metadata":{"annotations":{"instrumentation.opentelemetry.io/inject-java":"false"}}}} }' -n spring-petclinic
 kubectl patch deployment discovery-server -p '{"spec": {"template":{"metadata":{"annotations":{"instrumentation.opentelemetry.io/inject-java":"false"}}}} }' -n spring-petclinic
 kubectl patch deployment visits-service -p '{"spec": {"template":{"metadata":{"annotations":{"instrumentation.opentelemetry.io/inject-java":"false"}}}} }' -n spring-petclinic
+
 ```
 </details>
 
@@ -178,7 +167,7 @@ kubectl describe pod spring-petclinic-9d5bc5fff-5r5gr  -n spring-petclinic
 
 </details>
 
-#### 2.5 Check out the results at [Splunk Observability APM](https://app.us1.signalfx.com/#/apm)
+#### 2.4 Check out the results at [Splunk Observability APM](https://app.us1.signalfx.com/#/apm)
 
 <details>
 <summary> Expand for visual results </summary>

diff --git a/examples/enable-operator-and-auto-instrumentation/instrumentation.yaml b/examples/enable-operator-and-auto-instrumentation/instrumentation.yaml
diff --git a/...auto-instrumentation/instrumentation/instrumentation-add-custom-environment-span-tag.yaml b/...auto-instrumentation/instrumentation/instrumentation-add-custom-environment-span-tag.yaml
@@ -0,0 +1,43 @@
+---
+# Source: splunk-otel-collector/templates/operator/instrumentation.yaml
+# Description: Set a custom value (dev) for the deployment.environment tag on exported spans
+# Input values.yaml:
+# splunkObservability:
+#   accessToken: CHANGEME
+#   realm: us0
+# clusterName: CHANGEME
+# environment: CHANGEME
+# operator:
+#   enabled: true
+#   instrumentation:
+#     spec:
+#       env:
+#         - name: OTEL_RESOURCE_ATTRIBUTES
+#           value: "deployment.environment=dev"
+# certmanager:
+#   enabled: true
+# Output instrumentation.yaml:
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: splunk-instrumentation
+spec:
+    dotnet:
+      env:
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: http://$(SPLUNK_OTEL_AGENT):4318
+    env:
+    - name: OTEL_RESOURCE_ATTRIBUTES
+      value: deployment.environment=dev
+    exporter:
+      endpoint: http://$(SPLUNK_OTEL_AGENT):4317
+    java:
+      image: ghcr.io/signalfx/splunk-otel-java/splunk-otel-java:v1.25.0
+    propagators:
+    - tracecontext
+    - baggage
+    - b3
+    python:
+      env:
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: http://$(SPLUNK_OTEL_AGENT):4318
diff --git a/...-operator-and-auto-instrumentation/instrumentation/instrumentation-add-trace-sampler.yaml b/...-operator-and-auto-instrumentation/instrumentation/instrumentation-add-trace-sampler.yaml
@@ -0,0 +1,49 @@
+---
+# Source: splunk-otel-collector/templates/operator/instrumentation.yaml
+# Description: Add a trace sampler to control how many traces are exported
+# Input values.yaml:
+# splunkObservability:
+#   accessToken: CHANGEME
+#   realm: us0
+# clusterName: CHANGEME
+# environment: CHANGEME
+# operator:
+#   enabled: true
+#   instrumentation:
+#     spec:
+#       sampler:
+#         type: traceidratio
+#         argument: "0.95"
+# certmanager:
+#   enabled: true
+# Output instrumentation.yaml:
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: splunk-instrumentation
+spec:
+    dotnet:
+      env:
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: http://$(SPLUNK_OTEL_AGENT):4318
+    env:
+    - name: SPLUNK_OTEL_AGENT
+      valueFrom:
+        fieldRef:
+          apiVersion: v1
+          fieldPath: status.hostIP
+    exporter:
+      endpoint: http://$(SPLUNK_OTEL_AGENT):4317
+    java:
+      image: ghcr.io/signalfx/splunk-otel-java/splunk-otel-java:v1.25.0
+    propagators:
+    - tracecontext
+    - baggage
+    - b3
+    python:
+      env:
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: http://$(SPLUNK_OTEL_AGENT):4318
+    sampler:
+      argument: "0.95"
+      type: traceidratio