(WiP) Annotations for metrics index

docs/advanced-configuration.md

@@ -507,6 +507,7 @@ Manage Splunk OTel Collector Logging with these supported annotations.
 
 * Use `splunk.com/index` annotation on pod and/or namespace to tell which Splunk platform indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated.
   For example, the following command will make logs from `kube-system` namespace to be sent to `k8s_events` index: `kubectl annotate namespace kube-system splunk.com/index=k8s_events`
+* Use `splunk.com/metricsIndex` annotation on pod and/or namespace to tell which Splunk platform metrics indexes to ingest to. Works the same way as `splunk.com/index` annotation.
 * Filter logs using pod and/or namespace annotation
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested.
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `true` (default: false), set `splunk.com/include` annotation to `true` on pod and/or namespace to only include its logs from ingested. All other logs will be ignored.

helm-charts/splunk-otel-collector/templates/config/_common.tpl

@@ -138,6 +138,43 @@ k8sattributes:
     {{- end }}
 {{- end }}
 
+{{/*
+Common config for K8s attributes processor adding k8s metadata to resource attributes.
+*/}}
+{{- define "splunk-otel-collector.k8sAttributesProcessorMetrics" -}}
+k8sattributes/metrics:
+  pod_association:
+    - sources:
+      - from: resource_attribute
+        name: k8s.pod.uid
+    - sources:
+      - from: resource_attribute
+        name: k8s.pod.ip
+    - sources:
+      - from: resource_attribute
+        name: ip
+    - sources:
+      - from: connection
+    - sources:
+      - from: resource_attribute
+        name: host.name
+  extract:
+    metadata:
+      - k8s.pod.name
+      - k8s.pod.uid
+      - k8s.deployment.name
+      - k8s.namespace.name
+      - k8s.node.name
+      - k8s.pod.start_time
+    annotations:
+      - key: splunk.com/metricsIndex
+        tag_name: com.splunk.metricsIndex
+        from: namespace
+      - key: splunk.com/metricsIndex
+        tag_name: com.splunk.metricsIndex
+        from: pod
+{{- end }}
+
 {{/*
 Resource processor for logs manipulations
 */}}

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -481,6 +481,8 @@ processors:
     filter:
       node_from_env_var: K8S_NODE_NAME
 
+  {{- include "splunk-otel-collector.k8sAttributesProcessorMetrics" . | nindent 2 }}
+
   {{- if eq .Values.logsEngine "fluentd" }}
   # Move flat fluentd logs attributes to resource attributes
   groupbyattrs/logs:
@@ -575,6 +577,14 @@ processors:
         new_name: container.memory.usage
   {{- end }}
 
+  transform/metrics_index_update:
+    metric_statements:
+      - context: metric
+        statements:
+          - set(resource.attributes["com.splunk.index"], resource.attributes["com.splunk.metricsIndex"])
+          # If I do that it works fine (name of the index as a string):
+          # - set(resource.attributes["com.splunk.index"], "kube_metrics2")
+
   {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }}
   # This processor is used to remove excessive istio attributes to avoid running into the dimensions limit.
   # This configuration assumes single cluster istio deployment. If you run istio in multi-cluster scenarios or make use of the canonical service and revision labels,
@@ -794,6 +804,8 @@ service:
       receivers: [hostmetrics, kubeletstats, otlp, receiver_creator, signalfx]
       processors:
         - memory_limiter
+        - k8sattributes/metrics
+        - transform/metrics_index_update
         - batch
         {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }}
         - attributes/istio
@@ -821,6 +833,7 @@ service:
         - splunk_hec/platform_metrics
         {{- end }}
         {{- end }}
+        - logging
     {{- end }}
 
     {{- if or (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true") }}
@@ -829,6 +842,8 @@ service:
       receivers: [prometheus/agent]
       processors:
         - memory_limiter
+        - k8sattributes/metrics
+        - transform/metrics_index_update
         - batch
         - resource/add_agent_k8s
         - resourcedetection

helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl

@@ -34,6 +34,7 @@ receivers:
 # By default k8sattributes, memory_limiter and batch processors enabled.
 processors:
   {{- include "splunk-otel-collector.k8sAttributesProcessor" . | nindent 2 }}
+  {{- include "splunk-otel-collector.k8sAttributesProcessorMetrics" . | nindent 2 }}
   {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }}
   {{- if .Values.autodetect.istio }}
   {{- include "splunk-otel-collector.transformLogsProcessor" . | nindent 2 }}
@@ -172,6 +173,8 @@ service:
       receivers: [otlp, signalfx]
       processors:
         - memory_limiter
+        - k8sattributes/metrics
+        - transform/metrics_index_update
         - batch
         - resource/add_cluster_name
         {{- if .Values.extraAttributes.custom }}
@@ -229,6 +232,8 @@ service:
       receivers: [prometheus/collector]
       processors:
         - memory_limiter
+        - k8sattributes/metrics
+        - transform/metrics_index_update
         - batch
         - resource/add_collector_k8s
         - resourcedetection