From 95d45bb91672780a52487b985755a19bae95088a Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 11:32:19 +0200 Subject: [PATCH 01/23] test --- helm-charts/splunk-otel-collector/values.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 316070bf78..7d0481ffc0 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -14,7 +14,7 @@ fullnameOverride: "" # log as "k8s.cluster.name" attribute. ################################################################################ -clusterName: "" +clusterName: "test_cluster" ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. @@ -27,17 +27,17 @@ splunkPlatform: # to. e.g. "http://X.X.X.X:8088/services/collector/event". Setting this parameter # enables Splunk Platform as a destination. Use the /services/collector/event # endpoint for proper extraction of fields. - endpoint: "" + endpoint: "http://http://10.202.12.84:8088/services/collector/event" # Required for Splunk Enterprise/Cloud (if `endpoint` is specified). Splunk # Alternatively the token can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret # HTTP Event Collector token. - token: "" + token: "00000000-0000-0000-0000-000000000000" # Name of the Splunk event type index targeted. Required when ingesting logs to Splunk Platform. - index: "main" + index: "kube" # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform. - metricsIndex: "" + metricsIndex: "kube_mterics" # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform. tracesIndex: "" # Optional. Default value for `source` field. @@ -55,7 +55,7 @@ splunkPlatform: idleConnTimeout: 10s # Whether to skip checking the certificate of the HEC endpoint when sending # data over HTTPS. - insecureSkipVerify: false + insecureSkipVerify: true # The PEM-format CA certificate for this client. # Alternatively the clientCert, clientKey and caFile can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret @@ -75,7 +75,7 @@ splunkPlatform: # Splunk Platform. Only logs collection is enabled by default. logsEnabled: true # If you enable metrics collection, make sure that `metricsIndex` is provided as well. - metricsEnabled: false + metricsEnabled: true # If you enable traces collection, make sure that `tracesIndex` is provided as well. tracesEnabled: false # Field name conventions to use. (Only for those who are migrating from Splunk Connect for Kubernetes helm chart) From f21810a6491bc85c1dec92153444c6fb4b33d110 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 11:39:38 +0200 Subject: [PATCH 02/23] tedt --- .../splunk-otel-collector/templates/config/_common.tpl | 6 ++++++ .../splunk-otel-collector/templates/config/_otel-agent.tpl | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl index df8974180a..7072ca7232 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_common.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_common.tpl @@ -128,6 +128,12 @@ k8sattributes: - key: splunk.com/index tag_name: com.splunk.index from: pod + - key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex + from: namespace + - key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex + from: pod {{- include "splunk-otel-collector.addExtraAnnotations" . | nindent 6 }} {{- if or .Values.extraAttributes.podLabels .Values.extraAttributes.fromLabels }} labels: diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index 7dbdfd3185..a854dd1f29 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -448,7 +448,10 @@ receivers: value: 'EXPR("kube:journald:"+body._SYSTEMD_UNIT)' - type: add field: resource["com.splunk.index"] - value: {{ $.Values.logsCollection.journald.index | default $.Values.splunkPlatform.index }} + value: "kube2" + - type: add + field: resource["com.splunk.metricsIndex"] + value: {{ $.Values.logsCollection.journald.metricsIndex | default $.Values.splunkPlatform.metricsIndex }} - type: add field: resource["host.name"] value: 'EXPR(env("K8S_NODE_NAME"))' From a53e3b0f58683257a360c56a56ee37f69bf418fa Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 11:50:48 +0200 Subject: [PATCH 03/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 7d0481ffc0..82cae1da79 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -27,7 +27,7 @@ splunkPlatform: # to. e.g. "http://X.X.X.X:8088/services/collector/event". Setting this parameter # enables Splunk Platform as a destination. Use the /services/collector/event # endpoint for proper extraction of fields. - endpoint: "http://http://10.202.12.84:8088/services/collector/event" + endpoint: "http://10.202.12.84:8088/services/collector/event" # Required for Splunk Enterprise/Cloud (if `endpoint` is specified). Splunk # Alternatively the token can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret From ad4e7a99d457cf15ed6ff3e0f3d6e95dbb43f8c0 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 11:54:38 +0200 Subject: [PATCH 04/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 82cae1da79..dafa79c67e 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -27,7 +27,7 @@ splunkPlatform: # to. e.g. "http://X.X.X.X:8088/services/collector/event". Setting this parameter # enables Splunk Platform as a destination. Use the /services/collector/event # endpoint for proper extraction of fields. - endpoint: "http://10.202.12.84:8088/services/collector/event" + endpoint: "https://10.202.12.84:8088/services/collector/event" # Required for Splunk Enterprise/Cloud (if `endpoint` is specified). Splunk # Alternatively the token can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret From dd3d0256c7e680a438bb4284a4fc9a80ce57ac3d Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:03:21 +0200 Subject: [PATCH 05/23] . --- helm-charts/splunk-otel-collector/Chart.yaml | 62 +++++++++---------- .../templates/config/_otel-agent.tpl | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/helm-charts/splunk-otel-collector/Chart.yaml b/helm-charts/splunk-otel-collector/Chart.yaml index 0ef38e0a58..7a74701024 100644 --- a/helm-charts/splunk-otel-collector/Chart.yaml +++ b/helm-charts/splunk-otel-collector/Chart.yaml @@ -1,33 +1,33 @@ apiVersion: v2 name: splunk-otel-collector -version: 0.86.1 -appVersion: 0.86.0 -description: Splunk OpenTelemetry Collector for Kubernetes -icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png -type: application -keywords: -- splunk -- opentelemetry -- observability -- logs -- traces -- metrics -- apm -maintainers: -- name: dmitryax -- name: jvoravong -- name: emaderer -dependencies: - # Subchart Notes: - # - Avoid uppercase letters in names/aliases, they cause install failure due to subchart resource naming - # - Avoid hyphen characters in names/aliases, they introduce template rendering complications (https://github.com/helm/helm/issues/2192) - - name: cert-manager - version: v1.13.1 - alias: certmanager - repository: https://charts.jetstack.io - condition: certmanager.enabled - - name: opentelemetry-operator - version: 0.39.1 - alias: operator - repository: https://open-telemetry.github.io/opentelemetry-helm-charts - condition: operator.enabled +# version: 0.86.1 +# appVersion: 0.86.0 +# description: Splunk OpenTelemetry Collector for Kubernetes +# icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png +# type: application +# keywords: +# - splunk +# - opentelemetry +# - observability +# - logs +# - traces +# - metrics +# - apm +# maintainers: +# - name: dmitryax +# - name: jvoravong +# - name: emaderer +# dependencies: +# # Subchart Notes: +# # - Avoid uppercase letters in names/aliases, they cause install failure due to subchart resource naming +# # - Avoid hyphen characters in names/aliases, they introduce template rendering complications (https://github.com/helm/helm/issues/2192) +# - name: cert-manager +# version: v1.13.1 +# alias: certmanager +# repository: https://charts.jetstack.io +# condition: certmanager.enabled +# - name: opentelemetry-operator +# version: 0.39.1 +# alias: operator +# repository: https://open-telemetry.github.io/opentelemetry-helm-charts +# condition: operator.enabled diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index a854dd1f29..016bc8720c 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -448,7 +448,7 @@ receivers: value: 'EXPR("kube:journald:"+body._SYSTEMD_UNIT)' - type: add field: resource["com.splunk.index"] - value: "kube2" + value: {{ $.Values.logsCollection.journald.index | default $.Values.splunkPlatform.index }} - type: add field: resource["com.splunk.metricsIndex"] value: {{ $.Values.logsCollection.journald.metricsIndex | default $.Values.splunkPlatform.metricsIndex }} From 45eba1ba8e35285a6cedcf80dac9f2209aaaa78d Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:04:33 +0200 Subject: [PATCH 06/23] . --- helm-charts/splunk-otel-collector/Chart.yaml | 62 ++++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/helm-charts/splunk-otel-collector/Chart.yaml b/helm-charts/splunk-otel-collector/Chart.yaml index 7a74701024..0ef38e0a58 100644 --- a/helm-charts/splunk-otel-collector/Chart.yaml +++ b/helm-charts/splunk-otel-collector/Chart.yaml @@ -1,33 +1,33 @@ apiVersion: v2 name: splunk-otel-collector -# version: 0.86.1 -# appVersion: 0.86.0 -# description: Splunk OpenTelemetry Collector for Kubernetes -# icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png -# type: application -# keywords: -# - splunk -# - opentelemetry -# - observability -# - logs -# - traces -# - metrics -# - apm -# maintainers: -# - name: dmitryax -# - name: jvoravong -# - name: emaderer -# dependencies: -# # Subchart Notes: -# # - Avoid uppercase letters in names/aliases, they cause install failure due to subchart resource naming -# # - Avoid hyphen characters in names/aliases, they introduce template rendering complications (https://github.com/helm/helm/issues/2192) -# - name: cert-manager -# version: v1.13.1 -# alias: certmanager -# repository: https://charts.jetstack.io -# condition: certmanager.enabled -# - name: opentelemetry-operator -# version: 0.39.1 -# alias: operator -# repository: https://open-telemetry.github.io/opentelemetry-helm-charts -# condition: operator.enabled +version: 0.86.1 +appVersion: 0.86.0 +description: Splunk OpenTelemetry Collector for Kubernetes +icon: https://github.com/signalfx/splunk-otel-collector-chart/tree/main/splunk.png +type: application +keywords: +- splunk +- opentelemetry +- observability +- logs +- traces +- metrics +- apm +maintainers: +- name: dmitryax +- name: jvoravong +- name: emaderer +dependencies: + # Subchart Notes: + # - Avoid uppercase letters in names/aliases, they cause install failure due to subchart resource naming + # - Avoid hyphen characters in names/aliases, they introduce template rendering complications (https://github.com/helm/helm/issues/2192) + - name: cert-manager + version: v1.13.1 + alias: certmanager + repository: https://charts.jetstack.io + condition: certmanager.enabled + - name: opentelemetry-operator + version: 0.39.1 + alias: operator + repository: https://open-telemetry.github.io/opentelemetry-helm-charts + condition: operator.enabled From f495469e0a4bb6e2ce4b6f6dc70bf825f14fcf59 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:23:45 +0200 Subject: [PATCH 07/23] . --- helm-charts/splunk-otel-collector/values.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index dafa79c67e..e9fdb53bc1 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -254,8 +254,10 @@ extraAttributes: # and added as extra attributes to the telemetry in the following format: # k8s..annotations.: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. - fromAnnotations: [] - + fromAnnotations: + - key: splunk.com/metricsIndex + tag_name: metricsIndex + # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. custom: [] From 67ef7c6b5343d0f389181b5414533a890592cc09 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:43:44 +0200 Subject: [PATCH 08/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index e9fdb53bc1..ed07aa756b 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -37,7 +37,7 @@ splunkPlatform: # Name of the Splunk event type index targeted. Required when ingesting logs to Splunk Platform. index: "kube" # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform. - metricsIndex: "kube_mterics" + metricsIndex: "kube2_metrics" # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform. tracesIndex: "" # Optional. Default value for `source` field. From 92e57925aca6a9ca28e791e9d20bd347b5f3dc13 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:50:44 +0200 Subject: [PATCH 09/23] . --- helm-charts/splunk-otel-collector/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index ed07aa756b..b0079f50f1 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -37,7 +37,7 @@ splunkPlatform: # Name of the Splunk event type index targeted. Required when ingesting logs to Splunk Platform. index: "kube" # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform. - metricsIndex: "kube2_metrics" + metricsIndex: "kube_mterics" # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform. tracesIndex: "" # Optional. Default value for `source` field. @@ -257,6 +257,7 @@ extraAttributes: fromAnnotations: - key: splunk.com/metricsIndex tag_name: metricsIndex + from: kube-system # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. From 43417e74792fc3e91b218199155393a7d7f72441 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 13:53:29 +0200 Subject: [PATCH 10/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index b0079f50f1..8e1244494e 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -257,7 +257,7 @@ extraAttributes: fromAnnotations: - key: splunk.com/metricsIndex tag_name: metricsIndex - from: kube-system + from: namespace # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. From b3a62ce4e9efa07f78c7cc2ef7dc0d5bba5771aa Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:08:01 +0200 Subject: [PATCH 11/23] . --- helm-charts/splunk-otel-collector/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 8e1244494e..0eea365502 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -254,9 +254,9 @@ extraAttributes: # and added as extra attributes to the telemetry in the following format: # k8s..annotations.: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. - fromAnnotations: + fromAnnotations: - key: splunk.com/metricsIndex - tag_name: metricsIndex + tag_name: com.splunk.metricsIndex from: namespace # List of hardcoded key/value pairs that will be added as attributes to From ac4368755f8ba60d2d08aac3aa3f977cc404f5d9 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:13:32 +0200 Subject: [PATCH 12/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 0eea365502..f7e545b107 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -256,7 +256,7 @@ extraAttributes: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: - key: splunk.com/metricsIndex - tag_name: com.splunk.metricsIndex + tag_name: com.splunkPlatform.metricsIndex from: namespace # List of hardcoded key/value pairs that will be added as attributes to From e26b856ac6d4147706b06a88e49518e679893023 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:18:57 +0200 Subject: [PATCH 13/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index f7e545b107..e99cd38593 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -256,7 +256,7 @@ extraAttributes: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: - key: splunk.com/metricsIndex - tag_name: com.splunkPlatform.metricsIndex + tag_name: metricsIndex from: namespace # List of hardcoded key/value pairs that will be added as attributes to From ddae24bd8c61110bd8414ffa5de65428bcefe659 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:27:40 +0200 Subject: [PATCH 14/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index e99cd38593..a3e6b6171f 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -256,7 +256,7 @@ extraAttributes: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: - key: splunk.com/metricsIndex - tag_name: metricsIndex + tag_name: index from: namespace # List of hardcoded key/value pairs that will be added as attributes to From 74602cba28630009b132c21d7e8f3b26f1e73947 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:32:16 +0200 Subject: [PATCH 15/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index a3e6b6171f..e99cd38593 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -256,7 +256,7 @@ extraAttributes: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: - key: splunk.com/metricsIndex - tag_name: index + tag_name: metricsIndex from: namespace # List of hardcoded key/value pairs that will be added as attributes to From 107796fb4a35c7dd5b5633c3b09553168683ee5d Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Wed, 25 Oct 2023 14:43:16 +0200 Subject: [PATCH 16/23] . --- helm-charts/splunk-otel-collector/values.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index e99cd38593..4febaa7214 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -255,10 +255,13 @@ extraAttributes: # k8s..annotations.: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: - - key: splunk.com/metricsIndex - tag_name: metricsIndex - from: namespace + # - key: splunk.com/metricsIndex + # tag_name: metricsIndex + # from: namespace + - from: namespace + key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. custom: [] From 55e33fd9d786bbeebacbacf90be4e617a6506b6b Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Thu, 26 Oct 2023 09:11:31 +0200 Subject: [PATCH 17/23] . --- helm-charts/splunk-otel-collector/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 4febaa7214..4ad3107265 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -37,7 +37,7 @@ splunkPlatform: # Name of the Splunk event type index targeted. Required when ingesting logs to Splunk Platform. index: "kube" # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform. - metricsIndex: "kube_mterics" + metricsIndex: "kube_metrics" # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform. tracesIndex: "" # Optional. Default value for `source` field. From 37caebca48aba1f23b80370f5f5228aff805a57b Mon Sep 17 00:00:00 2001 From: sbylica Date: Thu, 26 Oct 2023 09:53:27 +0000 Subject: [PATCH 18/23] . --- helm-charts/splunk-otel-collector/values.yaml | 22 +++++------- my_values.yaml | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 my_values.yaml diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index 4ad3107265..c5c1038f9f 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -14,7 +14,7 @@ fullnameOverride: "" # log as "k8s.cluster.name" attribute. ################################################################################ -clusterName: "test_cluster" +clusterName: "" ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. @@ -27,17 +27,17 @@ splunkPlatform: # to. e.g. "http://X.X.X.X:8088/services/collector/event". Setting this parameter # enables Splunk Platform as a destination. Use the /services/collector/event # endpoint for proper extraction of fields. - endpoint: "https://10.202.12.84:8088/services/collector/event" + endpoint: "" # Required for Splunk Enterprise/Cloud (if `endpoint` is specified). Splunk # Alternatively the token can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret # HTTP Event Collector token. - token: "00000000-0000-0000-0000-000000000000" + token: "" # Name of the Splunk event type index targeted. Required when ingesting logs to Splunk Platform. - index: "kube" + index: "main" # Name of the Splunk metric type index targeted. Required when ingesting metrics to Splunk Platform. - metricsIndex: "kube_metrics" + metricsIndex: "" # Name of the Splunk event type index targeted. Required when ingesting traces to Splunk Platform. tracesIndex: "" # Optional. Default value for `source` field. @@ -254,14 +254,8 @@ extraAttributes: # and added as extra attributes to the telemetry in the following format: # k8s..annotations.: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. - fromAnnotations: - # - key: splunk.com/metricsIndex - # tag_name: metricsIndex - # from: namespace - - - from: namespace - key: splunk.com/metricsIndex - tag_name: com.splunk.metricsIndex + fromAnnotations: [] + # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. custom: [] @@ -319,7 +313,7 @@ agent: caFile: "" # Specifies whether the etcd's TLS cert will be verified. If set to false, a CA certificate must be made # available as part of the etcd secret to verify the TLS cert with. - skipVerify: true + skipVerify: false proxy: # Specifies whether to collect proxy metrics. enabled: true diff --git a/my_values.yaml b/my_values.yaml new file mode 100644 index 0000000000..225bbb044e --- /dev/null +++ b/my_values.yaml @@ -0,0 +1,35 @@ +clusterName: "test_cluster" + +splunkPlatform: + endpoint: "https://10.202.12.84:8088/services/collector/event" + token: "00000000-0000-0000-0000-000000000000" + + index: "kube" + metricsIndex: "kube_metrics" + +agent: + enabled: true + + config: + exporters: + logging: + loglevel: debug + + controlPlaneMetrics: + apiserver: + # Specifies whether to collect apiserver metrics. + enabled: true + controllerManager: + # Specifies whether to collect controller manager metrics. + enabled: true + coredns: + # Specifies whether to collect coredns metrics. + enabled: true + etcd: + skipVerify: false + +extraAttributes: + fromAnnotations: + - from: namespace + key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex From ac5409308cd68e830b1b853a8df9de74dfc7a4fa Mon Sep 17 00:00:00 2001 From: sbylica Date: Thu, 26 Oct 2023 09:55:42 +0000 Subject: [PATCH 19/23] . --- helm-charts/splunk-otel-collector/values.yaml | 8 ++++---- my_values.yaml | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/helm-charts/splunk-otel-collector/values.yaml b/helm-charts/splunk-otel-collector/values.yaml index c5c1038f9f..316070bf78 100644 --- a/helm-charts/splunk-otel-collector/values.yaml +++ b/helm-charts/splunk-otel-collector/values.yaml @@ -55,7 +55,7 @@ splunkPlatform: idleConnTimeout: 10s # Whether to skip checking the certificate of the HEC endpoint when sending # data over HTTPS. - insecureSkipVerify: true + insecureSkipVerify: false # The PEM-format CA certificate for this client. # Alternatively the clientCert, clientKey and caFile can be provided as a secret. # Refer to https://github.com/signalfx/splunk-otel-collector-chart/blob/main/docs/advanced-configuration.md#provide-tokens-as-a-secret @@ -75,7 +75,7 @@ splunkPlatform: # Splunk Platform. Only logs collection is enabled by default. logsEnabled: true # If you enable metrics collection, make sure that `metricsIndex` is provided as well. - metricsEnabled: true + metricsEnabled: false # If you enable traces collection, make sure that `tracesIndex` is provided as well. tracesEnabled: false # Field name conventions to use. (Only for those who are migrating from Splunk Connect for Kubernetes helm chart) @@ -255,7 +255,7 @@ extraAttributes: # k8s..annotations.: # fromAnnotations uses the same extraction rules as fromLabels option so refer examples from the fromLabels option. fromAnnotations: [] - + # List of hardcoded key/value pairs that will be added as attributes to # all the telemetry. custom: [] @@ -313,7 +313,7 @@ agent: caFile: "" # Specifies whether the etcd's TLS cert will be verified. If set to false, a CA certificate must be made # available as part of the etcd secret to verify the TLS cert with. - skipVerify: false + skipVerify: true proxy: # Specifies whether to collect proxy metrics. enabled: true diff --git a/my_values.yaml b/my_values.yaml index 225bbb044e..6d926b995a 100644 --- a/my_values.yaml +++ b/my_values.yaml @@ -6,6 +6,7 @@ splunkPlatform: index: "kube" metricsIndex: "kube_metrics" + insecureSkipVerify: true agent: enabled: true From 2956f05bdca719dc685c80b4a0a0c67c652a4a3d Mon Sep 17 00:00:00 2001 From: sbylica Date: Thu, 26 Oct 2023 09:55:46 +0000 Subject: [PATCH 20/23] . --- my_values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/my_values.yaml b/my_values.yaml index 6d926b995a..5a3e0bdc79 100644 --- a/my_values.yaml +++ b/my_values.yaml @@ -7,6 +7,7 @@ splunkPlatform: index: "kube" metricsIndex: "kube_metrics" insecureSkipVerify: true + metricsEnabled: true agent: enabled: true From f53f59a270b131711f3bf197ae9c1e32d3483311 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Thu, 9 Nov 2023 12:00:51 +0000 Subject: [PATCH 21/23] . --- .../templates/config/_common.tpl | 43 ++++++++++++++++--- .../templates/config/_otel-agent.tpl | 15 +++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl index 7072ca7232..ffc3b4fa8c 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_common.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_common.tpl @@ -128,12 +128,6 @@ k8sattributes: - key: splunk.com/index tag_name: com.splunk.index from: pod - - key: splunk.com/metricsIndex - tag_name: com.splunk.metricsIndex - from: namespace - - key: splunk.com/metricsIndex - tag_name: com.splunk.metricsIndex - from: pod {{- include "splunk-otel-collector.addExtraAnnotations" . | nindent 6 }} {{- if or .Values.extraAttributes.podLabels .Values.extraAttributes.fromLabels }} labels: @@ -144,6 +138,43 @@ k8sattributes: {{- end }} {{- end }} +{{/* +Common config for K8s attributes processor adding k8s metadata to resource attributes. +*/}} +{{- define "splunk-otel-collector.k8sAttributesProcessorMetrics" -}} +k8sattributes/metrics: + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: ip + - sources: + - from: connection + - sources: + - from: resource_attribute + name: host.name + extract: + metadata: + - k8s.pod.name + - k8s.pod.uid + - k8s.deployment.name + - k8s.namespace.name + - k8s.node.name + - k8s.pod.start_time + annotations: + - key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex + from: namespace + - key: splunk.com/metricsIndex + tag_name: com.splunk.metricsIndex + from: pod +{{- end }} + {{/* Resource processor for logs manipulations */}} diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index 016bc8720c..c7c7648905 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -484,6 +484,10 @@ processors: filter: node_from_env_var: K8S_NODE_NAME + {{- include "splunk-otel-collector.k8sAttributesProcessorMetrics" . | nindent 2 }} + filter: + node_from_env_var: K8S_NODE_NAME + {{- if eq .Values.logsEngine "fluentd" }} # Move flat fluentd logs attributes to resource attributes groupbyattrs/logs: @@ -578,6 +582,14 @@ processors: new_name: container.memory.usage {{- end }} + transform/metricsindexchange: + metric_statements: + - context: metric + statements: + - set(resource.attributes["k8s.pod.about"], "nothing") + - set(resource.attributes["k8s.pod.about"], resource.attributes["com.splunk.metricsIndex"]) + - set(resource.attributes["com.splunk.index"], resource.attributes["com.splunk.metricsIndex"]) + {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }} # This processor is used to remove excessive istio attributes to avoid running into the dimensions limit. # This configuration assumes single cluster istio deployment. If you run istio in multi-cluster scenarios or make use of the canonical service and revision labels, @@ -797,6 +809,7 @@ service: receivers: [hostmetrics, kubeletstats, otlp, receiver_creator, signalfx] processors: - memory_limiter + - k8sattributes/metrics - batch {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }} - attributes/istio @@ -824,6 +837,7 @@ service: - splunk_hec/platform_metrics {{- end }} {{- end }} + - logging {{- end }} {{- if or (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") (eq (include "splunk-otel-collector.platformMetricsEnabled" .) "true") }} @@ -832,6 +846,7 @@ service: receivers: [prometheus/agent] processors: - memory_limiter + - k8sattributes/metrics - batch - resource/add_agent_k8s - resourcedetection From 74fb5b99257848c41256a035647ed2fdd2dd4641 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Thu, 9 Nov 2023 13:30:01 +0000 Subject: [PATCH 22/23] . --- docs/advanced-configuration.md | 1 + .../templates/config/_otel-agent.tpl | 13 +++++-------- .../templates/config/_otel-collector.tpl | 5 +++++ my_values.yaml | 8 +------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md index 251b06a7c5..9423d5dcfa 100644 --- a/docs/advanced-configuration.md +++ b/docs/advanced-configuration.md @@ -507,6 +507,7 @@ Manage Splunk OTel Collector Logging with these supported annotations. * Use `splunk.com/index` annotation on pod and/or namespace to tell which Splunk platform indexes to ingest to. Pod annotation will take precedence over namespace annotation when both are annotated. For example, the following command will make logs from `kube-system` namespace to be sent to `k8s_events` index: `kubectl annotate namespace kube-system splunk.com/index=k8s_events` +* Use `splunk.com/metricsIndex` annotation on pod and/or namespace to tell which Splunk platform metrics indexes to ingest to. Works the same way as `splunk.com/index` annotation. * Filter logs using pod and/or namespace annotation * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested. * If `logsCollection.containers.useSplunkIncludeAnnotation` is `true` (default: false), set `splunk.com/include` annotation to `true` on pod and/or namespace to only include its logs from ingested. All other logs will be ignored. diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index c7c7648905..9fcbdcade8 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -449,9 +449,6 @@ receivers: - type: add field: resource["com.splunk.index"] value: {{ $.Values.logsCollection.journald.index | default $.Values.splunkPlatform.index }} - - type: add - field: resource["com.splunk.metricsIndex"] - value: {{ $.Values.logsCollection.journald.metricsIndex | default $.Values.splunkPlatform.metricsIndex }} - type: add field: resource["host.name"] value: 'EXPR(env("K8S_NODE_NAME"))' @@ -485,8 +482,6 @@ processors: node_from_env_var: K8S_NODE_NAME {{- include "splunk-otel-collector.k8sAttributesProcessorMetrics" . | nindent 2 }} - filter: - node_from_env_var: K8S_NODE_NAME {{- if eq .Values.logsEngine "fluentd" }} # Move flat fluentd logs attributes to resource attributes @@ -582,13 +577,13 @@ processors: new_name: container.memory.usage {{- end }} - transform/metricsindexchange: + transform/metrics_index_update: metric_statements: - context: metric statements: - - set(resource.attributes["k8s.pod.about"], "nothing") - - set(resource.attributes["k8s.pod.about"], resource.attributes["com.splunk.metricsIndex"]) - set(resource.attributes["com.splunk.index"], resource.attributes["com.splunk.metricsIndex"]) + # If I do that it works fine (name of the index as a string): + # - set(resource.attributes["com.splunk.index"], "kube_metrics2") {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }} # This processor is used to remove excessive istio attributes to avoid running into the dimensions limit. @@ -810,6 +805,7 @@ service: processors: - memory_limiter - k8sattributes/metrics + - transform/metrics_index_update - batch {{- if or .Values.autodetect.prometheus .Values.autodetect.istio }} - attributes/istio @@ -847,6 +843,7 @@ service: processors: - memory_limiter - k8sattributes/metrics + - transform/metrics_index_update - batch - resource/add_agent_k8s - resourcedetection diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl index 7f5c53d00a..49874c3097 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl @@ -34,6 +34,7 @@ receivers: # By default k8sattributes, memory_limiter and batch processors enabled. processors: {{- include "splunk-otel-collector.k8sAttributesProcessor" . | nindent 2 }} + {{- include "splunk-otel-collector.k8sAttributesProcessorMetrics" . | nindent 2 }} {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }} {{- if .Values.autodetect.istio }} {{- include "splunk-otel-collector.transformLogsProcessor" . | nindent 2 }} @@ -172,6 +173,8 @@ service: receivers: [otlp, signalfx] processors: - memory_limiter + - k8sattributes/metrics + - transform/metrics_index_update - batch - resource/add_cluster_name {{- if .Values.extraAttributes.custom }} @@ -229,6 +232,8 @@ service: receivers: [prometheus/collector] processors: - memory_limiter + - k8sattributes/metrics + - transform/metrics_index_update - batch - resource/add_collector_k8s - resourcedetection diff --git a/my_values.yaml b/my_values.yaml index 5a3e0bdc79..17c7676b95 100644 --- a/my_values.yaml +++ b/my_values.yaml @@ -1,7 +1,7 @@ clusterName: "test_cluster" splunkPlatform: - endpoint: "https://10.202.12.84:8088/services/collector/event" + endpoint: "https://10.202.19.233:8088/services/collector/event" token: "00000000-0000-0000-0000-000000000000" index: "kube" @@ -29,9 +29,3 @@ agent: enabled: true etcd: skipVerify: false - -extraAttributes: - fromAnnotations: - - from: namespace - key: splunk.com/metricsIndex - tag_name: com.splunk.metricsIndex From 1ed987517c4b9770dc5615ee97391e313c1023b8 Mon Sep 17 00:00:00 2001 From: sbylica-splunk Date: Thu, 9 Nov 2023 13:31:12 +0000 Subject: [PATCH 23/23] . --- my_values.yaml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 my_values.yaml diff --git a/my_values.yaml b/my_values.yaml deleted file mode 100644 index 17c7676b95..0000000000 --- a/my_values.yaml +++ /dev/null @@ -1,31 +0,0 @@ -clusterName: "test_cluster" - -splunkPlatform: - endpoint: "https://10.202.19.233:8088/services/collector/event" - token: "00000000-0000-0000-0000-000000000000" - - index: "kube" - metricsIndex: "kube_metrics" - insecureSkipVerify: true - metricsEnabled: true - -agent: - enabled: true - - config: - exporters: - logging: - loglevel: debug - - controlPlaneMetrics: - apiserver: - # Specifies whether to collect apiserver metrics. - enabled: true - controllerManager: - # Specifies whether to collect controller manager metrics. - enabled: true - coredns: - # Specifies whether to collect coredns metrics. - enabled: true - etcd: - skipVerify: false