From d155bf3f3d9c29287db336ea45c16b3829423c92 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Tue, 18 Jul 2023 17:26:16 +0200 Subject: [PATCH 1/3] add request logging to thanos components --- .../observatorium-metrics-template.yaml | 84 +++++++++++++++++++ ...rium-metrics-template-overwrites.libsonnet | 26 +++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/resources/services/observatorium-metrics-template.yaml b/resources/services/observatorium-metrics-template.yaml index ddef2a70a5..19d34f6d92 100644 --- a/resources/services/observatorium-metrics-template.yaml +++ b/resources/services/observatorium-metrics-template.yaml @@ -144,6 +144,13 @@ objects: - --downsample.concurrency=1 - --deduplication.replica-label=replica - --debug.max-compaction-level=3 + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" - ${THANOS_COMPACTOR_RETENTION_DISABLE_DOWNSAMPLING} env: - name: OBJSTORE_CONFIG @@ -1171,6 +1178,13 @@ objects: - --grpc.proxy-strategy=${THANOS_QUERIER_PROXY_STRATEGY} - --query.promql-engine=${THANOS_QUERIER_ENGINE} - --query.max-concurrent=${THANOS_QUERIER_MAX_CONCURRENT} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1495,6 +1509,13 @@ objects: - --labels.max-retries-per-request=${THANOS_QUERY_FRONTEND_MAX_RETRIES} - --labels.default-time-range=336h - --cache-compression-type=snappy + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1825,6 +1846,13 @@ objects: - --configmap-generated-name=observatorium-thanos-receive-controller-tenants-generated - --file-name=hashrings.json - --namespace=$(NAMESPACE) + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: NAMESPACE valueFrom: @@ -2085,6 +2113,13 @@ objects: - --receive.grpc-compression=none - --receive.hashrings-algorithm=${THANOS_RECEIVE_HASHRINGS_ALGORITHM} - --receive.hashrings-file-refresh-interval=5s + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: NAME valueFrom: @@ -2567,6 +2602,13 @@ objects: "sampler_type": "ratelimiting" "service_name": "thanos-rule" "type": "JAEGER" + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: NAME valueFrom: @@ -3143,6 +3185,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3400,6 +3449,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3657,6 +3713,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3914,6 +3977,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4171,6 +4241,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4428,6 +4505,13 @@ objects: - --store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT} - --store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT} - --max-time=${THANOS_STORE_MAX_TIME} + - |- + --request.logging-config="grpc": + "options": + "decision": + "log_end": true + "log_start": true + "level": "DEBUG" env: - name: OBJSTORE_CONFIG valueFrom: diff --git a/services/observatorium-metrics-template-overwrites.libsonnet b/services/observatorium-metrics-template-overwrites.libsonnet index e8029e70f1..deb6fa1a9e 100644 --- a/services/observatorium-metrics-template-overwrites.libsonnet +++ b/services/observatorium-metrics-template-overwrites.libsonnet @@ -6,6 +6,17 @@ local oauthProxy = import './sidecars/oauth-proxy.libsonnet'; local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; { + local request_logging = std.manifestYamlDoc({ + grpc: { + options: { + level: 'DEBUG', + decision: { + log_start: true, + log_end: true, + }, + }, + }, + }), local s3EnvVars = [ { name: 'AWS_ACCESS_KEY_ID', @@ -92,7 +103,10 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; // Temporary workaround on high cardinality blocks for 2w. // Since we have only 2w retention, there is no point in having 2w blocks. // See: https://issues.redhat.com/browse/OBS-437 - args+: ['--debug.max-compaction-level=3'] + disableDownsamplingFlag, + args+: [ + '--debug.max-compaction-level=3', + '--request.logging-config=' + request_logging, + ] + disableDownsamplingFlag, } else c for c in super.containers ], @@ -127,6 +141,9 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; failureThreshold: 10, periodSeconds: 120, }, + args+: [ + '--request.logging-config=' + request_logging, + ], } else c for c in super.containers ], @@ -171,6 +188,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; '--store.grpc.touched-series-limit=${THANOS_STORE_SERIES_TOUCHED_LIMIT}', '--store.grpc.series-sample-limit=${THANOS_STORE_SERIES_SAMPLE_LIMIT}', '--max-time=${THANOS_STORE_MAX_TIME}', + '--request.logging-config=' + request_logging, ], } else c for c in super.containers @@ -191,6 +209,9 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; containers: [ if c.name == 'thanos-receive-controller' then c { securityContext: {}, + args+: [ + '--request.logging-config=' + request_logging, + ], } else c for c in super.containers ], @@ -268,6 +289,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; '--grpc.proxy-strategy=${THANOS_QUERIER_PROXY_STRATEGY}', '--query.promql-engine=${THANOS_QUERIER_ENGINE}', '--query.max-concurrent=${THANOS_QUERIER_MAX_CONCURRENT}', + '--request.logging-config=' + request_logging, ], } else c for c in super.containers @@ -431,6 +453,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; '--labels.max-retries-per-request=%s' % '${THANOS_QUERY_FRONTEND_MAX_RETRIES}', '--labels.default-time-range=336h', '--cache-compression-type=snappy', + '--request.logging-config=' + request_logging, ], } else c for c in super.containers @@ -458,6 +481,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; '--receive.grpc-compression=none', '--receive.hashrings-algorithm=${THANOS_RECEIVE_HASHRINGS_ALGORITHM}', '--receive.hashrings-file-refresh-interval=5s', + '--request.logging-config=' + request_logging, ], env+: s3EnvVars + [{ name: 'DEBUG', From b30a55406feb4ef925342bc4c6061f704bc415f7 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Tue, 18 Jul 2023 17:34:17 +0200 Subject: [PATCH 2/3] change log level --- .../observatorium-metrics-template.yaml | 24 +++++++++---------- ...rium-metrics-template-overwrites.libsonnet | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/resources/services/observatorium-metrics-template.yaml b/resources/services/observatorium-metrics-template.yaml index 19d34f6d92..dd36204ea6 100644 --- a/resources/services/observatorium-metrics-template.yaml +++ b/resources/services/observatorium-metrics-template.yaml @@ -150,7 +150,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" - ${THANOS_COMPACTOR_RETENTION_DISABLE_DOWNSAMPLING} env: - name: OBJSTORE_CONFIG @@ -1184,7 +1184,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1515,7 +1515,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1852,7 +1852,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: NAMESPACE valueFrom: @@ -2119,7 +2119,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: NAME valueFrom: @@ -2608,7 +2608,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: NAME valueFrom: @@ -3191,7 +3191,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3455,7 +3455,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3719,7 +3719,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3983,7 +3983,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4247,7 +4247,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4511,7 +4511,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "DEBUG" + "level": "INFO" env: - name: OBJSTORE_CONFIG valueFrom: diff --git a/services/observatorium-metrics-template-overwrites.libsonnet b/services/observatorium-metrics-template-overwrites.libsonnet index deb6fa1a9e..e6ec1cda5d 100644 --- a/services/observatorium-metrics-template-overwrites.libsonnet +++ b/services/observatorium-metrics-template-overwrites.libsonnet @@ -9,7 +9,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; local request_logging = std.manifestYamlDoc({ grpc: { options: { - level: 'DEBUG', + level: 'INFO', decision: { log_start: true, log_end: true, From 14f72aa895e092d447d30bff15d9e5a29970e571 Mon Sep 17 00:00:00 2001 From: Coleen Iona Quadros Date: Tue, 18 Jul 2023 18:04:32 +0200 Subject: [PATCH 3/3] add var for log level --- .../observatorium-metrics-template.yaml | 26 ++++++++++--------- ...rium-metrics-template-overwrites.libsonnet | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/resources/services/observatorium-metrics-template.yaml b/resources/services/observatorium-metrics-template.yaml index dd36204ea6..f4b49591a1 100644 --- a/resources/services/observatorium-metrics-template.yaml +++ b/resources/services/observatorium-metrics-template.yaml @@ -150,7 +150,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" - ${THANOS_COMPACTOR_RETENTION_DISABLE_DOWNSAMPLING} env: - name: OBJSTORE_CONFIG @@ -1184,7 +1184,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1515,7 +1515,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: HOST_IP_ADDRESS valueFrom: @@ -1852,7 +1852,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: NAMESPACE valueFrom: @@ -2119,7 +2119,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: NAME valueFrom: @@ -2608,7 +2608,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: NAME valueFrom: @@ -3191,7 +3191,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3455,7 +3455,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3719,7 +3719,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -3983,7 +3983,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4247,7 +4247,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4511,7 +4511,7 @@ objects: "decision": "log_end": true "log_start": true - "level": "INFO" + "level": "${THANOS_REQUEST_LOGGING_LEVEL}" env: - name: OBJSTORE_CONFIG valueFrom: @@ -4938,3 +4938,5 @@ parameters: value: 1Gi - name: THANOS_VOLCANO_MEMORY_REQUEST value: 256Mi +- name: THANOS_REQUEST_LOGGING_LEVEL + value: INFO diff --git a/services/observatorium-metrics-template-overwrites.libsonnet b/services/observatorium-metrics-template-overwrites.libsonnet index e6ec1cda5d..d32305af4f 100644 --- a/services/observatorium-metrics-template-overwrites.libsonnet +++ b/services/observatorium-metrics-template-overwrites.libsonnet @@ -9,7 +9,7 @@ local thanosRuleSyncer = import './sidecars/thanos-rule-syncer.libsonnet'; local request_logging = std.manifestYamlDoc({ grpc: { options: { - level: 'INFO', + level: '${THANOS_REQUEST_LOGGING_LEVEL}', decision: { log_start: true, log_end: true,