diff --git a/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-store-telemeter-template.yaml b/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-store-telemeter-template.yaml index 26fafa5638..afef2d16d3 100755 --- a/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-store-telemeter-template.yaml +++ b/resources/services/telemeter-prod-01/rhobs/telemeter/observatorium-metrics-store-telemeter-template.yaml @@ -83,7 +83,7 @@ objects: name: observatorium-thanos-store-bucket-cache-memcached-telemeter namespace: rhobs spec: - replicas: 1 + replicas: 5 selector: matchLabels: app.kubernetes.io/component: store-bucket-cache @@ -135,10 +135,11 @@ objects: protocol: TCP resources: limits: - memory: 3Gi + cpu: "7" + memory: 12Gi requests: - cpu: 500m - memory: 2Gi + cpu: "6" + memory: 10Gi terminationMessagePolicy: FallbackToLogsOnError - args: - --memcached.address=localhost:0 @@ -240,7 +241,7 @@ objects: name: observatorium-thanos-store-index-cache-memcached-telemeter namespace: rhobs spec: - replicas: 1 + replicas: 5 selector: matchLabels: app.kubernetes.io/component: store-index-cache @@ -292,10 +293,11 @@ objects: protocol: TCP resources: limits: - memory: 3Gi + cpu: "7" + memory: 12Gi requests: - cpu: 500m - memory: 2Gi + cpu: "6" + memory: 10Gi terminationMessagePolicy: FallbackToLogsOnError - args: - --memcached.address=localhost:0 @@ -517,7 +519,7 @@ objects: dns_provider_update_interval: 10s - --log.format=logfmt - --log.level=${STORE_LOG_LEVEL} - - --max-time=-22h0m0s + - --max-time=-336h0m0s - --objstore.config=$(OBJSTORE_CONFIG) - --selector.relabel-config-file=/etc/thanos/hashmod/hashmod-config.yaml - | @@ -607,6 +609,7 @@ objects: periodSeconds: 5 resources: limits: + cpu: "7" memory: ${STORE_MEMORY_LIMIT} requests: cpu: ${STORE_CPU_REQUEST} @@ -706,7 +709,7 @@ objects: - ReadWriteOnce resources: requests: - storage: 5Gi + storage: 50Gi storageClassName: gp2 - apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor @@ -809,12 +812,12 @@ objects: observatorium/tenant: telemeter parameters: - name: STORE_CPU_REQUEST - value: "2" + value: "5" - name: STORE_LOG_LEVEL value: warn - name: STORE_MEMORY_LIMIT - value: 20Gi + value: 60Gi - name: STORE_MEMORY_REQUEST - value: 5Gi + value: 40Gi - name: STORE_REPLICAS - value: "1" + value: "5" diff --git a/services_go/instances/rhobs/rhobs.go b/services_go/instances/rhobs/rhobs.go index 198249b94a..df56899043 100644 --- a/services_go/instances/rhobs/rhobs.go +++ b/services_go/instances/rhobs/rhobs.go @@ -3,6 +3,8 @@ package rhobs import ( "encoding/json" "fmt" + "github.com/observatorium/observatorium/configuration_go/abstr/kubernetes/memcached" + thanostime "github.com/observatorium/observatorium/configuration_go/schemas/thanos/time" "sort" "time" @@ -331,7 +333,22 @@ func prodConfig() observatorium.Observatorium { ingestor.VolumeSize = "5Gi" }, StorePreManifestsHook: func(store *store.StoreStatefulSet) { - store.VolumeSize = "5Gi" + store.VolumeSize = "50Gi" + store.Replicas = 5 + store.ContainerResources = kghelpers.NewResourcesRequirements("5", "7", "40Gi", "60Gi") + }, + StoreOpts: func(opts *store.StoreOptions) { + // Telemeter Lookback is -14 days (336 hours) during migration to account for an overlap with our hot telemeter instance + telemeterMaxLookBack := time.Duration(-336) * time.Hour + opts.MaxTime = &thanostime.TimeOrDurationValue{Dur: &telemeterMaxLookBack} + }, + IndexCachePreManifestsHook: func(indexCache *memcached.MemcachedDeployment) { + indexCache.Replicas = 5 + indexCache.ContainerResources = kghelpers.NewResourcesRequirements("6", "7", "10Gi", "12Gi") + }, + BucketCachePreManifestsHook: func(bucketCache *memcached.MemcachedDeployment) { + bucketCache.Replicas = 5 + bucketCache.ContainerResources = kghelpers.NewResourcesRequirements("6", "7", "10Gi", "12Gi") }, }, }, diff --git a/services_go/observatorium/metrics.go b/services_go/observatorium/metrics.go index 121212ddfc..d5367105a5 100644 --- a/services_go/observatorium/metrics.go +++ b/services_go/observatorium/metrics.go @@ -88,6 +88,7 @@ type ObservatoriumMetricsInstance struct { ObjStoreSecret string Tenants []Tenants StorePreManifestsHook func(*store.StoreStatefulSet) + StoreOpts func(opts *store.StoreOptions) IndexCachePreManifestsHook func(*memcached.MemcachedDeployment) BucketCachePreManifestsHook func(*memcached.MemcachedDeployment) CompactorPreManifestsHook func(*compactor.CompactorStatefulSet) @@ -939,6 +940,7 @@ func (o *ObservatoriumMetrics) makeStore(instanceCfg *ObservatoriumMetricsInstan }, } opts.AddExtraOpts("--store.enable-index-header-lazy-reader") + executeIfNotNil(instanceCfg.StoreOpts, opts) indexCacheName := fmt.Sprintf("observatorium-thanos-store-index-cache-memcached-%s", instanceCfg.InstanceName) bucketCacheName := fmt.Sprintf("observatorium-thanos-store-bucket-cache-memcached-%s", instanceCfg.InstanceName)