rollup apps monitoring and alerts

astriaorg · Oct 3, 2024 · b708b35 · b708b35
1 parent b54ccb9
commit b708b35
Show file tree

Hide file tree

Showing 15 changed files with 124 additions and 14 deletions.
diff --git a/charts/composer/Chart.yaml b/charts/composer/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.4
+version: 0.1.5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/composer/templates/_helpers.tpl b/charts/composer/templates/_helpers.tpl
@@ -5,6 +5,27 @@ Namepsace to deploy elements into.
 {{- default .Release.Namespace .Values.global.namespaceOverride | trunc 63 | trimSuffix "-" -}}
 {{- end }}
 
+{{/*
+application name to deploy elements into.
+*/}}
+{{- define "composer.appName" -}}
+composer
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "composer.labels" -}}
+{{ include "rollup.selectorLabels" . }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "composer.selectorLabels" -}}
+app: {{ include "rollup.appName" . }}
+{{- end }}
+
 {{/*
 Single entry of rollup names
 */}}

diff --git a/charts/composer/templates/deployment.yaml b/charts/composer/templates/deployment.yaml
@@ -39,7 +39,7 @@ spec:
           ports:
             {{- if .Values.metrics.enabled }}
             - containerPort: {{ .Values.ports.metrics }}
-              name: metrics
+              name: composer-metrics
             {{- end }}
             - containerPort: {{ .Values.ports.healthApi }}
               name: health-api

diff --git a/charts/composer/templates/service.yaml b/charts/composer/templates/service.yaml
@@ -15,15 +15,15 @@ spec:
 kind: Service
 apiVersion: v1
 metadata:
-  name: metrics
+  name: composer-metrics
   namespace: {{ include "composer.namespace" . }}
   labels:
     app: composer
 spec:
   selector:
     app: composer
   ports:
-    - name: metrics
+    - name: composer-metrics
       port: {{ .Values.ports.metrics }}
-      targetPort: geth-metr
+      targetPort: composer-metrics
 {{- end }}
diff --git a/charts/composer/templates/servicemonitor.yaml b/charts/composer/templates/servicemonitor.yaml
@@ -16,7 +16,7 @@ spec:
     matchLabels:
       app: composer
   endpoints:
-    - port: metrics
+    - port: composer-metrics
       path: /
       {{- with .Values.serviceMonitor.interval }}
       interval: {{ . }}

diff --git a/charts/composer/values.yaml b/charts/composer/values.yaml
@@ -50,6 +50,29 @@ serviceMonitor:
   additionalLabels:
     release: kube-prometheus-stack
 
+alerting:
+  enabled: false
+  interval: ""
+  additionalLabels:
+    release: kube-prometheus-stack
+  annotations: {}
+  # scrapeTimeout: 10s
+  # path: /metrics
+  prometheusRule:
+    enabled: true
+    additionalLabels:
+      release: kube-prometheus-stack
+    namespace: monitoring
+    rules:
+    - alert: Composer_Node_Down
+      expr: up{container="composer"} == 0  # Insert your query Expression
+      for: 1m  # Rough number but should be enough to init warn
+      labels:
+        severity: warning
+      annotations:
+        summary: Composer is Down (instance {{ $labels.instance }})
+        description: "composer node '{{ $labels.namespace }}' has disappeared from Prometheus target discovery.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
 # When deploying in a production environment should use a secret provider
 # This is configured for use with GCP, need to set own resource names
 # and keys

diff --git a/charts/evm-bridge-withdrawer/Chart.yaml b/charts/evm-bridge-withdrawer/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.3.0
+version: 0.3.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/evm-bridge-withdrawer/templates/_helpers.tpl b/charts/evm-bridge-withdrawer/templates/_helpers.tpl
@@ -8,3 +8,24 @@ Namepsace to deploy elements into.
 {{- define "evm-bridge-withdrawer.image" -}}
 {{ .Values.images.evmBridgeWithdrawer.repo }}:{{ if .Values.global.dev }}{{ .Values.images.evmBridgeWithdrawer.devTag }}{{ else }}{{ .Values.images.evmBridgeWithdrawer.tag }}{{ end }}
 {{- end }}
+
+{{/*
+application name to deploy elements into.
+*/}}
+{{- define "evm-bridge-withdrawer.appName" -}}
+evm-bridge-withdrawer
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "evm-bridge-withdrawer.labels" -}}
+{{ include "evm-bridge-withdrawer.selectorLabels" . }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "evm-bridge-withdrawer.selectorLabels" -}}
+app: {{ include "evm-bridge-withdrawer.appName" . }}
+{{- end }}
diff --git a/charts/evm-bridge-withdrawer/templates/prometheusrule.yaml b/charts/evm-bridge-withdrawer/templates/prometheusrule.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.alerting.enabled -}}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: evm-bridge-withdrawer-alerting
+{{- if .Values.alerting.prometheusRule.namespace }}
+  namespace: {{ .Values.alerting.prometheusRule.namespace | quote }}
+{{- end }}
+  labels:
+    {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }}
+  {{- if .Values.alerting.prometheusRule.additionalLabels }}
+    {{- toYaml .Values.alerting.prometheusRule.additionalLabels | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.alerting.prometheusRule.rules }}
+  groups:
+  - name: {{ template "evm-bridge-withdrawer.appName" . }}
+    rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }}
+{{- end }}
+{{- end }}
diff --git a/charts/evm-bridge-withdrawer/templates/service.yaml b/charts/evm-bridge-withdrawer/templates/service.yaml
@@ -4,9 +4,11 @@ apiVersion: v1
 metadata:
   name: evm-bridge-withdrawer-metrics
   namespace: {{ include "evm-bridge-withdrawer.namespace" . }}
+  labels:
+    {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }}
 spec:
   selector:
-    app: evm-bridge-withdrawer
+    {{- include "evm-bridge-withdrawer.labels" . | nindent 4 }}
   ports:
     {{- if .Values.metrics.enabled }}
     - name: metrics

diff --git a/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml b/charts/evm-bridge-withdrawer/templates/servicemonitor.yaml
@@ -17,7 +17,7 @@ spec:
     matchLabels:
       app: evm-bridge-withdrawer
   endpoints:
-    - port: metric
+    - port: metrics
       path: /
       {{- with .Values.serviceMonitor.interval }}
       interval: {{ . }}

diff --git a/charts/evm-bridge-withdrawer/values.yaml b/charts/evm-bridge-withdrawer/values.yaml
@@ -54,6 +54,29 @@ serviceMonitor:
   additionalLabels:
     release: kube-prometheus-stack
 
+alerting:
+  enabled: false
+  interval: ""
+  additionalLabels:
+    release: kube-prometheus-stack
+  annotations: {}
+  # scrapeTimeout: 10s
+  # path: /metrics
+  prometheusRule:
+    enabled: true
+    additionalLabels:
+      release: kube-prometheus-stack
+    namespace: monitoring
+    rules:
+    - alert: Bridge_Withdrawer_Down
+      expr: up{container="evm-bridge-withdrawer"} == 0  # Insert your query Expression
+      for: 1m  # Rough number but should be enough to init warn
+      labels:
+        severity: warning
+      annotations:
+        summary: Bridge Withdrawer is Down (instance {{ $labels.instance }})
+        description: "bridge withdrawer node '{{ $labels.namespace }}' has disappeared from Prometheus target discovery.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
 # When deploying in a production environment should use a secret provider
 # This is configured for use with GCP, need to set own resource names
 # and keys

diff --git a/charts/evm-rollup/templates/prometheusrule.yaml b/charts/evm-rollup/templates/prometheusrule.yaml
@@ -12,7 +12,7 @@ metadata:
     {{- toYaml .Values.alerting.prometheusRule.additionalLabels | nindent 4 }}
   {{- end }}
 spec:
-{{- if .Values.config.rollup.alerting.prometheusRule.rules }}
+{{- if .Values.alerting.prometheusRule.rules }}
   groups:
   - name: {{ template "rollup.name" . }}
     rules: {{- toYaml .Values.alerting.prometheusRule.rules | nindent 4 }}

diff --git a/charts/evm-stack/Chart.lock b/charts/evm-stack/Chart.lock
@@ -7,13 +7,13 @@ dependencies:
   version: 0.27.5
 - name: composer
   repository: file://../composer
-  version: 0.1.4
+  version: 0.1.5
 - name: evm-faucet
   repository: file://../evm-faucet
   version: 0.1.2
 - name: evm-bridge-withdrawer
   repository: file://../evm-bridge-withdrawer
-  version: 0.3.0
+  version: 0.3.1
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 15.2.4

diff --git a/charts/evm-stack/Chart.yaml b/charts/evm-stack/Chart.yaml
@@ -26,15 +26,15 @@ dependencies:
     version: 0.27.5
     repository: "file://../evm-rollup"
   - name: composer
-    version: 0.1.4
+    version: 0.1.5
     repository: "file://../composer"
     condition: composer.enabled
   - name: evm-faucet
     version: 0.1.2
     repository: "file://../evm-faucet"
     condition: evm-faucet.enabled
   - name: evm-bridge-withdrawer
-    version: 0.3.0
+    version: 0.3.1
     repository: "file://../evm-bridge-withdrawer"
     condition: evm-bridge-withdrawer.enabled
   - name: postgresql