[ops] Introduce GitpodWsManagerMk2BackupFailureError and GitpodWsMana…

…gerMk2BackupFailureCritical (#20259)

* [ops] Introduce GitpodWsManagerMk2BackupFailureError and GitpodWsManagerMk2BackupFailureCritical

* Fix

operations/observability/mixins/workspace/rules/satellite/workspaces.yaml

@@ -45,3 +45,23 @@ spec:
         sum by(cluster) (avg_over_time(gitpod_workspace_regular_not_active_percentage_mk2[1m]) > 0)
         AND
         sum by(cluster) (rate(gitpod_ws_manager_mk2_workspace_startup_seconds_sum{type="Regular"}[1m])) == 0
+    - alert: GitpodWsManagerMk2BackupFailureError
+      labels:
+        severity: error
+        team: engine
+      annotations:
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/WorkspaceBackupFailures.md
+        summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
+        description: This can happen when a single node has failed in the cloud provider
+      expr: |
+        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) <= 16
+    - alert: GitpodWsManagerMk2BackupFailureCritical
+      labels:
+        severity: critical
+        team: engine
+      annotations:
+        runbook_url: https://github.com/gitpod-io/runbooks/blob/main/runbooks/WorkspaceBackupFailures.md
+        summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
+        description: This can be an indicator of two or more nodes failing in a cloud provider
+      expr: |
+        sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 16