From 00ba20bf6fac0d2a1f06040abc96b3349c713c5d Mon Sep 17 00:00:00 2001
From: Gianni Carafa <gianni.carafa@srf.ch>
Date: Mon, 13 May 2024 05:00:57 +0200
Subject: [PATCH] add monitoring alerts

---
 client/src/components/apps/alerts.vue  | 95 ++++++++++++++++++++++++++
 client/src/components/apps/metrics.vue | 31 ++++++++-
 server/src/modules/metrics.ts          | 54 ++++++++++++++-
 server/src/routes/metrics.ts           | 13 +++-
 4 files changed, 188 insertions(+), 5 deletions(-)
 create mode 100644 client/src/components/apps/alerts.vue

diff --git a/client/src/components/apps/alerts.vue b/client/src/components/apps/alerts.vue
new file mode 100644
index 00000000..0bac8ae4
--- /dev/null
+++ b/client/src/components/apps/alerts.vue
@@ -0,0 +1,95 @@
+<template>
+    <div>
+
+        <v-alert 
+            v-for="rule in rules" :key="rule.name"
+            :text="rule.alert?.annotations.description"
+            :title="rule.name"
+            :type="rule.alerting ? 'error' : 'success'"
+            variant="tonal"
+            density="compact"
+            style="transform: scale(0.7); transform-origin: left; margin-bottom: -8px;"
+        ></v-alert>
+    </div>
+</template>
+
+<script lang="ts">
+import axios from 'axios'
+
+
+export interface Rule {
+  duration: number
+  health: string
+  labels: Object
+  name: string
+  query: string
+  alerting: boolean
+  alert?: Alert
+}
+
+
+export interface Alert {
+  activeAt: string
+  annotations: {
+    description: string
+    summary: string
+  }
+  labels: Object
+  state: string
+  value: number
+}
+
+export default {
+    data() {
+        return {
+            rules: [] as Rule[],
+            timer: null as any
+        }
+    },
+    props: {
+        pipeline: {
+            type: String,
+            required: true
+        },
+        phase: {
+            type: String,
+            required: true
+        },
+        app: {
+            type: String,
+            required: true
+        }
+    },
+    computed: {
+        alertstate(rule: Rule) {
+            if (rule.alerting ) {
+                return 'error'
+            } else {
+                return 'info'
+            }
+        }
+    },
+    created() {
+        this.loadRules()
+    },
+    mounted() {
+        this.startTimer();
+    },
+    unmounted() {
+        clearInterval(this.timer);
+    },
+    methods: {
+        startTimer() {
+            this.timer = setInterval(() => {
+                this.loadRules()
+            }, 10000);
+        },
+        loadRules() {
+            axios.get(`/api/rules/${this.pipeline}/${this.phase}/${this.app}`)
+            .then(response => {
+                this.rules = response.data
+            })
+        }
+    }
+}
+</script>
\ No newline at end of file
diff --git a/client/src/components/apps/metrics.vue b/client/src/components/apps/metrics.vue
index ffef5e64..80ae4070 100644
--- a/client/src/components/apps/metrics.vue
+++ b/client/src/components/apps/metrics.vue
@@ -25,7 +25,7 @@
       <div v-if="kubero.metricsEnabled">
         <v-row class="justify-space-between mb-2">
             <v-col cols="8" sm="8" md="8" lg="8" xl="8">
-                <!--<h1>Vulnerabilities in {{ app }}</h1>-->
+              <Alerts :app="app" :phase="phase" :pipeline="pipeline"/>
             </v-col>
             <v-col>
               <v-select
@@ -53,10 +53,14 @@
             </v-col>
         </v-row>
         <v-row>
-            <v-col cols="12" sm="12" md="12">
+            <v-col cols="6" sm="6" md="6">
                 CPU usage
                 <VueApexCharts type="line" height="180" :options="cpuOptions" :series="cpuData"></VueApexCharts>
             </v-col>
+            <v-col cols="6" sm="6" md="6">
+                CPU usage
+                <VueApexCharts type="line" height="180" :options="cpuOptions" :series="cpuDataRate"></VueApexCharts>
+            </v-col>
         </v-row>
         <!--
         <v-row>
@@ -99,7 +103,7 @@ import { useKuberoStore } from '../../stores/kubero'
 import { mapState } from 'pinia'
 
 import axios from "axios";
-import { start } from 'repl';
+import Alerts from './alerts.vue';
 
 const colors = ['#8560a9', '#a887c9', '#b99bd6', '#d2bde6']
 
@@ -568,6 +572,10 @@ export default defineComponent({
             name: string,
             data: number[][],
       }[],
+      cpuDataRate: [] as {
+            name: string,
+            data: number[][],
+      }[],
       loadData: [] as {
             name: string,
             data: number[][],
@@ -597,6 +605,7 @@ export default defineComponent({
     }),
     components: {
         VueApexCharts,
+        Alerts,
     },
     mounted() {
         this.refreshMetrics();
@@ -635,6 +644,7 @@ export default defineComponent({
             this.getMemoryMetrics();
             //this.getLoadMetrics();
             this.getCpuMetrics();
+            this.getCpuMetricsRate();
             this.getHttpStatusCodeMetrics();
             this.getResponseTimeMetrics();
             this.getHttpStatusCodeIncreaseMetrics();
@@ -721,6 +731,7 @@ export default defineComponent({
             });
         },
         getCpuMetrics() {
+            // use 'rate' instead of 'increase' when comparing to limit and request
             axios.get(`/api/longtermmetrics/cpu/${this.pipeline}/${this.phase}/${this.app}/increase`, {
                 params: {
                     scale: this.scale
@@ -733,6 +744,20 @@ export default defineComponent({
                 console.log(error);
             });
         },
+        getCpuMetricsRate() {
+            // use 'rate' instead of 'increase' when comparing to limit and request
+            axios.get(`/api/longtermmetrics/cpu/${this.pipeline}/${this.phase}/${this.app}/rate`, {
+                params: {
+                    scale: this.scale
+                }
+            })
+            .then((response) => {
+              this.cpuDataRate = response.data;
+            })
+            .catch((error) => {
+                console.log(error);
+            });
+        },
     },
 });
 </script>
\ No newline at end of file
diff --git a/server/src/modules/metrics.ts b/server/src/modules/metrics.ts
index 83c53721..e4991a2e 100644
--- a/server/src/modules/metrics.ts
+++ b/server/src/modules/metrics.ts
@@ -23,6 +23,16 @@ interface IMetric {
     }[]
 }
 
+type Rule = {
+    alert: any,
+    duration: number,
+    health: string,
+    labels: any,
+    name: string,
+    query: string,
+    alerting: boolean,
+}
+
 export class Metrics {
     private prom: PrometheusDriver
 
@@ -185,7 +195,7 @@ export class Metrics {
 
         const { end, start, step, vector } = this.getStepsAndStart(q.scale);
         // rate(nginx_ingress_controller_requests{namespace="asdf-production", host="a.a.localhost"}[10m])
-        const query = `${q.calc}(container_cpu_system_seconds_total{namespace="${q.pipeline}-${q.phase}", container=~"kuberoapp-web|kuberoapp-worker"}[${vector}])`;
+        const query = `${q.calc}(container_cpu_usage_seconds_total{namespace="${q.pipeline}-${q.phase}", container=~"kuberoapp-web|kuberoapp-worker"}[${vector}])`;
         //console.log(query);
         try {
             metrics = await this.prom.rangeQuery(query, start, end, step);
@@ -300,4 +310,46 @@ export class Metrics {
         return resp;
     }
 
+    public async getRules(q: {app: string, phase: string, pipeline: string}): Promise<any> {
+        let rules = await this.prom.rules();
+
+        let ruleslist: Rule[] = [];
+        
+        // filter for dedicated app
+        for (let i = 0; i < rules.length; i++) {
+            for (let j = 0; j < rules[i].rules.length; j++) {
+                // remove not matching alerts
+                rules[i].rules[j].alerts = rules[i].rules[j].alerts.filter((a: any) => {
+                    console.log("a.labels.namespace: "+a.labels.namespace+" == q.pipeline: "+q.pipeline+"-"+q.phase)
+                    console.log("a.labels.service: "+a.labels.service+" q.app: "+q.app+"-kuberoapp");
+                    return a.labels.namespace === q.pipeline+"-"+q.phase && (
+                        a.labels.service === q.app+"-kuberoapp" || 
+                        a.labels.deployment?.startsWith(q.app+"-kuberoapp") ||
+                        a.labels.replicaset?.startsWith(q.app+"-kuberoapp") ||
+                        a.labels.statefulset === q.app+"-kuberoapp" ||
+                        a.labels.daemonset === q.app+"-kuberoapp" ||
+                        a.labels.pod === q.app+"-kuberoapp" ||
+                        a.labels.container === q.app+"-kuberoapp" ||
+                        a.labels.job === q.app+"-kuberoapp"
+                    )
+                });
+
+                let r: Rule = {
+                    alert: rules[i].rules[j].alerts[0],
+                    duration: rules[i].rules[j].duration || 0,
+                    health: rules[i].rules[j].health || '',
+                    labels: rules[i].rules[j].labels || {},
+                    name: rules[i].rules[j].name || '',
+                    query: rules[i].rules[j].query || '',
+                    alerting: rules[i].rules[j].alerts.length > 0 ? true : false,
+                };
+
+                if (rules[i].rules[j].type === 'alerting') {
+                    ruleslist.push(r);
+                }
+            }
+        }
+
+        return ruleslist;
+    }
 }
\ No newline at end of file
diff --git a/server/src/routes/metrics.ts b/server/src/routes/metrics.ts
index 2b9f043d..55aa1a34 100644
--- a/server/src/routes/metrics.ts
+++ b/server/src/routes/metrics.ts
@@ -141,4 +141,15 @@ Router.get('/longtermmetrics/cpu/:pipeline/:phase/:app/:calc', authMiddleware, a
         console.log(error)
         res.send('error') 
     }
-});
\ No newline at end of file
+});
+
+Router.get('/rules/:pipeline/:phase/:app', authMiddleware, async function (req: Request, res: Response) {
+    // #swagger.tags = ['UI']
+    // #swagger.summary = 'Get alerts'
+    const alerts = await req.app.locals.metrics.getRules({
+        pipeline: req.params.pipeline,
+        phase: req.params.phase,
+        app: req.params.app
+    });
+    res.send(alerts);
+});