Support setting timestamp on metric

README.md

@@ -75,6 +75,7 @@ Application Options:
       --azure.servicediscovery.cache=      Duration for caching Azure ServiceDiscovery of workspaces to reduce API
                                            calls (time.Duration) (default: 30m) [$AZURE_SERVICEDISCOVERY_CACHE]
       --metrics.resourceid.lowercase       Publish lowercase Azure Resoruce ID in metrics [$METRIC_RESOURCEID_LOWERCASE]
+      --metrics.set-timestamp              Set timestamp on scraped metrics [$METRIC_SET_TIMESTAMP]
       --metrics.template=                  Template for metric name (default: {name}) [$METRIC_TEMPLATE]
       --metrics.help=                      Metric help (with template support) (default: Azure monitor insight metric)
                                            [$METRIC_HELP]

config/opts.go

@@ -26,6 +26,7 @@ type (
 
 		Metrics struct {
 			ResourceIdLowercase bool   `long:"metrics.resourceid.lowercase"   env:"METRIC_RESOURCEID_LOWERCASE"       description:"Publish lowercase Azure Resoruce ID in metrics"`
+			SetTimestamp        bool   `long:"metrics.set-timestamp"          env:"METRIC_SET_TIMESTAMP"              description:"Set timestamp on scraped metrics"`
 			Template            string `long:"metrics.template"               env:"METRIC_TEMPLATE"                   description:"Template for metric name"   default:"{name}"`
 			Help                string `long:"metrics.help"                   env:"METRIC_HELP"                       description:"Metric help (with template support)"   default:"Azure monitor insight metric"`
 		}

go.mod

@@ -25,7 +25,10 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 )
 
-require github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7
+require (
+	github.com/google/go-cmp v0.5.5
+	github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7
+)
 
 require (
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect

metrics/collector.go

@@ -0,0 +1,68 @@
+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type metricListCollectorDetails struct {
+	gauge prometheus.Gauge
+	desc  *prometheus.Desc
+	ts    time.Time
+}
+
+type metricListCollector struct {
+	details []*metricListCollectorDetails
+}
+
+func NewMetricListCollector(list *MetricList) *metricListCollector {
+	collector := &metricListCollector{
+		details: []*metricListCollectorDetails{},
+	}
+
+	if list == nil {
+		return collector
+	}
+
+	// create prometheus metrics and set rows
+	for _, metricName := range list.GetMetricNames() {
+		gaugeVec := prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: metricName,
+				Help: list.GetMetricHelp(metricName),
+			},
+			list.GetMetricLabelNames(metricName),
+		)
+
+		for _, metric := range list.GetUniqueMetricList(metricName) {
+			gauge := gaugeVec.With(metric.Labels)
+			gauge.Set(metric.Value)
+
+			desc := prometheus.NewDesc(metricName, list.GetMetricHelp(metricName), []string{}, metric.Labels)
+
+			details := &metricListCollectorDetails{
+				gauge,
+				desc,
+				metric.Timestamp,
+			}
+
+			collector.details = append(collector.details, details)
+		}
+	}
+
+	return collector
+}
+
+func (c *metricListCollector) Describe(ch chan<- *prometheus.Desc) {
+	for _, detail := range c.details {
+		ch <- detail.desc
+	}
+}
+
+func (c *metricListCollector) Collect(ch chan<- prometheus.Metric) {
+	for _, detail := range c.details {
+		s := prometheus.NewMetricWithTimestamp(detail.ts, detail.gauge)
+		ch <- s
+	}
+}

metrics/insights.go

@@ -1,12 +1,14 @@
 package metrics
 
 import (
+	"regexp"
+	"strings"
+	"time"
+
 	"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/prometheus/client_golang/prometheus"
-	"regexp"
-	"strings"
 )
 
 var (
@@ -26,10 +28,11 @@ type (
 	}
 
 	PrometheusMetricResult struct {
-		Name   string
-		Labels prometheus.Labels
-		Value  float64
-		Help   string
+		Name      string
+		Labels    prometheus.Labels
+		Value     float64
+		Timestamp time.Time
+		Help      string
 	}
 )
 
@@ -70,17 +73,18 @@ func (p *MetricProber) FetchMetricsFromTarget(client *insights.MetricsClient, ta
 	return ret, err
 }
 
-func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64) (metric PrometheusMetricResult) {
+func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64, timestamp time.Time) (metric PrometheusMetricResult) {
 	// copy map to ensure we don't keep references
 	metricLabels := prometheus.Labels{}
 	for labelName, labelValue := range labels {
 		metricLabels[labelName] = labelValue
 	}
 
 	metric = PrometheusMetricResult{
-		Name:   r.settings.MetricTemplate,
-		Labels: metricLabels,
-		Value:  value,
+		Name:      r.settings.MetricTemplate,
+		Labels:    metricLabels,
+		Value:     value,
+		Timestamp: timestamp,
 	}
 
 	// fallback if template is empty (should not be)
@@ -140,8 +144,8 @@ func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value
 func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- PrometheusMetricResult) {
 	if r.Result.Value != nil {
 		// DEBUGGING
-		//data, _ := json.Marshal(r.Result)
-		//fmt.Println(string(data))
+		// data, _ := json.Marshal(r.Result)
+		// fmt.Println(string(data))
 
 		for _, metric := range *r.Result.Value {
 			if metric.Timeseries != nil {
@@ -203,6 +207,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu
 								channel <- r.buildMetric(
 									metricLabels,
 									*timeseriesData.Total,
+									timeseriesData.TimeStamp.Time,
 								)
 							}
 
@@ -211,6 +216,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu
 								channel <- r.buildMetric(
 									metricLabels,
 									*timeseriesData.Minimum,
+									timeseriesData.TimeStamp.Time,
 								)
 							}
 
@@ -219,6 +225,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu
 								channel <- r.buildMetric(
 									metricLabels,
 									*timeseriesData.Maximum,
+									timeseriesData.TimeStamp.Time,
 								)
 							}
 
@@ -227,6 +234,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu
 								channel <- r.buildMetric(
 									metricLabels,
 									*timeseriesData.Average,
+									timeseriesData.TimeStamp.Time,
 								)
 							}
 
@@ -235,6 +243,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu
 								channel <- r.buildMetric(
 									metricLabels,
 									*timeseriesData.Count,
+									timeseriesData.TimeStamp.Time,
 								)
 							}
 						}

metrics/metrics.go

@@ -1,6 +1,11 @@
 package metrics
 
-import "github.com/prometheus/client_golang/prometheus"
+import (
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/prometheus/client_golang/prometheus"
+)
 
 const (
 	MetricHelpDefault = "Azure monitor insight metric"
@@ -13,8 +18,9 @@ type (
 	}
 
 	MetricRow struct {
-		Labels prometheus.Labels
-		Value  float64
+		Labels    prometheus.Labels
+		Value     float64
+		Timestamp time.Time
 	}
 )
 
@@ -55,6 +61,30 @@ func (l *MetricList) GetMetricList(name string) []MetricRow {
 	return l.List[name]
 }
 
+func (l *MetricList) GetUniqueMetricList(name string) []MetricRow {
+	rows := []MetricRow{}
+
+	for _, row := range l.List[name] {
+		existed := false
+
+		for idx, existedRow := range rows {
+			if cmp.Equal(row.Labels, existedRow.Labels) {
+				existed = true
+
+				if row.Timestamp.After(existedRow.Timestamp) {
+					rows[idx] = row
+				}
+			}
+		}
+
+		if !existed {
+			rows = append(rows, row)
+		}
+	}
+
+	return rows
+}
+
 func (l *MetricList) GetMetricLabelNames(name string) []string {
 	var list []string
 	uniqueLabelMap := map[string]string{}

metrics/prober.go

@@ -211,15 +211,21 @@ func (p *MetricProber) collectMetricsFromTargets() {
 
 	for result := range metricsChannel {
 		metric := MetricRow{
-			Labels: result.Labels,
-			Value:  result.Value,
+			Labels:    result.Labels,
+			Value:     result.Value,
+			Timestamp: result.Timestamp,
 		}
 		p.metricList.Add(result.Name, metric)
 		p.metricList.SetMetricHelp(result.Name, result.Help)
 	}
 }
 
 func (p *MetricProber) publishMetricList() {
+	if p.Conf.Metrics.SetTimestamp {
+		p.prometheus.registry.MustRegister(NewMetricListCollector(p.metricList))
+		return
+	}
+
 	if p.metricList == nil {
 		return
 	}