[Metricbeat][vSphere] New metrics support and minor changes to existi…

…ng metricsets (#40766) * initial commit * update field mappings * update virtual machine data.json * format data.json * resolve review comments * add changelog entry (cherry picked from commit 3f44bd1) # Conflicts: # metricbeat/module/vsphere/datastore/_meta/data.json # metricbeat/module/vsphere/datastore/datastore.go # metricbeat/module/vsphere/fields.go # metricbeat/module/vsphere/host/_meta/data.json # metricbeat/module/vsphere/host/host.go # metricbeat/module/vsphere/virtualmachine/_meta/data.json # metricbeat/module/vsphere/virtualmachine/data_test.go # metricbeat/module/vsphere/virtualmachine/virtualmachine.go
elastic · Sep 13, 2024 · 4b6842e · 4b6842e
1 parent 3103a8a
commit 4b6842e
Show file tree

Hide file tree

Showing 15 changed files with 474 additions and 118 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -328,6 +328,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset network for the vSphere module. {pull}40559[40559]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
 - Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661]
+- Add new metricset datastorecluster for vSphere module. {pull}40634[40634] {pull}40694[40694]
+- Add new metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]
+- Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683]
+- Update fields to use mapstr in vSphere virtualmachine metricset  {pull}40707[40707]
+- Add support for period based intervalID in vSphere host and datastore metricsets {pull}40678[40678]
+- Add new metrics fot datastore and minor changes to overall vSphere metrics {pull}40766[40766]
+- Add `metrics_count` to Prometheus module if `metrics_count: true` is set. {pull}40411[40411]
 
 *Metricbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -67010,76 +67010,92 @@ format: percent
 
 --
 
-*`vsphere.datastore.fstype`*::
+*`vsphere.datastore.disk.capacity.bytes`*::
 +
 --
-Filesystem type.
+Configured size of the datastore.
 
 
-type: keyword
+type: long
+
+format: bytes
 
 --
 
-*`vsphere.datastore.host.count`*::
+*`vsphere.datastore.disk.capacity.usage.bytes`*::
 +
 --
-Number of hosts.
+The amount of storage capacity currently being consumed by datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
-*`vsphere.datastore.host.names`*::
+*`vsphere.datastore.disk.provisioned.bytes`*::
 +
 --
-List of all the host names.
+Amount of storage set aside for use by a datastore.
+
+
+type: long
+
+format: bytes
+
+--
+
+*`vsphere.datastore.fstype`*::
++
+--
+Filesystem type.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.iops`*::
+*`vsphere.datastore.host.count`*::
 +
 --
-Storage I/O Control aggregated Input/Output Operations Per Second.
+Number of hosts.
 
 
 type: long
 
 --
 
-*`vsphere.datastore.name`*::
+*`vsphere.datastore.host.names`*::
 +
 --
-Datastore name.
+List of all the host names.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.read.bytes`*::
+*`vsphere.datastore.name`*::
 +
 --
-Rate of reading data from the datastore.
-
+Datastore name.
 
-type: long
 
-format: bytes
+type: keyword
 
 --
 
-*`vsphere.datastore.read.latency.total.ms`*::
+*`vsphere.datastore.read.bytes`*::
 +
 --
-Average amount of time for a read operation from the datastore in milliseconds.
+Rate of reading data from the datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
 *`vsphere.datastore.status`*::
@@ -67124,16 +67140,6 @@ format: bytes
 
 --
 
-*`vsphere.datastore.write.latency.total.ms`*::
-+
---
-Average amount of time for a write operation from the datastore in milliseconds.
-
-
-type: long
-
---
-
 [float]
 === datastorecluster
 
@@ -68115,10 +68121,10 @@ type: long
 --
 
 
-*`vsphere.virtualmachine.snapshot.info`*::
+*`vsphere.virtualmachine.snapshot.info.*`*::
 +
 --
-Deatils of the snapshots of this virtualmachine.
+Details of the snapshots of this virtualmachine.
 
 type: object
 

diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json
@@ -1,4 +1,5 @@
 {
+<<<<<<< HEAD
   "@timestamp": "2017-10-12T08:05:34.853Z",
   "event": {
     "dataset": "vsphere.datastore",
@@ -35,6 +36,68 @@
           "total": {
             "ms": 0
           }
+=======
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "event": {
+        "dataset": "vsphere.datastore",
+        "duration": 115000,
+        "module": "vsphere"
+    },
+    "metricset": {
+        "name": "datastore",
+        "period": 10000
+    },
+    "service": {
+        "address": "127.0.0.1:33365",
+        "type": "vsphere"
+    },
+    "vsphere": {
+        "datastore": {
+            "host": {
+                "count": 1,
+                "names": [
+                    "DC3_H0"
+                ]
+            },
+            "status": "green",
+            "vm": {
+                "count": 6,
+                "names": [
+                    "DC3_H0_VM0"
+                ]
+            },
+            "read": {
+                "bytes": 0
+            },
+            "write": {
+                "bytes": 337000
+            },
+            "disk": {
+                "capacity": {
+                    "usage": {
+                        "bytes": 520505786368
+                    },
+                    "bytes": 1610344300544
+                },
+                "provisioned": {
+                    "bytes": 520505786368
+                }
+            },
+            "capacity": {
+                "free": {
+                    "bytes": 37120094208
+                },
+                "total": {
+                    "bytes": 74686664704
+                },
+                "used": {
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
+                }
+            },
+            "fstype": "local",
+            "name": "LocalDS_0"
+>>>>>>> 3f44bd1f9b ([Metricbeat][vSphere] New metrics support and minor changes to existing metricsets (#40766))
         }
       },
       "write": {

diff --git a/metricbeat/module/vsphere/datastore/_meta/fields.yml b/metricbeat/module/vsphere/datastore/_meta/fields.yml
@@ -24,6 +24,21 @@
       description: >
         Percentage of datastore capacity used.
       format: percent
+    - name: disk.capacity.bytes
+      type: long
+      description: >
+        Configured size of the datastore.
+      format: bytes
+    - name: disk.capacity.usage.bytes
+      type: long
+      description: >
+        The amount of storage capacity currently being consumed by datastore.
+      format: bytes
+    - name: disk.provisioned.bytes
+      type: long
+      description: >
+        Amount of storage set aside for use by a datastore.
+      format: bytes
     - name: fstype
       type: keyword
       description: >
@@ -36,10 +51,6 @@
       type: keyword
       description: >
         List of all the host names.
-    - name: iops
-      type: long
-      description: >
-        Storage I/O Control aggregated Input/Output Operations Per Second.
     - name: name
       type: keyword
       description: >
@@ -49,10 +60,6 @@
       description: >
         Rate of reading data from the datastore.
       format: bytes
-    - name: read.latency.total.ms
-      type: long
-      description: >
-        Average amount of time for a read operation from the datastore in milliseconds.
     - name: status
       type: keyword
       description: >
@@ -70,8 +77,4 @@
       description: >
         Rate of writing data to the datastore.
       format: bytes
-    - name: write.latency.total.ms
-      type: long
-      description: >
-        Average amount of time for a write operation from the datastore in milliseconds.
 
diff --git a/metricbeat/module/vsphere/datastore/data.go b/metricbeat/module/vsphere/datastore/data.go
@@ -65,16 +65,16 @@ func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{})
 	if val, exist := perfMetricMap["datastore.read.average"]; exist {
 		event.Put("read.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.totalReadLatency.average"]; exist {
-		event.Put("read.latency.total.ms", val)
-	}
 	if val, exist := perfMetricMap["datastore.write.average"]; exist {
 		event.Put("write.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.totalWriteLatency.average"]; exist {
-		event.Put("write.latency.total.ms", val)
+	if val, exist := perfMetricMap["disk.capacity.latest"]; exist {
+		event.Put("disk.capacity.bytes", val.(int64)*bytesMultiplier)
+	}
+	if val, exist := perfMetricMap["disk.capacity.usage.average"]; exist {
+		event.Put("disk.capacity.usage.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.datastoreIops.average"]; exist {
-		event.Put("iops", val)
+	if val, exist := perfMetricMap["disk.provisioned.latest"]; exist {
+		event.Put("disk.provisioned.bytes", val.(int64)*bytesMultiplier)
 	}
 }
diff --git a/metricbeat/module/vsphere/datastore/data_test.go b/metricbeat/module/vsphere/datastore/data_test.go
@@ -28,7 +28,6 @@ import (
 )
 
 func TestEventMapping(t *testing.T) {
-	var m *DataStoreMetricSet
 	var datastoreTest = mo.Datastore{
 		Summary: types.DatastoreSummary{
 			Name:      "datastore-test",
@@ -47,23 +46,22 @@ func TestEventMapping(t *testing.T) {
 
 	var metricDataTest = metricData{
 		perfMetrics: map[string]interface{}{
-			"datastore.read.average":              int64(100),
-			"datastore.write.average":             int64(200),
-			"datastore.datastoreIops.average":     int64(10),
-			"datastore.totalReadLatency.average":  int64(100),
-			"datastore.totalWriteLatency.average": int64(100),
+			"datastore.read.average":      int64(100),
+			"datastore.write.average":     int64(200),
+			"disk.capacity.latest":        int64(10000),
+			"disk.capacity.usage.average": int64(5000),
+			"disk.provisioned.latest":     int64(5000),
 		},
 		assetNames: assetNames{
 			outputHostNames: []string{"DC3_H0"},
 			outputVmNames:   []string{"DC3_H0_VM0"},
 		},
 	}
 
-	outputEvent := m.mapEvent(datastoreTest, &metricDataTest)
+	outputEvent := (&DataStoreMetricSet{}).mapEvent(datastoreTest, &metricDataTest)
 	testEvent := mapstr.M{
 		"fstype": "local",
 		"status": types.ManagedEntityStatus("green"),
-		"iops":   int64(10),
 		"name":   "datastore-test",
 		"host": mapstr.M{
 			"count": 1,
@@ -75,19 +73,20 @@ func TestEventMapping(t *testing.T) {
 		},
 		"read": mapstr.M{
 			"bytes": int64(102400),
-			"latency": mapstr.M{
-				"total": mapstr.M{
-					"ms": int64(100),
-				},
-			},
 		},
 		"write": mapstr.M{
 			"bytes": int64(204800),
-			"latency": mapstr.M{
-				"total": mapstr.M{
-					"ms": int64(100),
+		},
+		"disk": mapstr.M{
+			"capacity": mapstr.M{
+				"bytes": int64(10240000),
+				"usage": mapstr.M{
+					"bytes": int64(5120000),
 				},
 			},
+			"provisioned": mapstr.M{
+				"bytes": int64(5120000),
+			},
 		},
 		"capacity": mapstr.M{
 			"free": mapstr.M{
@@ -104,5 +103,4 @@ func TestEventMapping(t *testing.T) {
 	}
 
 	assert.Exactly(t, outputEvent, testEvent)
-
 }