[metricbeats][mongodb] handle extra collstats metrics

CHANGELOG.next.asciidoc

@@ -432,6 +432,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Collect .NET CLR (IIS) Memory, Exceptions and LocksAndThreads metrics {pull}41929[41929]
 - Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
 - Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
+- Add support of additional `collstats` metrics in mongodb module. {pull}42171[42171]
 
 *Metricbeat*
 - Add benchmark module {pull}41801[41801]

metricbeat/docs/fields.asciidoc

@@ -50999,6 +50999,87 @@ type: long
 Number of database commands executed.
 
 
+type: long
+
+--
+
+
+*`mongodb.collstats.stats.stats.size`*::
++
+--
+The total uncompressed size in memory of all records in a collection.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.count`*::
++
+--
+The number of objects or documents in this collection.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.avgObjSize`*::
++
+--
+The average size of an object in the collection.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.storageSize`*::
++
+--
+The total amount of storage allocated to this collection for document storage.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.totalIndexSize`*::
++
+--
+The total size of all indexes.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.totalSize`*::
++
+--
+The sum of the storageSize and totalIndexSize.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.max`*::
++
+--
+Shows the maximum number of documents that may be present in a capped collection.
+
+
+type: long
+
+--
+
+*`mongodb.collstats.stats.stats.nindexes`*::
++
+--
+The number of indexes on the collection. All collections have at least one index on the _id field.
+
+
 type: long
 
 --

metricbeat/module/mongodb/collstats/_meta/data.json

@@ -69,11 +69,21 @@
                 "time": {
                     "us": 0
                 }
+            },
+            "stats": {
+                "totalSize": 8192,
+                "max": 5000,
+                "nindexes": 1,
+                "size": 36,
+                "count": 1,
+                "avgObjSize": 36,
+                "storageSize": 4096,
+                "totalIndexSize": 4096
             }
         }
     },
     "service": {
         "address": "172.28.0.5:27017",
         "type": "mongodb"
     }
-}
+}

metricbeat/module/mongodb/collstats/_meta/fields.yml

@@ -102,3 +102,39 @@
       type: long
       description: >
         Number of database commands executed.
+
+    - name: stats
+      type: group
+      fields:
+        - name: stats.size
+          type: long
+          description: >
+            The total uncompressed size in memory of all records in a collection.
+        - name: stats.count
+          type: long
+          description: >
+            The number of objects or documents in this collection.
+        - name: stats.avgObjSize
+          type: long
+          description: >
+            The average size of an object in the collection.
+        - name: stats.storageSize
+          type: long
+          description: >
+            The total amount of storage allocated to this collection for document storage.
+        - name: stats.totalIndexSize
+          type: long
+          description: >
+            The total size of all indexes.
+        - name: stats.totalSize
+          type: long
+          description: >
+            The sum of the storageSize and totalIndexSize.
+        - name: stats.max
+          type: long
+          description: >
+            Shows the maximum number of documents that may be present in a capped collection.
+        - name: stats.nindexes
+          type: long
+          description: >
+            The number of indexes on the collection. All collections have at least one index on the _id field.

metricbeat/module/mongodb/collstats/collstats.go

@@ -21,11 +21,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/module/mongodb"
 
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
 )
 
 func init() {
@@ -70,10 +72,6 @@ func (m *Metricset) Fetch(reporter mb.ReporterV2) error {
 		}
 	}()
 
-	if err != nil {
-		return fmt.Errorf("could not get a list of databases: %w", err)
-	}
-
 	// This info is only stored in 'admin' database
 	db := client.Database("admin")
 	res := db.RunCommand(context.Background(), bson.D{bson.E{Key: "top"}})
@@ -95,6 +93,12 @@ func (m *Metricset) Fetch(reporter mb.ReporterV2) error {
 		return errors.New("collection 'totals' are not a map")
 	}
 
+	if err = res.Err(); err != nil {
+		return fmt.Errorf("'top' command failed: %w", err)
+	}
+
+	wg := &sync.WaitGroup{}
+
 	for group, info := range totals {
 		if group == "note" {
 			continue
@@ -106,16 +110,48 @@ func (m *Metricset) Fetch(reporter mb.ReporterV2) error {
 			continue
 		}
 
-		event, err := eventMapping(group, infoMap)
-		if err != nil {
-			reporter.Error(fmt.Errorf("mapping of the event data filed: %w", err))
-			continue
-		}
-
-		reporter.Event(mb.Event{
-			MetricSetFields: event,
-		})
+		wg.Add(1)
+		go func(eventReporter mb.ReporterV2, mongoClient *mongo.Client, group string) {
+			defer wg.Done()
+
+			names, err := splitKey(group)
+			if err != nil {
+				eventReporter.Error(fmt.Errorf("splitting a collection key failed: %w", err))
+				return
+			}
+
+			collStats, err := fetchCollStats(mongoClient, names[0], names[1])
+			if err != nil {
+				eventReporter.Error(fmt.Errorf("fetching collStats failed: %w", err))
+				return
+			}
+
+			infoMap["stats"] = collStats
+
+			event, err := eventMapping(group, infoMap)
+			if err != nil {
+				eventReporter.Error(fmt.Errorf("mapping of the event data failed: %w", err))
+				return
+			}
+
+			eventReporter.Event(mb.Event{
+				MetricSetFields: event,
+			})
+		}(reporter, client, group)
 	}
 
+	wg.Wait()
+
 	return nil
 }
+
+func fetchCollStats(client *mongo.Client, dbName, collectionName string) (map[string]interface{}, error) {
+	db := client.Database(dbName)
+	colStats := db.RunCommand(context.Background(), bson.M{"collStats": collectionName})
+	var statsRes map[string]interface{}
+	if err := colStats.Decode(&statsRes); err != nil {
+		return nil, fmt.Errorf("could not decode mongo response for database=%s, collection=%s: %w", dbName, collectionName, err)
+	}
+
+	return statsRes, nil
+}

metricbeat/module/mongodb/collstats/data.go

@@ -25,10 +25,9 @@ import (
 )
 
 func eventMapping(key string, data mapstr.M) (mapstr.M, error) {
-	names := strings.SplitN(key, ".", 2)
-
-	if len(names) < 2 {
-		return nil, errors.New("collection name invalid")
+	names, err := splitKey(key)
+	if err != nil {
+		return nil, err
 	}
 
 	event := mapstr.M{
@@ -91,6 +90,16 @@ func eventMapping(key string, data mapstr.M) (mapstr.M, error) {
 			},
 			"count": mustGetMapStrValue(data, "commands.count"),
 		},
+		"stats": mapstr.M{
+			"size":           mustGetMapStrValue(data, "stats.size"),
+			"count":          mustGetMapStrValue(data, "stats.count"),
+			"avgObjSize":     mustGetMapStrValue(data, "stats.avgObjSize"),
+			"storageSize":    mustGetMapStrValue(data, "stats.storageSize"),
+			"totalIndexSize": mustGetMapStrValue(data, "stats.totalIndexSize"),
+			"totalSize":      mustGetMapStrValue(data, "stats.totalSize"),
+			"max":            mustGetMapStrValue(data, "stats.max"),
+			"nindexes":       mustGetMapStrValue(data, "stats.nindexes"),
+		},
 	}
 
 	return event, nil
@@ -100,3 +109,13 @@ func mustGetMapStrValue(m mapstr.M, key string) interface{} {
 	v, _ := m.GetValue(key)
 	return v
 }
+
+func splitKey(key string) ([]string, error) {
+	names := strings.SplitN(key, ".", 2)
+
+	if len(names) < 2 {
+		return nil, errors.New("collection name invalid")
+	}
+
+	return names, nil
+}