Aggregate overhead metrics in userspace before reporting

This is done to avoid duplicate metrics, that would cause the entire metrics
collection job to fail.

Signed-off-by: Anna Kapuscinska <anna@isovalent.com>

pkg/metrics/overhead/collector.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cilium/tetragon/pkg/logger"
 	"github.com/cilium/tetragon/pkg/metrics"
 	"github.com/cilium/tetragon/pkg/observer"
+	"github.com/cilium/tetragon/pkg/sensors"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -34,9 +35,19 @@ func collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
+	// Aggregate metrics before reporting, to avoid duplicates that would cause
+	// the entire metrics collection job to fail.
+	times := map[sensors.Prog]uint64{}
+	counts := map[sensors.Prog]uint64{}
 	for _, ovh := range overheads {
-		ch <- time.MustMetric(float64(ovh.RunTime), ovh.Namespace, ovh.Policy, ovh.Sensor, ovh.Attach, ovh.Label)
-		ch <- runs.MustMetric(float64(ovh.RunCnt), ovh.Namespace, ovh.Policy, ovh.Sensor, ovh.Attach, ovh.Label)
+		times[ovh.Prog] += ovh.RunTime
+		counts[ovh.Prog] += ovh.RunCnt
+	}
+	for prog, m := range times {
+		ch <- time.MustMetric(float64(m), prog.Namespace, prog.Policy, prog.Sensor, prog.Attach, prog.Label)
+	}
+	for prog, m := range counts {
+		ch <- runs.MustMetric(float64(m), prog.Namespace, prog.Policy, prog.Sensor, prog.Attach, prog.Label)
 	}
 }
 

pkg/metrics/overhead/overhead.go

@@ -15,9 +15,9 @@ var (
 		nil, nil, []metrics.UnconstrainedLabel{
 			metrics.LabelPolicyNamespace,
 			metrics.LabelPolicy,
-			metrics.UnconstrainedLabel{Name: "sensor", ExampleValue: "generic_kprobe"},
-			metrics.UnconstrainedLabel{Name: "attach", ExampleValue: "sys_open"},
-			metrics.UnconstrainedLabel{Name: "section", ExampleValue: "kprobe/sys_open"},
+			{Name: "sensor", ExampleValue: "generic_kprobe"},
+			{Name: "attach", ExampleValue: "sys_open"},
+			{Name: "section", ExampleValue: "kprobe/sys_open"},
 		},
 	))
 
@@ -27,9 +27,9 @@ var (
 		nil, nil, []metrics.UnconstrainedLabel{
 			metrics.LabelPolicyNamespace,
 			metrics.LabelPolicy,
-			metrics.UnconstrainedLabel{Name: "sensor", ExampleValue: "generic_kprobe"},
-			metrics.UnconstrainedLabel{Name: "attach", ExampleValue: "sys_open"},
-			metrics.UnconstrainedLabel{Name: "section", ExampleValue: "kprobe/sys_open"},
+			{Name: "sensor", ExampleValue: "generic_kprobe"},
+			{Name: "attach", ExampleValue: "sys_open"},
+			{Name: "section", ExampleValue: "kprobe/sys_open"},
 		},
 	))
 )

pkg/sensors/sensors.go

@@ -93,14 +93,18 @@ func sanitize(name string) string {
 	return strings.ReplaceAll(name, "/", "_")
 }
 
-type ProgOverhead struct {
+type Prog struct {
 	Namespace string
 	Policy    string
 	Sensor    string
 	Attach    string
 	Label     string
-	RunTime   uint64
-	RunCnt    uint64
+}
+
+type ProgOverhead struct {
+	Prog
+	RunTime uint64
+	RunCnt  uint64
 }
 
 // SensorIface is an interface for sensors.Sensor that allows implementing sensors for testing.
@@ -131,9 +135,11 @@ func (s *Sensor) Overhead() ([]ProgOverhead, bool) {
 		runCnt, _ := info.RunCount()
 
 		list = append(list, ProgOverhead{
-			Attach:  p.Attach,
-			Label:   p.Label,
-			Sensor:  s.Name,
+			Prog: Prog{
+				Attach: p.Attach,
+				Label:  p.Label,
+				Sensor: s.Name,
+			},
 			RunTime: uint64(runTime),
 			RunCnt:  runCnt,
 		})