Adds Some Prometheus Metrics To Lookout (#3781)

* Better metrics for lookout Signed-off-by: Chris Martin <chris@cmartinit.co.uk> * lint Signed-off-by: Chris Martin <chris@cmartinit.co.uk> * add service monitor Signed-off-by: Chris Martin <chris@cmartinit.co.uk> --------- Signed-off-by: Chris Martin <chris@cmartinit.co.uk> Co-authored-by: Chris Martin <chris@cmartinit.co.uk>
armadaproject · Jul 12, 2024 · 6da3fbd · 6da3fbd
1 parent f6f44b0
commit 6da3fbd
Show file tree

Hide file tree

Showing 8 changed files with 73 additions and 3 deletions.
diff --git a/config/lookoutv2/config.yaml b/config/lookoutv2/config.yaml
@@ -1,4 +1,5 @@
 apiPort: 10000
+metricsPort: 9003
 corsAllowedOrigins:
   - "http://localhost:3000"
   - "http://localhost:8089"

diff --git a/deployment/lookout-v2/templates/servicemonitor.yaml b/deployment/lookout-v2/templates/servicemonitor.yaml
@@ -0,0 +1,20 @@
+{{- if .Values.prometheus.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "lookout_v2.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "lookout_v2.name.labels.all" . | nindent 4 -}}
+    {{- if .Values.prometheus.labels }}
+    {{- toYaml .Values.prometheus.labels | nindent 4 -}}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "lookout_v2.name.labels.identity" . | nindent 6 }}
+  endpoints:
+    - port: metrics
+      interval: {{ .Values.prometheus.scrapeInterval }}
+      scrapeTimeout: {{ .Values.prometheus.scrapeTimeout }}
+{{- end }}
diff --git a/go.mod b/go.mod
@@ -65,6 +65,7 @@ require (
 )
 
 require (
+	github.com/IBM/pgxpoolprometheus v1.1.1
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/benbjohnson/immutable v0.4.3
 	github.com/charmbracelet/glamour v0.7.0

diff --git a/go.sum b/go.sum
@@ -35,6 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
 github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
+github.com/IBM/pgxpoolprometheus v1.1.1 h1:xkWNUe87TIuBj/ypdSiDgNYktsuM7MoZCT8a+kjhh2s=
+github.com/IBM/pgxpoolprometheus v1.1.1/go.mod h1:GFJDkHbidFfB2APbhBTSy2X4PKH3bLWsEMBhmzK1ipo=
 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink=

diff --git a/internal/lookoutv2/application.go b/internal/lookoutv2/application.go
@@ -3,15 +3,18 @@
 package lookoutv2
 
 import (
+	"github.com/IBM/pgxpoolprometheus"
 	"github.com/go-openapi/loads"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/jessevdk/go-flags"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
+	"github.com/armadaproject/armada/internal/common"
 	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/compress"
 	"github.com/armadaproject/armada/internal/common/database"
-	slices "github.com/armadaproject/armada/internal/common/slices"
+	"github.com/armadaproject/armada/internal/common/slices"
 	"github.com/armadaproject/armada/internal/lookoutv2/configuration"
 	"github.com/armadaproject/armada/internal/lookoutv2/conversions"
 	"github.com/armadaproject/armada/internal/lookoutv2/gen/restapi"
@@ -31,6 +34,9 @@ func Serve(configuration configuration.LookoutV2Config) error {
 		return err
 	}
 
+	collector := pgxpoolprometheus.NewCollector(db, map[string]string{})
+	prometheus.MustRegister(collector)
+
 	getJobsRepo := repository.NewSqlGetJobsRepository(db)
 	groupJobsRepo := repository.NewSqlGroupJobsRepository(db)
 	decompressor := compress.NewThreadSafeZlibDecompressor()
@@ -148,6 +154,9 @@ func Serve(configuration configuration.LookoutV2Config) error {
 		},
 	)
 
+	shutdownMetricServer := common.ServeMetrics(uint16(configuration.MetricsPort))
+	defer shutdownMetricServer()
+
 	server := restapi.NewServer(api)
 	defer func() {
 		shutdownErr := server.Shutdown()

diff --git a/internal/lookoutv2/configuration/types.go b/internal/lookoutv2/configuration/types.go
@@ -7,7 +7,8 @@ import (
 )
 
 type LookoutV2Config struct {
-	ApiPort int
+	ApiPort     int
+	MetricsPort int
 	// If non-nil, net/http/pprof endpoints are exposed on localhost on this port.
 	PprofPort *uint16
 

diff --git a/internal/lookoutv2/gen/restapi/configure_lookout.go b/internal/lookoutv2/gen/restapi/configure_lookout.go
@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/go-openapi/errors"
 	"github.com/go-openapi/runtime"
@@ -16,6 +17,7 @@ import (
 	"github.com/armadaproject/armada/internal/common/serve"
 	"github.com/armadaproject/armada/internal/lookoutv2/configuration"
 	"github.com/armadaproject/armada/internal/lookoutv2/gen/restapi/operations"
+	"github.com/armadaproject/armada/internal/lookoutv2/metrics"
 )
 
 //go:generate swagger generate server --target ../../gen --name Lookout --spec ../../swagger.yaml --principal interface{} --exclude-main
@@ -89,7 +91,7 @@ var UIConfig configuration.UIConfig
 // The middleware configuration happens before anything, this middleware also applies to serving the swagger.json document.
 // So this is a good place to plug in a panic handling middleware, logging and metrics.
 func setupGlobalMiddleware(apiHandler http.Handler) http.Handler {
-	return allowCORS(uiHandler(apiHandler), corsAllowedOrigins)
+	return recordRequestDuration(allowCORS(uiHandler(apiHandler), corsAllowedOrigins))
 }
 
 func uiHandler(apiHandler http.Handler) http.Handler {
@@ -137,6 +139,19 @@ func allowCORS(handler http.Handler, corsAllowedOrigins []string) http.Handler {
 	})
 }
 
+func recordRequestDuration(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// TODO: for autheticated users, record the username
+		const unknownUser = "unknown"
+		start := time.Now()
+		handler.ServeHTTP(w, r)
+		duration := time.Since(start)
+		if strings.HasPrefix(r.URL.Path, "/api/v1/") {
+			metrics.RecordRequestDuration(unknownUser, r.URL.Path, float64(duration.Milliseconds()))
+		}
+	})
+}
+
 func preflightHandler(w http.ResponseWriter) {
 	headers := []string{"Content-Type", "Accept", "Authorization"}
 	w.Header().Set("Access-Control-Allow-Headers", strings.Join(headers, ","))

diff --git a/internal/lookoutv2/metrics/metrics.go b/internal/lookoutv2/metrics/metrics.go
@@ -0,0 +1,21 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var requestDuration = promauto.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name:    "lookout_request_duration_ms",
+		Help:    "Request duration in milliseconds",
+		Buckets: []float64{1, 10, 100, 1000, 10000, 100000, 1000000},
+	},
+	[]string{"user", "endpoint"},
+)
+
+func RecordRequestDuration(user, endpoint string, duration float64) {
+	requestDuration.
+		With(map[string]string{"user": user, "endpoint": endpoint}).
+		Observe(duration)
+}