-
Notifications
You must be signed in to change notification settings - Fork 41
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Vyom Yadav <jackhammervyom@gmail.com>
- Loading branch information
1 parent
a6d9b22
commit ccde7be
Showing
5 changed files
with
242 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2024 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| // Package metrics provides metrics for the reminder service | ||
| package metrics | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "go.opentelemetry.io/otel/metric" | ||
| ) | ||
|
|
||
| // Default bucket boundaries in seconds for the delay histograms | ||
| var delayBuckets = []float64{ | ||
| 60, // 1 minute | ||
| 300, // 5 minutes | ||
| 600, // 10 minutes | ||
| 1800, // 30 minutes | ||
| 3600, // 1 hour | ||
| 7200, // 2 hours | ||
| 10800, // 3 hours | ||
| 18000, // 5 hours | ||
| 25200, // 7 hours | ||
| 36000, // 10 hours | ||
| } | ||
|
|
||
| // Metrics contains all the metrics for the reminder service | ||
| type Metrics struct { | ||
| // Time between when a reminder became eligible and when it was sent | ||
| SendDelay metric.Float64Histogram | ||
|
|
||
| // Time between when a reminder became eligible and when it was sent | ||
| NewSendDelay metric.Float64Histogram | ||
|
|
||
| // Current number of reminders in the batch | ||
| BatchSize metric.Int64Histogram | ||
|
|
||
| // Total number of batches processed | ||
| TotalBatches metric.Int64Counter | ||
|
|
||
| // Total number of reminders processed (total entities reconciled) | ||
| TotalReminders metric.Int64Counter | ||
| } | ||
|
|
||
| // NewMetrics creates a new metrics instance | ||
| func NewMetrics(meter metric.Meter) (*Metrics, error) { | ||
| sendDelay, err := meter.Float64Histogram( | ||
| "send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds)"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| newSendDelay, err := meter.Float64Histogram( | ||
| "new_send_delay", | ||
| metric.WithDescription("Time between reminder becoming eligible and actual send (seconds) for first time reminders"), | ||
| metric.WithUnit("s"), | ||
| metric.WithExplicitBucketBoundaries(delayBuckets...), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| batchSize, err := meter.Int64Histogram( | ||
| "batch_size", | ||
| metric.WithDescription("Current number of reminders in the batch"), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| totalBatches, err := meter.Int64Counter( | ||
| "total_batches", | ||
| metric.WithDescription("Total number of batches processed"), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| totalReminders, err := meter.Int64Counter( | ||
| "total_reminders", | ||
| metric.WithDescription("Total number of reminders processed"), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &Metrics{ | ||
| SendDelay: sendDelay, | ||
| NewSendDelay: newSendDelay, | ||
| BatchSize: batchSize, | ||
| TotalBatches: totalBatches, | ||
| TotalReminders: totalReminders, | ||
| }, nil | ||
| } | ||
|
|
||
| // RecordBatch records the metrics for a batch of reminders | ||
| func (m *Metrics) RecordBatch(ctx context.Context, size int64) { | ||
| m.BatchSize.Record(ctx, size) | ||
| m.TotalBatches.Add(ctx, 1) | ||
| m.TotalReminders.Add(ctx, size) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| // SPDX-FileCopyrightText: Copyright 2024 The Minder Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package reminder | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus/promhttp" | ||
| "github.com/rs/zerolog" | ||
| "go.opentelemetry.io/otel" | ||
| "go.opentelemetry.io/otel/exporters/prometheus" | ||
| sdkmetric "go.opentelemetry.io/otel/sdk/metric" | ||
| "go.opentelemetry.io/otel/sdk/resource" | ||
| semconv "go.opentelemetry.io/otel/semconv/v1.17.0" | ||
| ) | ||
|
|
||
| const ( | ||
| metricsPath = "/metrics" | ||
| readHeaderTimeout = 2 * time.Second | ||
| ) | ||
|
|
||
| func (r *reminder) startMetricServer(ctx context.Context) error { | ||
| logger := zerolog.Ctx(ctx) | ||
|
|
||
| prometheusExporter, err := prometheus.New( | ||
| prometheus.WithNamespace("reminder"), | ||
| ) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create Prometheus exporter: %w", err) | ||
| } | ||
|
|
||
| res := resource.NewWithAttributes( | ||
| semconv.SchemaURL, | ||
| semconv.ServiceName("reminder"), | ||
| // TODO: Make this auto-generated | ||
| semconv.ServiceVersion("v0.1.0"), | ||
| ) | ||
|
|
||
| mp := sdkmetric.NewMeterProvider( | ||
| sdkmetric.WithReader(prometheusExporter), | ||
| sdkmetric.WithResource(res), | ||
| ) | ||
|
|
||
| otel.SetMeterProvider(mp) | ||
|
|
||
| mux := http.NewServeMux() | ||
| mux.Handle(metricsPath, promhttp.Handler()) | ||
|
|
||
| server := &http.Server{ | ||
| Addr: r.cfg.MetricServer.GetAddress(), | ||
| Handler: mux, | ||
| ReadHeaderTimeout: readHeaderTimeout, | ||
| } | ||
|
|
||
| logger.Info().Msgf("starting metrics server on %s", server.Addr) | ||
|
|
||
| errCh := make(chan error) | ||
| go func() { | ||
| errCh <- server.ListenAndServe() | ||
| }() | ||
|
|
||
| select { | ||
| case err := <-errCh: | ||
| return err | ||
| case <-ctx.Done(): | ||
| case <-r.stop: | ||
| } | ||
|
|
||
| // shutdown the metrics server when either the context is done or when reminder is stopped | ||
| shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 5*time.Second) | ||
| defer shutdownRelease() | ||
|
|
||
| logger.Info().Msg("shutting down metrics server") | ||
|
|
||
| if err := mp.Shutdown(shutdownCtx); err != nil { | ||
| logger.Err(err).Msg("error shutting down metrics provider") | ||
| } | ||
|
|
||
| return server.Shutdown(shutdownCtx) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters