Merge tag 'refs/tags/v2.48.1'

v2.48.1

# -----BEGIN PGP SIGNATURE-----
#
# iQIzBAABCgAdFiEEbrCu5o+YkYe4Rc28I6e96uhgtX0FAmVzpvQACgkQI6e96uhg
# tX1l0w//QJUoOfaTC8ELGpHhkU6N7piFI1/KAl6aiQ38qKNVSXInD7kShRQvhBGA
# yxWc67h3AkdaIulwLDFnFj+aFieH0g5m4V6ft/akGr4pEGorRld5GWGlCqYgHxCs
# 6m9aWb0UewP0yl3xTu3a4zCWgVIdhk9EfPYiwiluFNrfw+SkQu/uF43NdDHKsLxW
# ZnDPhDXX7QwinZet42ripVvfZlbxKd8rbB8roN07yVd0FTX8116DmF5Lo+xQsJjK
# 4h9pd1ZSaWXkDHE9l4WnoS0sQ2fpqESJ8Vsw9zQiCS3F8BJzHSyzHBOVuvar1KgX
# uquj8MJnVQzVtnfd4Tmrol7Kd84owse/DerLx9YlSQVaSL5N96GhPifGTObEUiF5
# xUF+Cqy+2wsFoSqD+Y2fK+wdg2zZ5V1VYwf/0t+VYTQk20tosSyxvRkfCGsrZVl5
# MsgVMSXMuTgNkjuDxtTNTISpgIFBurXfzyi/WXLfuOz+AGATR392FM5DbTWJMym5
# eBVSF59lcf38mHCfAi0illU8EijhoB6OjHe2ljqMCfluawe1Z+XE16+WR2XGqLxp
# FjkYetOFWrG3gkEQ8FvregV8bbzw+QJtMrJk6IflC5g5MfmA/yVgrISQVUwTYjCl
# 1PLZSRQ51YClH0ocEVvp1bO2dOQTAfPhpnJ1bJUXHwk8wc0eugA=
# =Gt2w
# -----END PGP SIGNATURE-----
# gpg: directory '/home/runner/.gnupg' created
# gpg: keybox '/home/runner/.gnupg/pubring.kbx' created
# gpg: Signature made Fri Dec  8 23:29:56 2023 UTC
# gpg:                using RSA key 6EB0AEE68F989187B845CDBC23A7BDEAE860B57D
# gpg: Can't check signature: No public key

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.48.1 / 2023-12-08
+
+* [BUGFIX] TSDB: Make the wlog watcher read segments synchronously when not tailing. #13224
+* [BUGFIX] Agent: Participate in notify calls (fixes slow down in remote write handling introduced in 2.45). #13223
+
 ## 2.48.0 / 2023-11-16
 
 * [CHANGE] Remote-write: respect Retry-After header on 5xx errors. #12677

VERSION

@@ -1 +1 @@
-2.48.0
+2.48.1

cmd/prometheus/main.go

@@ -1112,6 +1112,7 @@ func main() {
 				)
 
 				localStorage.Set(db, 0)
+				db.SetWriteNotified(remoteStorage)
 				close(dbOpen)
 				<-cancel
 				return nil

docs/feature_flags.md

@@ -125,7 +125,61 @@ histogram (albeit via the text format). With this flag enabled, Prometheus will
 still ingest those conventional histograms that do not come with a
 corresponding native histogram. However, if a native histogram is present,
 Prometheus will ignore the corresponding conventional histogram, with the
-notable exception of exemplars, which are always ingested.
+notable exception of exemplars, which are always ingested. To keep the
+conventional histograms as well, enable `scrape_classic_histograms` in the
+scrape job.
+
+_Note about the format of `le` and `quantile` label values:_
+
+In certain situations, the protobuf parsing changes the number formatting of
+the `le` labels of conventional histograms and the `quantile` labels of
+summaries. Typically, this happens if the scraped target is instrumented with
+[client_golang](https://github.com/prometheus/client_golang) provided that
+[promhttp.HandlerOpts.EnableOpenMetrics](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#HandlerOpts)
+is set to `false`. In such a case, integer label values are represented in the
+text format as such, e.g. `quantile="1"` or `le="2"`. However, the protobuf parsing
+changes the representation to float-like (following the OpenMetrics
+specification), so the examples above become `quantile="1.0"` and `le="2.0"` after
+ingestion into Prometheus, which changes the identity of the metric compared to
+what was ingested before via the text format.
+
+The effect of this change is that alerts, recording rules and dashboards that
+directly reference label values as whole numbers such as `le="1"` will stop
+working.
+
+Aggregation by the `le` and `quantile` labels for vectors that contain the old and
+new formatting will lead to unexpected results, and range vectors that span the
+transition between the different formatting will contain additional series.
+The most common use case for both is the quantile calculation via
+`histogram_quantile`, e.g.
+`histogram_quantile(0.95, sum by (le) (rate(histogram_bucket[10m])))`.
+The `histogram_quantile` function already tries to mitigate the effects to some
+extent, but there will be inaccuracies, in particular for shorter ranges that
+cover only a few samples.
+
+Ways to deal with this change either globally or on a per metric basis:
+
+- Fix references to integer `le`, `quantile` label values, but otherwise do
+nothing and accept that some queries that span the transition time will produce
+inaccurate or unexpected results.
+_This is the recommended solution, to get consistently normalized label values._
+Also Prometheus 3.0 is expected to enforce normalization of these label values.
+- Use `metric_relabel_config` to retain the old labels when scraping targets.
+This should **only** be applied to metrics that currently produce such labels.
+
+<!-- The following config snippet is unit tested in scrape/scrape_test.go. -->
+```yaml
+    metric_relabel_configs:
+      - source_labels:
+          - quantile
+        target_label: quantile
+        regex: (\d+)\.0+
+      - source_labels:
+          - le
+          - __name__
+        target_label: le
+        regex: (\d+)\.0+;.*_bucket
+```
 
 ## OTLP Receiver
 

scrape/scrape_test.go

@@ -3629,6 +3629,131 @@ func TestTargetScrapeIntervalAndTimeoutRelabel(t *testing.T) {
 	require.Equal(t, "750ms", sp.ActiveTargets()[0].labels.Get(model.ScrapeTimeoutLabel))
 }
 
+// Testing whether we can remove trailing .0 from histogram 'le' and summary 'quantile' labels.
+func TestLeQuantileReLabel(t *testing.T) {
+	simpleStorage := teststorage.New(t)
+	defer simpleStorage.Close()
+
+	config := &config.ScrapeConfig{
+		JobName: "test",
+		MetricRelabelConfigs: []*relabel.Config{
+			{
+				SourceLabels: model.LabelNames{"le", "__name__"},
+				Regex:        relabel.MustNewRegexp("(\\d+)\\.0+;.*_bucket"),
+				Replacement:  relabel.DefaultRelabelConfig.Replacement,
+				Separator:    relabel.DefaultRelabelConfig.Separator,
+				TargetLabel:  "le",
+				Action:       relabel.Replace,
+			},
+			{
+				SourceLabels: model.LabelNames{"quantile"},
+				Regex:        relabel.MustNewRegexp("(\\d+)\\.0+"),
+				Replacement:  relabel.DefaultRelabelConfig.Replacement,
+				Separator:    relabel.DefaultRelabelConfig.Separator,
+				TargetLabel:  "quantile",
+				Action:       relabel.Replace,
+			},
+		},
+		SampleLimit:    100,
+		Scheme:         "http",
+		ScrapeInterval: model.Duration(100 * time.Millisecond),
+		ScrapeTimeout:  model.Duration(100 * time.Millisecond),
+	}
+
+	metricsText := `
+# HELP test_histogram This is a histogram with default buckets
+# TYPE test_histogram histogram
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.005"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.01"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.025"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.05"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.1"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.25"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="0.5"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="1.0"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="2.5"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="5.0"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="10.0"} 0
+test_histogram_bucket{address="0.0.0.0",port="5001",le="+Inf"} 0
+test_histogram_sum{address="0.0.0.0",port="5001"} 0
+test_histogram_count{address="0.0.0.0",port="5001"} 0
+# HELP test_summary Number of inflight requests sampled at a regular interval. Quantile buckets keep track of inflight requests over the last 60s.
+# TYPE test_summary summary
+test_summary{quantile="0.5"} 0
+test_summary{quantile="0.9"} 0
+test_summary{quantile="0.95"} 0
+test_summary{quantile="0.99"} 0
+test_summary{quantile="1.0"} 1
+test_summary_sum 1
+test_summary_count 199
+`
+
+	// The expected "le" values do not have the trailing ".0".
+	expectedLeValues := []string{"0.005", "0.01", "0.025", "0.05", "0.1", "0.25", "0.5", "1", "2.5", "5", "10", "+Inf"}
+
+	// The expected "quantile" values do not have the trailing ".0".
+	expectedQuantileValues := []string{"0.5", "0.9", "0.95", "0.99", "1"}
+
+	scrapeCount := 0
+	scraped := make(chan bool)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, metricsText)
+		scrapeCount++
+		if scrapeCount > 2 {
+			close(scraped)
+		}
+	}))
+	defer ts.Close()
+
+	sp, err := newScrapePool(config, simpleStorage, 0, nil, &Options{})
+	require.NoError(t, err)
+	defer sp.stop()
+
+	testURL, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+	sp.Sync([]*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{{model.AddressLabel: model.LabelValue(testURL.Host)}},
+		},
+	})
+	require.Equal(t, 1, len(sp.ActiveTargets()))
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Fatalf("target was not scraped")
+	case <-scraped:
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	q, err := simpleStorage.Querier(time.Time{}.UnixNano(), time.Now().UnixNano())
+	require.NoError(t, err)
+	defer q.Close()
+
+	checkValues := func(labelName string, expectedValues []string, series storage.SeriesSet) {
+		foundLeValues := map[string]bool{}
+
+		for series.Next() {
+			s := series.At()
+			v := s.Labels().Get(labelName)
+			require.NotContains(t, foundLeValues, v, "duplicate label value found")
+			foundLeValues[v] = true
+		}
+
+		require.Equal(t, len(expectedValues), len(foundLeValues), "number of label values not as expected")
+		for _, v := range expectedValues {
+			require.Contains(t, foundLeValues, v, "label value not found")
+		}
+	}
+
+	series := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "test_histogram_bucket"))
+	checkValues("le", expectedLeValues, series)
+
+	series = q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", "test_summary"))
+	checkValues("quantile", expectedQuantileValues, series)
+}
+
 func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrapeForTimestampedMetrics(t *testing.T) {
 	appender := &collectResultAppender{}
 	var (

tsdb/agent/db.go

@@ -241,6 +241,8 @@ type DB struct {
 	donec chan struct{}
 	stopc chan struct{}
 
+	writeNotified wlog.WriteNotified
+
 	metrics *dbMetrics
 }
 
@@ -311,6 +313,12 @@ func Open(l log.Logger, reg prometheus.Registerer, rs *remote.Storage, dir strin
 	return db, nil
 }
 
+// SetWriteNotified allows to set an instance to notify when a write happens.
+// It must be used during initialization. It is not safe to use it during execution.
+func (db *DB) SetWriteNotified(wn wlog.WriteNotified) {
+	db.writeNotified = wn
+}
+
 func validateOptions(opts *Options) *Options {
 	if opts == nil {
 		opts = DefaultOptions()
@@ -961,6 +969,10 @@ func (a *appender) Commit() error {
 
 	a.clearData()
 	a.appenderPool.Put(a)
+
+	if a.writeNotified != nil {
+		a.writeNotified.Notify()
+	}
 	return nil
 }
 

tsdb/wlog/watcher.go

@@ -65,7 +65,7 @@ type WriteTo interface {
 	SeriesReset(int)
 }
 
-// Used to notifier the watcher that data has been written so that it can read.
+// Used to notify the watcher that data has been written so that it can read.
 type WriteNotified interface {
 	Notify()
 }
@@ -398,28 +398,26 @@ func (w *Watcher) watch(segmentNum int, tail bool) error {
 
 	reader := NewLiveReader(w.logger, w.readerMetrics, segment)
 
-	readTicker := time.NewTicker(readTimeout)
-	defer readTicker.Stop()
-
-	checkpointTicker := time.NewTicker(checkpointPeriod)
-	defer checkpointTicker.Stop()
-
-	segmentTicker := time.NewTicker(segmentCheckPeriod)
-	defer segmentTicker.Stop()
-
-	// If we're replaying the segment we need to know the size of the file to know
-	// when to return from watch and move on to the next segment.
 	size := int64(math.MaxInt64)
 	if !tail {
-		segmentTicker.Stop()
-		checkpointTicker.Stop()
 		var err error
 		size, err = getSegmentSize(w.walDir, segmentNum)
 		if err != nil {
 			return errors.Wrap(err, "getSegmentSize")
 		}
+
+		return w.readAndHandleError(reader, segmentNum, tail, size)
 	}
 
+	checkpointTicker := time.NewTicker(checkpointPeriod)
+	defer checkpointTicker.Stop()
+
+	segmentTicker := time.NewTicker(segmentCheckPeriod)
+	defer segmentTicker.Stop()
+
+	readTicker := time.NewTicker(readTimeout)
+	defer readTicker.Stop()
+
 	gcSem := make(chan struct{}, 1)
 	for {
 		select {

tsdb/wlog/watcher_test.go

@@ -630,3 +630,61 @@ func TestCheckpointSeriesReset(t *testing.T) {
 		})
 	}
 }
+
+func TestRun_StartupTime(t *testing.T) {
+	const pageSize = 32 * 1024
+	const segments = 10
+	const seriesCount = 20
+	const samplesCount = 300
+
+	for _, compress := range []CompressionType{CompressionNone, CompressionSnappy, CompressionZstd} {
+		t.Run(string(compress), func(t *testing.T) {
+			dir := t.TempDir()
+
+			wdir := path.Join(dir, "wal")
+			err := os.Mkdir(wdir, 0o777)
+			require.NoError(t, err)
+
+			enc := record.Encoder{}
+			w, err := NewSize(nil, nil, wdir, pageSize, compress)
+			require.NoError(t, err)
+
+			for i := 0; i < segments; i++ {
+				for j := 0; j < seriesCount; j++ {
+					ref := j + (i * 100)
+					series := enc.Series([]record.RefSeries{
+						{
+							Ref:    chunks.HeadSeriesRef(ref),
+							Labels: labels.FromStrings("__name__", fmt.Sprintf("metric_%d", i)),
+						},
+					}, nil)
+					require.NoError(t, w.Log(series))
+
+					for k := 0; k < samplesCount; k++ {
+						inner := rand.Intn(ref + 1)
+						sample := enc.Samples([]record.RefSample{
+							{
+								Ref: chunks.HeadSeriesRef(inner),
+								T:   int64(i),
+								V:   float64(i),
+							},
+						}, nil)
+						require.NoError(t, w.Log(sample))
+					}
+				}
+			}
+			require.NoError(t, w.Close())
+
+			wt := newWriteToMock()
+			watcher := NewWatcher(wMetrics, nil, nil, "", wt, dir, false, false)
+			watcher.MaxSegment = segments
+
+			watcher.setMetrics()
+			startTime := time.Now()
+
+			err = watcher.Run()
+			require.Less(t, time.Since(startTime), readTimeout)
+			require.NoError(t, err)
+		})
+	}
+}

web/ui/module/codemirror-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/codemirror-promql",
-  "version": "0.48.0",
+  "version": "0.48.1",
   "description": "a CodeMirror mode for the PromQL language",
   "types": "dist/esm/index.d.ts",
   "module": "dist/esm/index.js",
@@ -29,7 +29,7 @@
   },
   "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md",
   "dependencies": {
-    "@prometheus-io/lezer-promql": "0.48.0",
+    "@prometheus-io/lezer-promql": "0.48.1",
     "lru-cache": "^7.18.3"
   },
   "devDependencies": {

web/ui/module/lezer-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/lezer-promql",
-  "version": "0.48.0",
+  "version": "0.48.1",
   "description": "lezer-based PromQL grammar",
   "main": "dist/index.cjs",
   "type": "module",