MQE: Add support for histogram_stdvar (grafana#10211)

pkg/streamingpromql/engine_test.go

@@ -2671,6 +2671,7 @@ func TestCompareVariousMixedMetricsFunctions(t *testing.T) {
 		expressions = append(expressions, fmt.Sprintf(`histogram_quantile(0.8, series{label=~"(%s)"})`, labelRegex))
 		expressions = append(expressions, fmt.Sprintf(`histogram_quantile(scalar(series{label="i"}), series{label=~"(%s)"})`, labelRegex))
 		expressions = append(expressions, fmt.Sprintf(`histogram_stddev(series{label=~"(%s)"})`, labelRegex))
+		expressions = append(expressions, fmt.Sprintf(`histogram_stdvar(series{label=~"(%s)"})`, labelRegex))
 		expressions = append(expressions, fmt.Sprintf(`histogram_sum(series{label=~"(%s)"})`, labelRegex))
 	}
 

pkg/streamingpromql/functions.go

@@ -371,7 +371,8 @@ var instantVectorFunctionOperatorFactories = map[string]InstantVectorFunctionOpe
 	"histogram_count":    InstantVectorTransformationFunctionOperatorFactory("histogram_count", functions.HistogramCount),
 	"histogram_fraction": HistogramFractionFunctionOperatorFactory(),
 	"histogram_quantile": HistogramQuantileFunctionOperatorFactory(),
-	"histogram_stddev":   InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDev),
+	"histogram_stddev":   InstantVectorTransformationFunctionOperatorFactory("histogram_stddev", functions.HistogramStdDevStdVar(true)),
+	"histogram_stdvar":   InstantVectorTransformationFunctionOperatorFactory("histogram_stdvar", functions.HistogramStdDevStdVar(false)),
 	"histogram_sum":      InstantVectorTransformationFunctionOperatorFactory("histogram_sum", functions.HistogramSum),
 	"increase":           FunctionOverRangeVectorOperatorFactory("increase", functions.Increase),
 	"label_replace":      LabelReplaceFunctionOperatorFactory(),

pkg/streamingpromql/operators/functions/native_histograms.go

@@ -89,51 +89,56 @@ func HistogramFraction(seriesData types.InstantVectorSeriesData, scalarArgsData
 	return data, nil
 }
 
-// HistogramStdDev returns the estimated standard deviation of observations in a native histogram
-// Float values are ignored.
-func HistogramStdDev(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {
-	fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker)
-	if err != nil {
-		return types.InstantVectorSeriesData{}, err
-	}
+func HistogramStdDevStdVar(isStdDev bool) InstantVectorSeriesFunction {
+	// returns either the standard deviation, or standard variance of a native histogram.
+	// Float values are ignored.
+	return func(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {
+		fPoints, err := types.FPointSlicePool.Get(len(seriesData.Histograms), memoryConsumptionTracker)
+		if err != nil {
+			return types.InstantVectorSeriesData{}, err
+		}
 
-	data := types.InstantVectorSeriesData{
-		Floats: fPoints,
-	}
+		data := types.InstantVectorSeriesData{
+			Floats: fPoints,
+		}
 
-	for _, histogram := range seriesData.Histograms {
-		mean := histogram.H.Sum / histogram.H.Count
-		var variance, cVariance float64
-		it := histogram.H.AllBucketIterator()
-		for it.Next() {
-			bucket := it.At()
-			if bucket.Count == 0 {
-				continue
-			}
-			var val float64
-			if bucket.Lower <= 0 && 0 <= bucket.Upper {
-				val = 0
-			} else {
-				val = math.Sqrt(bucket.Upper * bucket.Lower)
-				if bucket.Upper < 0 {
-					val = -val
+		for _, histogram := range seriesData.Histograms {
+			mean := histogram.H.Sum / histogram.H.Count
+			var variance, cVariance float64
+			it := histogram.H.AllBucketIterator()
+			for it.Next() {
+				bucket := it.At()
+				if bucket.Count == 0 {
+					continue
 				}
+				var val float64
+				if bucket.Lower <= 0 && 0 <= bucket.Upper {
+					val = 0
+				} else {
+					val = math.Sqrt(bucket.Upper * bucket.Lower)
+					if bucket.Upper < 0 {
+						val = -val
+					}
+				}
+				delta := val - mean
+				variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance)
+			}
+			variance += cVariance
+			variance /= histogram.H.Count
+			if isStdDev {
+				variance = math.Sqrt(variance)
 			}
-			delta := val - mean
-			variance, cVariance = floats.KahanSumInc(bucket.Count*delta*delta, variance, cVariance)
-		}
-		variance += cVariance
-		variance /= histogram.H.Count
 
-		data.Floats = append(data.Floats, promql.FPoint{
-			T: histogram.T,
-			F: math.Sqrt(variance),
-		})
-	}
+			data.Floats = append(data.Floats, promql.FPoint{
+				T: histogram.T,
+				F: variance,
+			})
+		}
 
-	types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker)
+		types.PutInstantVectorSeriesData(seriesData, memoryConsumptionTracker)
 
-	return data, nil
+		return data, nil
+	}
 }
 
 func HistogramSum(seriesData types.InstantVectorSeriesData, _ []types.ScalarData, _ types.QueryTimeRange, memoryConsumptionTracker *limiting.MemoryConsumptionTracker) (types.InstantVectorSeriesData, error) {

pkg/streamingpromql/testdata/ours/native_histograms.test

@@ -27,6 +27,9 @@ eval range from 0 to 5m step 1m histogram_fraction(0, 2, single_histogram)
 eval range from 0 to 5m step 1m histogram_stddev(single_histogram)
 	{} 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 0.842629429717281 2.986282214238901
 
+eval range from 0 to 5m step 1m histogram_stdvar(single_histogram)
+	{} 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 0.7100243558256704 8.917881463079594
+
 # histogram_sum extracts the sum property from the histogram.
 eval range from 0 to 5m step 1m histogram_sum(single_histogram)
 	{} 5 5 5 5 5 20
@@ -52,6 +55,9 @@ eval instant at 3m histogram_fraction(0, 1, mixed_metric)
 eval instant at 4m histogram_stddev(mixed_metric)
 	{} 0.6650352854715079
 
+eval instant at 4m histogram_stdvar(mixed_metric)
+	{} 0.44227193092217004
+
 eval instant at 4m histogram_sum(mixed_metric)
 	{} 8
 
@@ -73,6 +79,9 @@ eval instant at 2m histogram_fraction(0, 1, mixed_metric)
 # histogram_stddev ignores any float values
 eval instant at 2m histogram_stddev(mixed_metric)
 
+# histogram_stdvar ignores any float values
+eval instant at 2m histogram_stdvar(mixed_metric)
+
 # histogram_sum ignores any float values
 eval instant at 2m histogram_sum(mixed_metric)
 
@@ -109,6 +118,11 @@ eval instant at 0 histogram_stddev(route)
 	{path="two"} 0.8415900492770793
 	{path="three"} 1.1865698706402301
 
+eval instant at 0 histogram_stdvar(route)
+	{path="one"} 0.7100243558256704
+	{path="two"} 0.7082738110421968
+	{path="three"} 1.4079480579111723
+
 eval instant at 0 histogram_sum(route)
 	{path="one"} 5
 	{path="two"} 10
@@ -149,6 +163,9 @@ eval range from 0 to 8m step 1m histogram_fraction(0, 1, mixed_metric)
 eval range from 0 to 8m step 1m histogram_stddev(mixed_metric)
 {} _ _ _ _ _ _ _ 0.8574122997574659 0.8574122997574659
 
+eval range from 0 to 8m step 1m histogram_stdvar(mixed_metric)
+{} _ _ _ _ _ _ _ 0.7351558517753866 0.7351558517753866
+
 eval range from 0 to 8m step 1m histogram_sum(mixed_metric)
 {} _ _ _ _ _ _ _ 18 18
 

pkg/streamingpromql/testdata/upstream/histograms.test

@@ -99,16 +99,14 @@ eval instant at 50m histogram_avg(testhistogram3)
 	{start="negative"} 4
 
 # Test histogram_stddev. This has no classic equivalent.
-# Unsupported by streaming engine.
-# eval instant at 50m histogram_stddev(testhistogram3)
-#	{start="positive"} 2.8189265757336734
-#	{start="negative"} 4.182715937754936
+eval instant at 50m histogram_stddev(testhistogram3)
+	{start="positive"} 2.8189265757336734
+	{start="negative"} 4.182715937754936
 
 # Test histogram_stdvar. This has no classic equivalent.
-# Unsupported by streaming engine.
-# eval instant at 50m histogram_stdvar(testhistogram3)
-#	{start="positive"} 7.946347039377573
-#	{start="negative"} 17.495112615949154
+eval instant at 50m histogram_stdvar(testhistogram3)
+	{start="positive"} 7.946347039377573
+	{start="negative"} 17.495112615949154
 
 # Test histogram_fraction.
 

pkg/streamingpromql/testdata/upstream/native_histograms.test

@@ -320,9 +320,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_1)
     {} 1.0787993180043811
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1)
-#     {} 1.163807968526718
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_1)
+    {} 1.163807968526718
 
 clear
 
@@ -333,9 +332,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_2)
     {} 0.0048960313898237465
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2)
-#     {} 2.3971123370139447e-05
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_2)
+    {} 2.3971123370139447e-05
 
 clear
 
@@ -346,9 +344,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_3)
     {} 42.947236400258
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3)
-#     {} 1844.4651144196398
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_3)
+    {} 1844.4651144196398
 
 clear
 
@@ -359,9 +356,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_4)
     {} 27556.344499842
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4)
-#     {} 759352122.1939945
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_4)
+    {} 759352122.1939945
 
 clear
 
@@ -372,9 +368,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_5)
     {} 1.3137084989848
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5)
-#     {} 1.725830020304794
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_5)
+    {} 1.725830020304794
 
 clear
 
@@ -385,9 +380,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_6)
     {} NaN
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6)
-#     {} NaN
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_6)
+    {} NaN
 
 clear
 
@@ -398,9 +392,8 @@ load 10m
 eval instant at 10m histogram_stddev(histogram_stddev_stdvar_7)
     {} Inf
 
-# Unsupported by streaming engine.
-# eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7)
-#     {} Inf
+eval instant at 10m histogram_stdvar(histogram_stddev_stdvar_7)
+    {} Inf
 
 clear
 
@@ -1102,9 +1095,8 @@ eval instant at 5m histogram_stddev(rate(const_histogram[5m]))
     {} NaN
 
 # Zero buckets mean no observations, so there is no standard variance.
-# Unsupported by streaming engine.
-# eval instant at 5m histogram_stdvar(rate(const_histogram[5m]))
-#     {} NaN
+eval instant at 5m histogram_stdvar(rate(const_histogram[5m]))
+    {} NaN
 
 clear