From 726bf0653f99083a7486fc00a2be6abc3f859050 Mon Sep 17 00:00:00 2001 From: "Christopher M. Wolff" Date: Thu, 23 Feb 2023 08:12:21 -0800 Subject: [PATCH] feat: add parameter to histogramQuantile (#5386) * feat: add parameter to histogramQuantile * chore: make fmt * feat: implement new parameter for histogramQuantile * chore: update interpreter_test.go --- interpreter/interpreter_test.go | 4 +- libflux/go/libflux/buildinfo.gen.go | 8 +- .../experimental/prometheus/prometheus.flux | 22 ++- .../prometheus_histogramQuantile_test.flux | 83 ++++++++++++ stdlib/universe/histogram_quantile.go | 58 ++++++-- stdlib/universe/histogram_quantile_test.flux | 126 ++++++++++++++++++ stdlib/universe/universe.flux | 9 ++ 7 files changed, 285 insertions(+), 25 deletions(-) diff --git a/interpreter/interpreter_test.go b/interpreter/interpreter_test.go index 3fbb8c6729..1a6306d66b 100644 --- a/interpreter/interpreter_test.go +++ b/interpreter/interpreter_test.go @@ -883,8 +883,8 @@ func TestStack(t *testing.T) { FunctionName: "window", Location: ast.SourceLocation{ File: "universe/universe.flux", - Start: ast.Position{Line: 3895, Column: 12}, - End: ast.Position{Line: 3895, Column: 51}, + Start: ast.Position{Line: 3904, Column: 12}, + End: ast.Position{Line: 3904, Column: 51}, Source: `window(every: inf, timeColumn: timeDst)`, }, }, diff --git a/libflux/go/libflux/buildinfo.gen.go b/libflux/go/libflux/buildinfo.gen.go index 3ab97ccb3a..dc6d832283 100644 --- a/libflux/go/libflux/buildinfo.gen.go +++ b/libflux/go/libflux/buildinfo.gen.go @@ -214,8 +214,8 @@ var sourceHashes = map[string]string{ "stdlib/experimental/polyline/polyline.flux": "09f8d405349236de713fef7639d5523b107401bbe349789924f2283296205b9b", "stdlib/experimental/polyline/polyline_test.flux": "16473dce4f71dcdbe1e3f90b350ab1e56a513679cb5c3f872e0f2d7678d42d1f", "stdlib/experimental/preview_test.flux": "cca570d25b17ed201a0ecc7ebf9e547ccff2aa0814a3ac49f12faa938cbdaf73", - "stdlib/experimental/prometheus/prometheus.flux": "e0b3df509c8f522edee1081e5e3907ce9628f783d33b47c2e2d8ffb766f3f948", - "stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux": "37bda52e9440820c3bb278b4e9d331c0330a68f405a7623b95e15a50ee176753", + "stdlib/experimental/prometheus/prometheus.flux": "dc01322d3c0655661a8c2279c69acf50d02791a670fb0d681947af6726bc2f4d", + "stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux": "15da11b9c9d43cbc1c579de7064d7f3870b1c77b610ce3c567e8eb14f40ee090", "stdlib/experimental/quantile_test.flux": "e3cf2ee9716c1179139d0a388c24b24f1c25ca329b27bebaeb53b7e1b608ead2", "stdlib/experimental/query/from.flux": "713b7feb6904d64cf4cbd235396ff6ff20e1eb96578190c0ac3be4f37df7c362", "stdlib/experimental/record/record.flux": "273eebb2ee5cb9b153940ca0f42ed5b97b3d86de07d91c96a75508682d84feae", @@ -492,7 +492,7 @@ var sourceHashes = map[string]string{ "stdlib/universe/highestAverage_test.flux": "65744d16b7c5d8ac2f4e3c47621195de1840ad72b12bfbb692d4f645e71a00a8", "stdlib/universe/highestCurrent_test.flux": "c285ff40a7d8789d2c20bcf90d8063b710499ddc24621d627f6693c30351ebe5", "stdlib/universe/highestMax_test.flux": "fff9f21422d2607afb9ff2786d67d173c7cd797418eb7b3521f8cf7e49443b88", - "stdlib/universe/histogram_quantile_test.flux": "7dcd4549a44f6a2889a445ae8c1adbb07a99747358b1d400d39b68d5c4233551", + "stdlib/universe/histogram_quantile_test.flux": "3ce3d5e6ce27fd8bb2e84da031faca2dafef2566737842e534fbd667ffa8a4f6", "stdlib/universe/histogram_test.flux": "e9e9775f80ac7c2a76044e6e0e8a89045d736c6ab618e8de6d7d1ebe9848938e", "stdlib/universe/holt_winters_panic_test.flux": "204eb8044d634e5350a364eac466eb51e7f549e4ac7f454de7b244ba272b248f", "stdlib/universe/holt_winters_test.flux": "9bc8441527867b6c075d003034a3def1748766df478ba8b434e2a2297ead0ec0", @@ -609,7 +609,7 @@ var sourceHashes = map[string]string{ "stdlib/universe/union_heterogeneous_test.flux": "7d8b47b3e96b859a5fed5985c051e2a3fdc947d3d6ff9cc104e40821581fb0cb", "stdlib/universe/union_test.flux": "f008260d48db70212ce64d3f51f4cf031532a9a67c1ba43242dbc4d43ef31293", "stdlib/universe/unique_test.flux": "c108ab7b0e4b0b77f0a320c8c4dacb8cfbccae8b389425754e9583e69cd64ee3", - "stdlib/universe/universe.flux": "060426aa8c8caf89187489f3bbd077cdba9efa389d76a09d3346636b4ba35e61", + "stdlib/universe/universe.flux": "8f3b92accba41d660339bcd6e2f83e7a0d650cab637630e320ba87b5cc2f2335", "stdlib/universe/universe_truncateTimeColumn_test.flux": "8acb700c612e9eba87c0525b33fd1f0528e6139cc912ed844932caef25d37b56", "stdlib/universe/window_aggregate_test.flux": "c8f66f7ee188bb2e979e5a8b526057b653922197ae441658f7c7f11251c96576", "stdlib/universe/window_default_start_align_test.flux": "0aaf612796fbb5ac421579151ad32a8861f4494a314ea615d0ccedd18067b980", diff --git a/stdlib/experimental/prometheus/prometheus.flux b/stdlib/experimental/prometheus/prometheus.flux index f87f1724bf..8cd68ce477 100644 --- a/stdlib/experimental/prometheus/prometheus.flux +++ b/stdlib/experimental/prometheus/prometheus.flux @@ -46,6 +46,14 @@ builtin scrape : (url: string) => stream[A] where A: Record // Available versions are `1` and `2`. // Default is `2`. // - tables: Input data. Default is piped-forward data (`<-`). +// - onNonmonotonic: Describes behavior when counts are not monotonically increasing +// when sorted by upper bound. Default is `error`. +// +// **Supported values**: +// - **error**: Produce an error. +// - **force**: Force bin counts to be monotonic by adding to each bin such that it +// is equal to the next smaller bin. +// - **drop**: When a nonmonotonic table is encountered, produce no output. // // ## Examples // @@ -72,31 +80,31 @@ builtin scrape : (url: string) => stream[A] where A: Record // ## Metadata // tags: transformations,aggregates,prometheus // -histogramQuantile = (tables=<-, quantile, metricVersion=2) => { - _version2 = () => +histogramQuantile = (tables=<-, quantile, metricVersion=2, onNonmonotonic="error") => { + _version2 = (onNonmonotonic) => tables |> group(mode: "except", columns: ["le", "_value"]) |> map(fn: (r) => ({r with le: float(v: r.le)})) - |> universe.histogramQuantile(quantile: quantile) + |> universe.histogramQuantile(quantile: quantile, onNonmonotonic: onNonmonotonic) |> group(mode: "except", columns: ["le", "_value", "_time"]) |> set(key: "quantile", value: string(v: quantile)) |> experimental.group(columns: ["quantile"], mode: "extend") - _version1 = () => + _version1 = (onNonmonotonic) => tables |> filter(fn: (r) => r._field != "sum" and r._field != "count") |> map(fn: (r) => ({r with le: float(v: r._field)})) |> group(mode: "except", columns: ["_field", "le", "_value"]) - |> universe.histogramQuantile(quantile: quantile) + |> universe.histogramQuantile(quantile: quantile, onNonmonotonic: onNonmonotonic) |> group(mode: "except", columns: ["le", "_value", "_time"]) |> set(key: "quantile", value: string(v: quantile)) |> experimental.group(columns: ["quantile"], mode: "extend") output = if metricVersion == 2 then - _version2() + _version2(onNonmonotonic) else if metricVersion == 1 then - _version1() + _version1(onNonmonotonic) else universe.die(msg: "Invalid metricVersion. Available versions are 1 and 2.") diff --git a/stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux b/stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux index fcb039ae1d..1f0038148a 100644 --- a/stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux +++ b/stdlib/experimental/prometheus/prometheus_histogramQuantile_test.flux @@ -292,3 +292,86 @@ testcase prometheus_histogramQuantile_v1 { testing.diff(got: got, want: want) } + +testcase prometheus_histogramQuantile_onNonmonotonicForce { + inData = + "#group,false,false,true,true,false,false,true,true +#datatype,string,long,string,string,dateTime:RFC3339,double,string,string +#default,_result,,,,,,, +,result,table,_field,_measurement,_time,_value,le,org +,,0,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,0,0.001,0001 +,,2,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,1,0.005,0001 +,,3,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,1,0.025,0001 +,,4,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,3,0.125,0001 +,,4,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,2,0.625,0001 +,,5,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,7,3.125,0001 +,,6,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,10,15.625,0001 +,,7,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00.412729Z,10,+Inf,0001 +" + outData = + "#group,false,false,true,true,true,true,false,true,false,true +#datatype,string,long,string,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,double,string +#default,_result,,,,,,,,, +,result,table,_field,_measurement,_start,_stop,_time,org,_value,quantile +,,0,qc_all_duration_seconds,prometheus,2021-10-08T00:00:00Z,2021-10-08T00:01:00Z,2021-10-08T00:00:00.412729Z,0001,15.208333333333336,0.99 +" + want = csv.from(csv: outData) + got = + csv.from(csv: inData) + |> range(start: 2021-10-08T00:00:00Z, stop: 2021-10-08T00:01:00Z) + |> prometheus.histogramQuantile( + quantile: 0.99, + metricVersion: 2, + onNonmonotonic: "force", + ) + + testing.diff(got: got, want: want) +} + +testcase prometheus_histogramQuantile_onNonmonotonicDrop { + // Data for org 0002 is not monotonic + inData = + "#group,false,false,true,true,false,false,true +#datatype,string,long,string,string,dateTime:RFC3339,double,string +#default,_result,,,,,, +,result,table,_field,_measurement,_time,_value,org +,,0,+Inf,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,10,0001 +,,1,+Inf,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1405,0002 +,,2,0.001,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,3,0.001,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1,0002 +,,4,0.005,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,5,0.005,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1,0002 +,,6,0.025,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,7,0.025,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,84,0002 +,,8,0.125,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,9,0.125,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,83,0002 +,,10,0.625,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,11,0.625,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1373,0002 +,,12,15.625,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,10,0001 +,,13,15.625,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1405,0002 +,,14,3.125,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,0,0001 +,,15,3.125,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1401,0002 +,,16,count,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,10,0001 +,,17,count,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,1405,0002 +,,18,sum,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,45.746700925,0001 +,,19,sum,qc_all_duration_seconds,2021-10-08T00:00:01.866064Z,178.4667627259998,0002 +" + outData = + "#group,false,false,true,true,true,false,true,false,true +#datatype,string,long,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,double,string +#default,_result,,,,,,,, +,result,table,_measurement,_start,_stop,_time,org,_value,quantile +,,0,qc_all_duration_seconds,2021-10-08T00:00:00Z,2021-10-08T00:01:00Z,2021-10-08T00:00:01.866064Z,0001,15.5,0.99 +" + want = csv.from(csv: outData) + got = + csv.from(csv: inData) + |> range(start: 2021-10-08T00:00:00Z, stop: 2021-10-08T00:01:00Z) + |> prometheus.histogramQuantile( + quantile: 0.99, + metricVersion: 1, + onNonmonotonic: "drop", + ) + + testing.diff(got: got, want: want) +} diff --git a/stdlib/universe/histogram_quantile.go b/stdlib/universe/histogram_quantile.go index d6f071b9df..51d4d4b3a7 100644 --- a/stdlib/universe/histogram_quantile.go +++ b/stdlib/universe/histogram_quantile.go @@ -12,9 +12,15 @@ import ( "github.com/influxdata/flux/runtime" ) -const HistogramQuantileKind = "histogramQuantile" +const ( + HistogramQuantileKind = "histogramQuantile" -const DefaultUpperBoundColumnLabel = "le" + DefaultUpperBoundColumnLabel = "le" + + onNonmonotonicError = "error" + onNonmonotonicDrop = "drop" + onNonmonotonicForce = "force" +) type HistogramQuantileOpSpec struct { Quantile float64 `json:"quantile"` @@ -22,6 +28,7 @@ type HistogramQuantileOpSpec struct { UpperBoundColumn string `json:"upperBoundColumn"` ValueColumn string `json:"valueColumn"` MinValue float64 `json:"minValue"` + OnNonmonotonic string `json:"onNonmonotonic"` } func init() { @@ -73,6 +80,18 @@ func CreateHistogramQuantileOpSpec(args flux.Arguments, a *flux.Administration) s.MinValue = min } + if onNonmonotonic, ok, err := args.GetString("onNonmonotonic"); err != nil { + return nil, err + } else if ok { + s.OnNonmonotonic = onNonmonotonic + } else { + s.OnNonmonotonic = onNonmonotonicError + } + + if s.OnNonmonotonic != onNonmonotonicError && s.OnNonmonotonic != onNonmonotonicForce && s.OnNonmonotonic != onNonmonotonicDrop { + return nil, errors.Newf(codes.Invalid, "value provided to histogramQuantile parameter onNonmonotonic is invalid; must be one of %q, %q or %q", onNonmonotonicError, onNonmonotonicForce, onNonmonotonicDrop) + } + return s, nil } @@ -87,6 +106,7 @@ type HistogramQuantileProcedureSpec struct { UpperBoundColumn string `json:"upperBoundColumn"` ValueColumn string `json:"valueColumn"` MinValue float64 `json:"minValue"` + OnNonmonotonic string `json:"onNonmonotonic"` } func newHistogramQuantileProcedure(qs flux.OperationSpec, a plan.Administration) (plan.ProcedureSpec, error) { @@ -100,6 +120,7 @@ func newHistogramQuantileProcedure(qs flux.OperationSpec, a plan.Administration) UpperBoundColumn: spec.UpperBoundColumn, ValueColumn: spec.ValueColumn, MinValue: spec.MinValue, + OnNonmonotonic: spec.OnNonmonotonic, }, nil } @@ -230,10 +251,13 @@ func (t histogramQuantileTransformation) Process(id execute.DatasetID, tbl flux. }) } - q, err := t.computeQuantile(cdf) + q, ok, err := t.computeQuantile(cdf) if err != nil { return err } + if !ok { + return nil + } if err := execute.AppendKeyValues(tbl.Key(), builder); err != nil { return err } @@ -243,9 +267,9 @@ func (t histogramQuantileTransformation) Process(id execute.DatasetID, tbl flux. return nil } -func (t *histogramQuantileTransformation) computeQuantile(cdf []bucket) (float64, error) { +func (t *histogramQuantileTransformation) computeQuantile(cdf []bucket) (float64, bool, error) { if len(cdf) == 0 { - return 0, errors.New(codes.FailedPrecondition, "histogram is empty") + return 0, false, errors.New(codes.FailedPrecondition, "histogram is empty") } // Find rank index and check counts are monotonic prevCount := 0.0 @@ -254,9 +278,19 @@ func (t *histogramQuantileTransformation) computeQuantile(cdf []bucket) (float64 rankIdx := -1 for i, b := range cdf { if b.count < prevCount { - return 0, errors.New(codes.FailedPrecondition, "histogram records counts are not monotonic") + switch t.spec.OnNonmonotonic { + case onNonmonotonicError: + return 0, false, errors.New(codes.FailedPrecondition, "histogram records counts are not monotonic") + case onNonmonotonicForce: + b.count = prevCount + case onNonmonotonicDrop: + return 0, false, nil + default: + return 0, false, errors.Newf(codes.Internal, "unknown value for onNonmonotonic: %q", t.spec.OnNonmonotonic) + } + } else { + prevCount = b.count } - prevCount = b.count if rank >= b.count { rankIdx = i @@ -277,7 +311,7 @@ func (t *histogramQuantileTransformation) computeQuantile(cdf []bucket) (float64 upperBound = cdf[0].upperBound case len(cdf) - 1: // Quantile is above the highest upper bound, simply return it as it must be finite - return cdf[len(cdf)-1].upperBound, nil + return cdf[len(cdf)-1].upperBound, true, nil default: lowerCount = cdf[rankIdx].count lowerBound = cdf[rankIdx].upperBound @@ -286,19 +320,19 @@ func (t *histogramQuantileTransformation) computeQuantile(cdf []bucket) (float64 } if rank == lowerCount { // No need to interpolate - return lowerBound, nil + return lowerBound, true, nil } if math.IsInf(lowerBound, -1) { // We cannot interpolate with infinity - return upperBound, nil + return upperBound, true, nil } if math.IsInf(upperBound, 1) { // We cannot interpolate with infinity - return lowerBound, nil + return lowerBound, true, nil } // Compute quantile using linear interpolation scale := (rank - lowerCount) / (upperCount - lowerCount) - return lowerBound + (upperBound-lowerBound)*scale, nil + return lowerBound + (upperBound-lowerBound)*scale, true, nil } func (t histogramQuantileTransformation) UpdateWatermark(id execute.DatasetID, mark execute.Time) error { diff --git a/stdlib/universe/histogram_quantile_test.flux b/stdlib/universe/histogram_quantile_test.flux index 1accd52771..01e4425920 100644 --- a/stdlib/universe/histogram_quantile_test.flux +++ b/stdlib/universe/histogram_quantile_test.flux @@ -88,3 +88,129 @@ testcase histogram_quantile_minvalue { testing.diff(got, want) } + +testcase histogramQuantileInvalidOnNonmonotonic { + inData = + " +#datatype,string,long,dateTime:RFC3339,string,double,double,string +#group,false,false,true,true,false,false,true +#default,_result,,,,,, +,result,table,_time,_field,_value,le,_measurement +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,-80,mm +" + fn = () => + csv.from(csv: inData) + |> range(start: 2018-05-22T19:53:00Z) + |> histogramQuantile(quantile: 0.25, minValue: -100.0, onNonmonotonic: "asdf") + + testing.shouldError( + fn: fn, + want: /value provided to histogramQuantile parameter onNonmonotonic is invalid/, + ) +} + +testcase histogramQuantileOnNonmonotonicError { + inData = + " +#datatype,string,long,dateTime:RFC3339,string,double,double,string +#group,false,false,true,true,false,false,true +#default,_result,,,,,, +,result,table,_time,_field,_value,le,_measurement +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.1,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.2,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.3,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,3,0.4,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.5,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.6,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.7,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,8,0.8,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,0.9,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,+Inf,l +" + fn = () => + csv.from(csv: inData) + |> histogramQuantile(quantile: 0.9) + |> tableFind(fn: (key) => true) + |> findRecord(fn: (key) => true, idx: 0) + + testing.shouldError(fn: fn, want: /histogram records counts are not monotonic/) +} + +testcase histogramQuantileOnNonmonotonicForce { + inData = + " +#datatype,string,long,dateTime:RFC3339,string,double,double,string +#group,false,false,true,true,false,false,true +#default,_result,,,,,, +,result,table,_time,_field,_value,le,_measurement +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.1,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.2,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.3,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,0,0.4,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.5,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.6,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.7,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,8,0.8,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,0.9,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,+Inf,l +" + outData = + " +#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,double,string +#group,false,false,true,true,true,true,false,true +#default,_result,,,,,,, +,result,table,_start,_stop,_time,_field,_value,_measurement +,,0,2018-05-22T19:53:00Z,2030-01-01T00:00:00Z,2018-05-22T19:53:00Z,x_duration_seconds,0.8500000000000001,l +" + + got = + csv.from(csv: inData) + |> range(start: 2018-05-22T19:53:00Z) + |> histogramQuantile(quantile: 0.9, onNonmonotonic: "force") + want = csv.from(csv: outData) + + testing.diff(got, want) +} + +testcase histogramQuantileOnNonmonotonicDrop { + inData = + " +#datatype,string,long,dateTime:RFC3339,string,double,double,string +#group,false,false,true,true,false,false,true +#default,_result,,,,,, +,result,table,_time,_field,_value,le,_measurement +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.1,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.2,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.3,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,0,0.4,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.5,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,1,0.6,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,2,0.7,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,8,0.8,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,0.9,l +,,0,2018-05-22T19:53:00Z,x_duration_seconds,10,+Inf,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,0,-Inf,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,10,0.2,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,15,0.4,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,25,0.6,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,35,0.8,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,45,1,l +,,1,2018-05-22T19:53:00Z,y_duration_seconds,45,+Inf,l +" + outData = + " +#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,double,string +#group,false,false,true,true,true,true,false,true +#default,_result,,,,,,, +,result,table,_start,_stop,_time,_field,_value,_measurement +,,1,2018-05-22T19:53:00Z,2030-01-01T00:00:00Z,2018-05-22T19:53:00Z,y_duration_seconds,0.91,l +" + + got = + csv.from(csv: inData) + |> range(start: 2018-05-22T19:53:00Z) + |> histogramQuantile(quantile: 0.9, onNonmonotonic: "drop") + want = csv.from(csv: outData) + + testing.diff(got, want) +} diff --git a/stdlib/universe/universe.flux b/stdlib/universe/universe.flux index dd744cd6fc..1d0dbe516a 100644 --- a/stdlib/universe/universe.flux +++ b/stdlib/universe/universe.flux @@ -850,6 +850,14 @@ builtin histogram : ( // Default is `le`. // - valueColumn: Column to store the computed quantile in. Default is `_value. // - minValue: Assumed minimum value of the dataset. Default is `0.0`. +// - onNonmonotonic: Describes behavior when counts are not monotonically increasing +// when sorted by upper bound. Default is `error`. +// +// **Supported values**: +// - **error**: Produce an error. +// - **force**: Force bin counts to be monotonic by adding to each bin such that it +// is equal to the next smaller bin. +// - **drop**: When a nonmonotonic table is encountered, produce no output. // // If the quantile falls below the lowest upper bound, interpolation is // performed between `minValue` and the lowest upper bound. @@ -880,6 +888,7 @@ builtin histogramQuantile : ( ?upperBoundColumn: string, ?valueColumn: string, ?minValue: float, + ?onNonmonotonic: string, ) => stream[B] where A: Record,