Make sure non-normalized data integrate well with categorical probes (#…

…2619)
mozilla · Dec 6, 2023 · 11f55c2 · 11f55c2
1 parent fba4f81
commit 11f55c2
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 24 deletions.
diff --git a/src/components/explore/AggregationsOverTimeGraph.svelte b/src/components/explore/AggregationsOverTimeGraph.svelte
@@ -33,6 +33,8 @@
   import {
     getPercentileName,
     getTransformedPercentileName,
+    getProportionName,
+    getCountName,
   } from '../../config/shared';
 
   export let title;
@@ -233,12 +235,26 @@
       if ($store.activeBuckets.length === 10) return yDomain;
       if ($store.proportionMetricType === 'proportions') {
         buckets.forEach((bucket) => {
-          yData = yData.concat([...data.map((arr) => arr.proportions[bucket])]);
+          yData = yData.concat([
+            ...data.map(
+              (arr) =>
+                arr[
+                  getProportionName($store.productDimensions.normalizationType)
+                ][bucket]
+            ),
+          ]);
         });
       }
       if ($store.proportionMetricType === 'counts') {
         buckets.forEach((bucket) => {
-          yData = yData.concat([...data.map((arr) => arr.counts[bucket])]);
+          yData = yData.concat([
+            ...data.map(
+              (arr) =>
+                arr[getCountName($store.productDimensions.normalizationType)][
+                  bucket
+                ]
+            ),
+          ]);
         });
       }
     }

diff --git a/src/components/explore/ProbeExplorer.svelte b/src/components/explore/ProbeExplorer.svelte
@@ -1,4 +1,5 @@
 <script>
+  import { isEmpty } from 'lodash';
   import { writable } from 'svelte/store';
 
   import { window1D } from '@graph-paper/core/utils/window-functions';
@@ -48,7 +49,7 @@
     // data which caused the graph to break.
     // so, we filter out these empty data points.
     normType === 'non_normalized'
-      ? normData.filter((d) => d.non_norm_histogram !== '')
+      ? normData.filter((d) => !isEmpty(d.non_norm_histogram))
       : normData;
 
   let data = filterData(

diff --git a/src/components/explore/ProportionExplorerView.svelte b/src/components/explore/ProportionExplorerView.svelte
@@ -26,7 +26,12 @@
     gatherAggregationTypes,
   } from '../../utils/probe-utils';
   import { numericStringsSort } from '../../utils/sort';
-  import { getHistogramName, numHighlightedBuckets } from '../../config/shared';
+  import {
+    getHistogramName,
+    getProportionName,
+    getCountName,
+    numHighlightedBuckets,
+  } from '../../config/shared';
 
   export let aggregationLevel = 'build_id';
   export let data;
@@ -52,28 +57,27 @@
   let currentKey = probeKeys[0];
   let currentAggregation = aggregationTypes[0];
 
+  function filterResponseData(d, agg, key) {
+    return d.filter(
+      (di) => di.client_agg_type === agg && di.metric_key === key
+    );
+  }
+
+  $: selectedData = filterResponseData(data, currentAggregation, currentKey);
+
   // set the audience size when the reference updates.
   let ref;
   const movingAudienceSize = tweened(0, { duration: 500, easing });
   $: if (ref) movingAudienceSize.set(ref.audienceSize);
 
   $: if (currentKey && ref) {
-    if (data[currentKey] !== undefined) {
-      const r = data[currentKey][currentAggregation].find(
+    if (selectedData[currentKey] !== undefined) {
+      const r = selectedData[currentKey][currentAggregation].find(
         (d) => d.label.toString() === ref.label.toString()
       );
       ref = r;
     }
   }
-
-  function filterResponseData(d, agg, key) {
-    return d.filter(
-      (di) => di.client_agg_type === agg && di.metric_key === key
-    );
-  }
-
-  $: selectedData = filterResponseData(data, currentAggregation, currentKey);
-
   let showOptionMenu = false;
   let coloredBuckets = [];
   let everActiveBuckets = [];
@@ -84,7 +88,9 @@
     showOptionMenu = true;
     const lastDataset = data[data.length - 1];
 
-    coloredBuckets = Object.entries(lastDataset.counts)
+    coloredBuckets = Object.entries(
+      lastDataset[getCountName($store.productDimensions.normalizationType)]
+    )
       .sort(([bucketAName, bucketAValue], [bucketBName, bucketBValue]) => {
         const bucketValueDifference = bucketAValue - bucketBValue;
         if (bucketValueDifference === 0) {
@@ -154,7 +160,12 @@
       {/if}
     </div>
 
-    <CategoricalMenu {data} {activeBuckets} {bucketColorMap} {bucketOptions} />
+    <CategoricalMenu
+      data={selectedData}
+      {activeBuckets}
+      {bucketColorMap}
+      {bucketOptions}
+    />
   </div>
 
   <div class="body-control-row  body-control-row--stretch">
@@ -195,7 +206,9 @@
               binColorMap={bucketColorMap}
               showViolins={false}
               {aggregationLevel}
-              pointMetricType={metricType}
+              pointMetricType={getProportionName(
+                $store.productDimensions.normalizationType
+              )}
               {densityMetricType}
               yTickFormatter={metricType === 'proportions'
                 ? formatPercent
@@ -208,7 +221,19 @@
                 0,
                 Math.max(
                   ...selectedData
-                    .map((d) => Object.values(d[metricType]))
+                    .map((d) =>
+                      Object.values(
+                        d[
+                          metricType === 'proportions'
+                            ? getProportionName(
+                                $store.productDimensions.normalizationType
+                              )
+                            : getCountName(
+                                $store.productDimensions.normalizationType
+                              )
+                        ]
+                      )
+                    )
                     .flat()
                 ),
               ]}

diff --git a/src/components/table/ProbeTableView.svelte b/src/components/table/ProbeTableView.svelte
@@ -9,7 +9,7 @@
     gatherAggregationTypes,
   } from '../../utils/probe-utils';
   import { PERCENTILES } from '../../utils/constants';
-  import { getPercentileName } from '../../config/shared';
+  import { getProportionName, getPercentileName } from '../../config/shared';
   import { store } from '../../state/store';
 
   export let data;
@@ -84,7 +84,7 @@
       ? formatPercentDecimal
       : formatCount}
     key={probeType === 'categorical'
-      ? 'proportions'
+      ? getProportionName($store.productDimensions.normalizationType)
       : getPercentileName($store.productDimensions.normalizationType)}
     tooltipFormatter={probeType === 'categorical'
       ? () => undefined

diff --git a/src/components/table/TableView.svelte b/src/components/table/TableView.svelte
@@ -1,6 +1,6 @@
 <script>
   import { Button, ButtonGroup } from '@graph-paper/button';
-
+  import { isEmpty } from 'lodash';
   import DataTable from './DataTable.svelte';
   import Row from './Row.svelte';
   import Cell from './Cell.svelte';
@@ -49,7 +49,7 @@
     // empty non-normalized data
     let filtered =
       $store.productDimensions.normalizationType === 'non_normalized'
-        ? data.filter((d) => d.non_norm_histogram !== '')
+        ? data.filter((d) => !isEmpty(d.non_norm_histogram))
         : data;
     updatedData = filtered.map((d) => ({
       ...d,

diff --git a/src/config/shared.js b/src/config/shared.js
@@ -13,6 +13,10 @@ const dataNormalizationNameMap = {
     non_normalized: 'non_norm_proportions',
     normalized: 'proportions',
   },
+  counts: {
+    non_normalized: 'non_norm_counts',
+    normalized: 'counts',
+  },
 };
 
 export const numHighlightedBuckets = 10;
@@ -63,6 +67,13 @@ export function getProportionName(type = 'proportions') {
   return dataNormalizationNameMap.proportions[type];
 }
 
+export function getCountName(type = 'counts') {
+  if (!Object.hasOwn(dataNormalizationNameMap.counts, type)) {
+    throw new Error(`Unknown normalization type: ${type}`);
+  }
+  return dataNormalizationNameMap.counts[type];
+}
+
 export function extractBucketMetadata(transformedData) {
   const etc = {};
   const options = getBucketKeys(transformedData);

diff --git a/src/utils/probe-utils.js b/src/utils/probe-utils.js
@@ -34,3 +34,15 @@ export function convertValueToPercentage(data) {
   const sum = data.reduce((a, b) => a + b.value, 0);
   return data.map((a) => ({ bin: a.bin, value: a.value / sum }));
 }
+
+export function convertValueToProportions(obj) {
+  const newObj = { ...obj };
+
+  // Calculate the total of all values
+  const total = Object.values(newObj).reduce((a, b) => a + b, 0);
+  // Convert each value to a proportion of the total
+  Object.keys(newObj).forEach((key) => {
+    newObj[key] /= total;
+  });
+  return newObj;
+}
diff --git a/src/utils/transform-data.js b/src/utils/transform-data.js
@@ -17,6 +17,7 @@ Some transform functions are checks that, if they fail, throw an error.
 import produce from 'immer';
 import { fullBuildIDToDate, buildDateStringToDate } from './build-id-utils';
 import { nearestBelow } from './stats';
+import { convertValueToProportions } from './probe-utils';
 
 const errors = {
   MISSING_PERCENTILES: {
@@ -92,7 +93,12 @@ export function checkForTotalUsers(draft) {
 export function addProportion(draft) {
   // requires point.histogram.
   draft.proportions = { ...draft.histogram };
-  draft.non_norm_proportions = { ...draft.non_norm_histogram };
+
+  // draft.non_norm_histogram is not in proportion format
+  // like draft.histogram so we need to convert it here
+  draft.non_norm_proportions = convertValueToProportions(
+    draft.non_norm_histogram
+  );
 }
 
 export function changeBooleanHistogramResponse(draft) {
@@ -109,6 +115,11 @@ export function proportionsToCounts(draft) {
   Object.keys(draft.proportions).forEach((p) => {
     draft.counts[p] = draft.proportions[p] * draft.total_users;
   });
+  draft.non_norm_counts = {};
+  Object.keys(draft.non_norm_proportions).forEach((p) => {
+    draft.non_norm_counts[p] =
+      draft.non_norm_proportions[p] * draft.total_users;
+  });
 }
 
 export function toAudienceSize(draft) {