From 23f20962b2118620feac0a4e09405666032c7775 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT
Date: Mon, 5 Feb 2018 18:01:08 +0100
Subject: [PATCH] Test object becomes a case class
---
README.md | 8 +-
build.sbt | 2 +-
docs/com/spark_helper/monitoring/Monitor.html | 12 +-
docs/com/spark_helper/monitoring/Test.html | 147 +++++++++++++-----
docs/com/spark_helper/monitoring/package.html | 22 +--
docs/index.html | 2 +-
docs/index.js | 2 +-
docs/index/index-a.html | 3 +
docs/index/index-d.html | 3 +
docs/index/index-k.html | 6 +
docs/index/index-t.html | 3 +
.../com/spark_helper/monitoring/Monitor.scala | 12 +-
.../com/spark_helper/monitoring/Test.scala | 14 +-
.../spark_helper/monitoring/MonitorTest.scala | 10 +-
14 files changed, 163 insertions(+), 83 deletions(-)
diff --git a/README.md b/README.md
index cfc0eff..c4ff407 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
## Overview
-Version: 1.0.19
+Version: 1.0.20
API Scaladoc: [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$)
@@ -131,7 +131,7 @@ With sbt, add these lines to your build.sbt:
```scala
resolvers += "jitpack" at "https://jitpack.io"
-libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.0.19"
+libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.0.20"
```
With maven, add these lines to your pom.xml:
@@ -147,7 +147,7 @@ With maven, add these lines to your pom.xml:
com.github.xavierguihot
spark_helper
- v1.0.19
+ v1.0.20
```
@@ -161,7 +161,7 @@ allprojects {
}
dependencies {
- compile 'com.github.xavierguihot:spark_helper:v1.0.19'
+ compile 'com.github.xavierguihot:spark_helper:v1.0.20'
}
```
diff --git a/build.sbt b/build.sbt
index 436aa07..43594bd 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,6 +1,6 @@
name := "spark_helper"
-version := "1.0.19"
+version := "1.0.20"
scalaVersion := "2.11.12"
diff --git a/docs/com/spark_helper/monitoring/Monitor.html b/docs/com/spark_helper/monitoring/Monitor.html
index 3d48412..e3ffdf1 100644
--- a/docs/com/spark_helper/monitoring/Monitor.html
+++ b/docs/com/spark_helper/monitoring/Monitor.html
@@ -65,8 +65,8 @@
// Let's say you want to get some KPIs on your output before storing it:
val outputIsValid = monitor.updateByKpisValidation(
List(
- new Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR),
- new Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT)
+ Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR),
+ Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT)
),
"My pipeline descirption"
)
@@ -581,7 +581,7 @@
validation and the success status of the monitoring is set to false if the
KPI isn't valid.
If the validation is a failure then after that the status will never be
success again, even if you update the report with success tasks.
Using this method like this:
monitor.updateByKpiValidation(
- new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
+ Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
"Tests for whatever")
will result in this to be appended to the report:
[10:35-10:37] Tests for whatever: success
KPI: pctOfWhatever
Value: 0.06%
@@ -609,9 +609,9 @@
if at least one KPI isn't valid.
If the validation of tests is a failure then after that the status will
never be success again, even if you update the report with success tasks.
Using this method like this:
monitor.updateByKpisValidation(
List(
- new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
- new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT),
- new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
+ Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
+ Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT),
+ Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
),
"Tests for whatever"
)
will result in this to be appended to the report:
[10:35-10:37] Tests for whatever: failed
diff --git a/docs/com/spark_helper/monitoring/Test.html b/docs/com/spark_helper/monitoring/Test.html
index 448dc99..f1cd8b7 100644
--- a/docs/com/spark_helper/monitoring/Test.html
+++ b/docs/com/spark_helper/monitoring/Test.html
@@ -41,20 +41,22 @@ Test
Related Doc:
-
- class
+ final
+ case class
- Test extends AnyRef
+ Test(description: String, kpiValue: Double, thresholdType: ThresholdType, appliedThreshold: Double, kpiType: KpiType) extends Product with Serializable
@@ -72,7 +74,7 @@
Inherited
- - Test
- AnyRef
- Any
+ - Test
- Serializable
- Serializable
- Product
- Equals
- AnyRef
- Any
@@ -108,9 +110,9 @@
- description
the name/description of the KPI which will appear on the
validation report.
- kpiValue
the value for this KPI
- thresholdType
the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or
EQUAL_TO).
- appliedThreshold
the threshold to apply
- kpiType
the type of KPI (PCT or NBR)
@@ -173,6 +175,23 @@
+
+
+
+
+
+
+ val
+
+
+ appliedThreshold: Double
+
+
+
+
+
+
+
@@ -213,40 +232,41 @@
)
-
-
-
+
+
+
- final
- def
+
+ val
- eq(arg0: AnyRef): Boolean
+ description: String
-
+
-
-
-
-
+
+
+
+
-
+ final
def
- equals(arg0: Any): Boolean
+ eq(arg0: AnyRef): Boolean
-
+
-
+
@@ -287,40 +307,57 @@
-
-
-
+
+
+
-
+ final
def
- hashCode(): Int
+ isInstanceOf[T0]: Boolean
-
+
-
-
-
-
+
+
+
+
- final
- def
+
+ val
- isInstanceOf[T0]: Boolean
+ kpiType: KpiType
-
+
-
+
+
+
+
+
+
+
+ val
+
+
+ kpiValue: Double
+
+
+
+
+
+
+
@@ -389,6 +426,24 @@
+
+
+
+
+
+
+ val
+
+
+ thresholdType: ThresholdType
+
+
+
+
+
+
+
@@ -484,7 +539,15 @@
-
+
+
Inherited from Serializable
+
+
Inherited from Serializable
+
+
Inherited from Product
+
+
Inherited from Equals
+
Inherited from AnyRef
Inherited from Any
diff --git a/docs/com/spark_helper/monitoring/package.html b/docs/com/spark_helper/monitoring/package.html
index 9f37e62..c15c60a 100644
--- a/docs/com/spark_helper/monitoring/package.html
+++ b/docs/com/spark_helper/monitoring/package.html
@@ -115,8 +115,8 @@
// Let's say you want to get some KPIs on your output before storing it:
val outputIsValid = monitor.updateByKpisValidation(
List(
- new Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR),
- new Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT)
+ Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR),
+ Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT)
),
"My pipeline descirption"
)
@@ -193,25 +193,27 @@
/main/scala/com/spark_helper/monitoring/Monitor.scala">Monitor
- Since
2017-02
-
+
-
- class
+ final
+ case class
- Test extends AnyRef
+ Test(description: String, kpiValue: Double, thresholdType: ThresholdType, appliedThreshold: Double, kpiType: KpiType) extends Product with Serializable
-
+
+and Monitor.updateByKpisValidation methods.Some exemples of Test objects:
Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
+Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
+Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
- description
the name/description of the KPI which will appear on the
+validation report.
- kpiValue
the value for this KPI
- thresholdType
the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or
+EQUAL_TO).
- appliedThreshold
the threshold to apply
- kpiType
the type of KPI (PCT or NBR)
- Since
2016-12
diff --git a/docs/index.html b/docs/index.html
index e14b9e1..d733151 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -38,7 +38,7 @@
- (object)DateHelper
- (object)HdfsHelper
- (object)SparkHelper
-
com.spark_helper.monitoring
-
- (object)EQUAL_TO
- (object)INFERIOR_THAN
- (trait)KpiType
- (class)Monitor
- (object)NBR
- (object)PCT
- (object)SUPERIOR_THAN
- (class)Test
- (trait)ThresholdType
+ - (object)EQUAL_TO
- (object)INFERIOR_THAN
- (trait)KpiType
- (class)Monitor
- (object)NBR
- (object)PCT
- (object)SUPERIOR_THAN
- (case class)Test
- (trait)ThresholdType
diff --git a/docs/index.js b/docs/index.js
index b6778d0..6d54c9b 100644
--- a/docs/index.js
+++ b/docs/index.js
@@ -1 +1 @@
-Index.PACKAGES = {"com" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"class" : "com\/spark_helper\/monitoring\/Monitor.html", "name" : "com.spark_helper.monitoring.Monitor"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}]};
\ No newline at end of file
+Index.PACKAGES = {"com" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"class" : "com\/spark_helper\/monitoring\/Monitor.html", "name" : "com.spark_helper.monitoring.Monitor"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"case class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}]};
\ No newline at end of file
diff --git a/docs/index/index-a.html b/docs/index/index-a.html
index 0359fcb..16e8c0e 100644
--- a/docs/index/index-a.html
+++ b/docs/index/index-a.html
@@ -19,5 +19,8 @@
A class which represents a KPI to validate.
This is intended to be used as parameter of Monitor.updateByKpiValidation -and Monitor.updateByKpisValidation methods.
Some exemples of Test objects:
2016-12
Some exemples of Test objects:
the name/description of the KPI which will appear on the +validation report.
the value for this KPI
the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or +EQUAL_TO).
the threshold to apply
the type of KPI (PCT or NBR)
2016-12