From 23f20962b2118620feac0a4e09405666032c7775 Mon Sep 17 00:00:00 2001 From: Xavier GUIHOT Date: Mon, 5 Feb 2018 18:01:08 +0100 Subject: [PATCH] Test object becomes a case class --- README.md | 8 +- build.sbt | 2 +- docs/com/spark_helper/monitoring/Monitor.html | 12 +- docs/com/spark_helper/monitoring/Test.html | 147 +++++++++++++----- docs/com/spark_helper/monitoring/package.html | 22 +-- docs/index.html | 2 +- docs/index.js | 2 +- docs/index/index-a.html | 3 + docs/index/index-d.html | 3 + docs/index/index-k.html | 6 + docs/index/index-t.html | 3 + .../com/spark_helper/monitoring/Monitor.scala | 12 +- .../com/spark_helper/monitoring/Test.scala | 14 +- .../spark_helper/monitoring/MonitorTest.scala | 10 +- 14 files changed, 163 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index cfc0eff..c4ff407 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## Overview -Version: 1.0.19 +Version: 1.0.20 API Scaladoc: [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$) @@ -131,7 +131,7 @@ With sbt, add these lines to your build.sbt: ```scala resolvers += "jitpack" at "https://jitpack.io" -libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.0.19" +libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.0.20" ``` With maven, add these lines to your pom.xml: @@ -147,7 +147,7 @@ With maven, add these lines to your pom.xml: com.github.xavierguihot spark_helper - v1.0.19 + v1.0.20 ``` @@ -161,7 +161,7 @@ allprojects { } dependencies { - compile 'com.github.xavierguihot:spark_helper:v1.0.19' + compile 'com.github.xavierguihot:spark_helper:v1.0.20' } ``` diff --git a/build.sbt b/build.sbt index 436aa07..43594bd 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "spark_helper" -version := "1.0.19" +version := "1.0.20" scalaVersion := "2.11.12" diff --git a/docs/com/spark_helper/monitoring/Monitor.html b/docs/com/spark_helper/monitoring/Monitor.html index 3d48412..e3ffdf1 100644 --- a/docs/com/spark_helper/monitoring/Monitor.html +++ b/docs/com/spark_helper/monitoring/Monitor.html @@ -65,8 +65,8 @@

// Let's say you want to get some KPIs on your output before storing it: val outputIsValid = monitor.updateByKpisValidation( List( - new Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), - new Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) + Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), + Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) ), "My pipeline descirption" ) @@ -581,7 +581,7 @@

validation and the success status of the monitoring is set to false if the KPI isn't valid.

If the validation is a failure then after that the status will never be success again, even if you update the report with success tasks.

Using this method like this:

monitor.updateByKpiValidation(
-  new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
+  Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
   "Tests for whatever")

will result in this to be appended to the report:

[10:35-10:37] Tests for whatever: success
   KPI: pctOfWhatever
     Value: 0.06%
@@ -609,9 +609,9 @@ 

if at least one KPI isn't valid.

If the validation of tests is a failure then after that the status will never be success again, even if you update the report with success tasks.

Using this method like this:

monitor.updateByKpisValidation(
   List(
-    new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
-    new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT),
-    new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
+    Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT),
+    Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT),
+    Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
   ),
   "Tests for whatever"
 )

will result in this to be appended to the report:

[10:35-10:37] Tests for whatever: failed
diff --git a/docs/com/spark_helper/monitoring/Test.html b/docs/com/spark_helper/monitoring/Test.html
index 448dc99..f1cd8b7 100644
--- a/docs/com/spark_helper/monitoring/Test.html
+++ b/docs/com/spark_helper/monitoring/Test.html
@@ -41,20 +41,22 @@ 

Test

Related Doc:

- - class + final + case class - Test extends AnyRef + Test(description: String, kpiValue: Double, thresholdType: ThresholdType, appliedThreshold: Double, kpiType: KpiType) extends Product with Serializable

A class which represents a KPI to validate.

This is intended to be used as parameter of Monitor.updateByKpiValidation -and Monitor.updateByKpisValidation methods.

Some exemples of Test objects:

new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
-new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
-new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
Since

2016-12

+and Monitor.updateByKpisValidation methods.

Some exemples of Test objects:

Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
+Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
+Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
description

the name/description of the KPI which will appear on the +validation report.

kpiValue

the value for this KPI

thresholdType

the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or +EQUAL_TO).

appliedThreshold

the threshold to apply

kpiType

the type of KPI (PCT or NBR)

Since

2016-12

Linear Supertypes -
AnyRef, Any
+
Serializable, Serializable, Product, Equals, AnyRef, Any
@@ -72,7 +74,7 @@

Inherited
    -
  1. Test
  2. AnyRef
  3. Any
  4. +
  5. Test
  6. Serializable
  7. Serializable
  8. Product
  9. Equals
  10. AnyRef
  11. Any

@@ -108,9 +110,9 @@

Creates a Test object.

Creates a Test object. -Some exemples of Test objects:

new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
-new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
-new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
description

the name/description of the KPI which will appear on the +Some exemples of Test objects:

Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
+Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
+Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
description

the name/description of the KPI which will appear on the validation report.

kpiValue

the value for this KPI

thresholdType

the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or EQUAL_TO).

appliedThreshold

the threshold to apply

kpiType

the type of KPI (PCT or NBR)

@@ -173,6 +175,23 @@

Definition Classes
AnyRef → Any
+
  • + + +

    + + + val + + + appliedThreshold: Double + +

    + + Permalink + + +

    the threshold to apply

  • @@ -213,40 +232,41 @@

    ) -

  • - - +
  • + +

    - final - def + + val - eq(arg0: AnyRef): Boolean + description: String

    - + Permalink -
    Definition Classes
    AnyRef
    -
  • - - +

    the name/description of the KPI which will appear on the +validation report.

    +
  • + +

    - + final def - equals(arg0: Any): Boolean + eq(arg0: AnyRef): Boolean

    - + Permalink -
    Definition Classes
    AnyRef → Any
    +
    Definition Classes
    AnyRef
  • @@ -287,40 +307,57 @@

    Definition Classes
    AnyRef → Any
    -

  • - - +
  • + +

    - + final def - hashCode(): Int + isInstanceOf[T0]: Boolean

    - + Permalink -
    Definition Classes
    AnyRef → Any
    -
  • - - +
    Definition Classes
    Any
    +
  • + +

    - final - def + + val - isInstanceOf[T0]: Boolean + kpiType: KpiType

    - + Permalink -
    Definition Classes
    Any
    +

    the type of KPI (PCT or NBR)

    +
  • + + +

    + + + val + + + kpiValue: Double + +

    + + Permalink + + +

    the value for this KPI

  • @@ -389,6 +426,24 @@

    Definition Classes
    AnyRef
    +

  • + + +

    + + + val + + + thresholdType: ThresholdType + +

    + + Permalink + + +

    the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or +EQUAL_TO).

  • @@ -484,7 +539,15 @@

    -
    +
    +

    Inherited from Serializable

    +
    +

    Inherited from Serializable

    +
    +

    Inherited from Product

    +
    +

    Inherited from Equals

    +

    Inherited from AnyRef

    Inherited from Any

    diff --git a/docs/com/spark_helper/monitoring/package.html b/docs/com/spark_helper/monitoring/package.html index 9f37e62..c15c60a 100644 --- a/docs/com/spark_helper/monitoring/package.html +++ b/docs/com/spark_helper/monitoring/package.html @@ -115,8 +115,8 @@

    // Let's say you want to get some KPIs on your output before storing it: val outputIsValid = monitor.updateByKpisValidation( List( - new Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), - new Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) + Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), + Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) ), "My pipeline descirption" ) @@ -193,25 +193,27 @@

    /main/scala/com/spark_helper/monitoring/Monitor.scala">Monitor

    Since

    2017-02

  • - +

    - - class + final + case class - Test extends AnyRef + Test(description: String, kpiValue: Double, thresholdType: ThresholdType, appliedThreshold: Double, kpiType: KpiType) extends Product with Serializable

    - + Permalink

    A class which represents a KPI to validate.

    A class which represents a KPI to validate.

    This is intended to be used as parameter of Monitor.updateByKpiValidation -and Monitor.updateByKpisValidation methods.

    Some exemples of Test objects:

    new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
    -new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
    -new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
    Since

    2016-12

    +and Monitor.updateByKpisValidation methods.

    Some exemples of Test objects:

    Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
    +Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
    +Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR)
    description

    the name/description of the KPI which will appear on the +validation report.

    kpiValue

    the value for this KPI

    thresholdType

    the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or +EQUAL_TO).

    appliedThreshold

    the threshold to apply

    kpiType

    the type of KPI (PCT or NBR)

    Since

    2016-12

  • diff --git a/docs/index.html b/docs/index.html index e14b9e1..d733151 100644 --- a/docs/index.html +++ b/docs/index.html @@ -38,7 +38,7 @@
    1. (object)
      DateHelper
    2. (object)
      HdfsHelper
    3. (object)
      SparkHelper
    1. com.spark_helper.monitoring -
      1. (object)
        EQUAL_TO
      2. (object)
        INFERIOR_THAN
      3. (trait)KpiType
      4. (class)Monitor
      5. (object)
        NBR
      6. (object)
        PCT
      7. (object)
        SUPERIOR_THAN
      8. (class)Test
      9. (trait)ThresholdType
      +
      1. (object)
        EQUAL_TO
      2. (object)
        INFERIOR_THAN
      3. (trait)KpiType
      4. (class)Monitor
      5. (object)
        NBR
      6. (object)
        PCT
      7. (object)
        SUPERIOR_THAN
      8. (case class)Test
      9. (trait)ThresholdType
  • diff --git a/docs/index.js b/docs/index.js index b6778d0..6d54c9b 100644 --- a/docs/index.js +++ b/docs/index.js @@ -1 +1 @@ -Index.PACKAGES = {"com" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"class" : "com\/spark_helper\/monitoring\/Monitor.html", "name" : "com.spark_helper.monitoring.Monitor"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}]}; \ No newline at end of file +Index.PACKAGES = {"com" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"class" : "com\/spark_helper\/monitoring\/Monitor.html", "name" : "com.spark_helper.monitoring.Monitor"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"case class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}]}; \ No newline at end of file diff --git a/docs/index/index-a.html b/docs/index/index-a.html index 0359fcb..16e8c0e 100644 --- a/docs/index/index-a.html +++ b/docs/index/index-a.html @@ -19,5 +19,8 @@
    appendHeaderAndFooter
    +
    +
    appliedThreshold
    +
    diff --git a/docs/index/index-d.html b/docs/index/index-d.html index ead6c4e..860d355 100644 --- a/docs/index/index-d.html +++ b/docs/index/index-d.html @@ -31,5 +31,8 @@
    deleteFolder
    +
    +
    description
    +
    diff --git a/docs/index/index-k.html b/docs/index/index-k.html index 467fbc1..63990cb 100644 --- a/docs/index/index-k.html +++ b/docs/index/index-k.html @@ -13,5 +13,11 @@
    KpiType
    +
    +
    kpiType
    + +
    +
    kpiValue
    +
    diff --git a/docs/index/index-t.html b/docs/index/index-t.html index c097dd3..67f3a04 100644 --- a/docs/index/index-t.html +++ b/docs/index/index-t.html @@ -19,6 +19,9 @@
    textFileWithDelimiter
    +
    +
    thresholdType
    +
    toString
    diff --git a/src/main/scala/com/spark_helper/monitoring/Monitor.scala b/src/main/scala/com/spark_helper/monitoring/Monitor.scala index cfcc64a..8dba0c3 100644 --- a/src/main/scala/com/spark_helper/monitoring/Monitor.scala +++ b/src/main/scala/com/spark_helper/monitoring/Monitor.scala @@ -31,8 +31,8 @@ import java.lang.Throwable * // Let's say you want to get some KPIs on your output before storing it: * val outputIsValid = monitor.updateByKpisValidation( * List( - * new Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), - * new Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) + * Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR), + * Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT) * ), * "My pipeline descirption" * ) @@ -310,9 +310,9 @@ class Monitor( * {{{ * monitor.updateByKpisValidation( * List( - * new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), - * new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT), - * new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) + * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), + * Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT), + * Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) * ), * "Tests for whatever" * ) @@ -377,7 +377,7 @@ class Monitor( * Using this method like this: * {{{ * monitor.updateByKpiValidation( - * new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), + * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), * "Tests for whatever") * }}} * will result in this to be appended to the report: diff --git a/src/main/scala/com/spark_helper/monitoring/Test.scala b/src/main/scala/com/spark_helper/monitoring/Test.scala index f777e21..83cb4b6 100644 --- a/src/main/scala/com/spark_helper/monitoring/Test.scala +++ b/src/main/scala/com/spark_helper/monitoring/Test.scala @@ -9,9 +9,9 @@ import java.lang.Math.abs * * Some exemples of Test objects: * {{{ - * new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT) - * new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT) - * new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) + * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT) + * Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT) + * Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) * }}} * * @author Xavier Guihot @@ -21,9 +21,9 @@ import java.lang.Math.abs * * Some exemples of Test objects: * {{{ - * new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT) - * new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT) - * new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) + * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT) + * Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT) + * Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) * }}} * * @param description the name/description of the KPI which will appear on the @@ -34,7 +34,7 @@ import java.lang.Math.abs * @param appliedThreshold the threshold to apply * @param kpiType the type of KPI (PCT or NBR) */ -class Test( +final case class Test( description: String, kpiValue: Double, thresholdType: ThresholdType, diff --git a/src/test/scala/com/spark_helper/monitoring/MonitorTest.scala b/src/test/scala/com/spark_helper/monitoring/MonitorTest.scala index bcc3c88..1ec64a5 100644 --- a/src/test/scala/com/spark_helper/monitoring/MonitorTest.scala +++ b/src/test/scala/com/spark_helper/monitoring/MonitorTest.scala @@ -136,7 +136,7 @@ class MonitorTest extends FunSuite with SharedSparkContext { try { "a".toInt } catch { - case nfe: NumberFormatException => + case nfe: NumberFormatException => monitor.updateReportWithError(nfe, "Parse to integer", "my diagnostic") } // Warning, here I remove the stack trace because it depends on the @@ -157,9 +157,9 @@ class MonitorTest extends FunSuite with SharedSparkContext { var monitor = new Monitor() var success = monitor.updateByKpisValidation( List( - new Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), - new Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT), - new Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) + Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT), + Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT), + Test("someNbr", 1235d, EQUAL_TO, 1235d, NBR) ), "Tests for whatever" ) @@ -189,7 +189,7 @@ class MonitorTest extends FunSuite with SharedSparkContext { // 2: Single test: monitor = new Monitor() success = monitor.updateByKpiValidation( - new Test("someNbr", 55e6d, SUPERIOR_THAN, 50e6d, NBR), + Test("someNbr", 55e6d, SUPERIOR_THAN, 50e6d, NBR), "Tests for whatever") assert(success)