feat: extend runner with --only-failures option

Package.swift

@@ -25,6 +25,7 @@ let package = Package(
         .package(url: "https://github.com/JohnSundell/Files", from: "4.0.0"),
         .package(url: "https://github.com/jpsim/Yams.git", from: "5.0.6"),
         .package(url: "https://github.com/qiuzhifei/swift-commands", from: "0.6.0"),
+        .package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0")
     ],
     targets: [
         // Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -48,6 +49,7 @@ let package = Package(
                 "Files",
                 "Yams",
                 .product(name: "Commands", package: "swift-commands"),
+                .product(name: "ArgumentParser", package: "swift-argument-parser")
             ],
             path: "Sources/FeaturevisorTestRunner"
         ),

README.md

@@ -322,6 +322,14 @@ let f = try createInstance(options: options)
 
 @TODO: Work still in progress. Currently we have an early POC.
 
+### Options
+
+```bash
+'only-failures'
+```
+
+If you are interested to see only the test specs that fail:
+
 Example command:
 
 First you need to install the Swift Test Runner using above steps (until we release official version)

Sources/FeaturevisorTestRunner/FeaturevisorTestRunner+Print.swift

@@ -0,0 +1,51 @@
+import FeaturevisorTypes
+import Foundation
+
+extension FeaturevisorTestRunner {
+
+    enum ResultMark: String {
+        case success = "✔"
+        case failure = "✘"
+
+        init(result: Bool) {
+            self = result ? .success : .failure
+        }
+
+        var mark: String {
+            return rawValue
+        }
+    }
+
+    func printAssertionResult(
+        environment: Environment,
+        index: Int,
+        feature: String,
+        assertionResult: Bool,
+        expectedValueFailures: [VariableKey: (expected: VariableValue, got: VariableValue?)],
+        description: String,
+        onlyFailures: Bool
+    ) {
+
+        guard onlyFailures && !assertionResult || !onlyFailures else {
+            return
+        }
+
+        print("\nTesting: \(feature).feature.yml")
+        print(" feature \"\(feature)\"")
+
+        let resultMark = ResultMark(result: assertionResult)
+
+        print(" \(resultMark) Assertion #\(index): \(environment.rawValue) \(description)")
+
+        expectedValueFailures.forEach({ (key, value) in
+            print("   => variable key: \(key)")
+            print("          => expected: \(value.expected)")
+            if let got = value.got {
+                print("          => received: \(got)")
+            }
+            else {
+                print("          => received: nil")
+            }
+        })
+    }
+}

Sources/FeaturevisorTestRunner/FeaturevisorTestRunner.swift

@@ -1,3 +1,4 @@
+import ArgumentParser
 import Commands
 import FeaturevisorSDK
 import FeaturevisorTypes
@@ -6,16 +7,20 @@ import Foundation
 import Yams
 
 @main
-struct FeaturevisorTestRunner {
+struct FeaturevisorTestRunner: ParsableCommand {
 
-    public static func main() throws {
-        try FeaturevisorTestRunner().run()
-    }
+    static let configuration = CommandConfiguration(
+        abstract:
+            "We can write test specs in the same expressive way as we defined our features to test against Featurevisor Swift SDK."
+    )
+
+    @Argument(help: "The path to features test directory.")
+    var featuresTestDirectoryPath: String
 
-    func run() throws {
+    @Flag(help: "If you are interested to see only the test specs that fail.")
+    var onlyFailures = false
 
-        // TODO Handle command line parameters better
-        let featuresTestDirectoryPath = CommandLine.arguments[1]
+    mutating func run() throws {
 
         // Run Featurevisor CLI to build the datafiles
         // TODO: Handle better, react on errors etc.
@@ -44,9 +49,6 @@ struct FeaturevisorTestRunner {
             totalTestSpecs += 1
             totalAssertionsCount += testSuit.assertions.count
 
-            print("\nTesting: \(testSuit.feature).feature.yml")
-            print(" \(testSuit.feature).feature.yml")
-
             var isTestSpecFailing = false
 
             for (index, testCase) in testSuit.assertions.enumerated() {
@@ -76,7 +78,7 @@ struct FeaturevisorTestRunner {
 
                 let isFeatureEnabledResult: Bool
 
-                var expectedValuesFalses:
+                var expectedValueFailures:
                     [VariableKey: (expected: VariableValue, got: VariableValue?)] = [:]
 
                 switch testCase.environment {
@@ -117,38 +119,28 @@ struct FeaturevisorTestRunner {
                                     )
 
                                 if variable != variableExpectedValue {
-                                    expectedValuesFalses[variableKey] = (
+                                    expectedValueFailures[variableKey] = (
                                         variableExpectedValue, variable
                                     )
                                 }
                             })
 
                         let finalAssertionResult =
-                            isFeatureEnabledResult && expectedValuesFalses.isEmpty
-
-                        let resultMark = finalAssertionResult ? "✔" : "✘"
+                            isFeatureEnabledResult && expectedValueFailures.isEmpty
 
                         isTestSpecFailing = isTestSpecFailing || !finalAssertionResult
                         failedAssertionsCount =
                             finalAssertionResult ? failedAssertionsCount : failedAssertionsCount + 1
 
-                        //                    guard !result else { // Skip passing assertion
-                        //                        break
-                        //                    }
-
-                        print(
-                            " \(resultMark) Assertion #\(index): (staging) \(testCase.description)"
+                        printAssertionResult(
+                            environment: testCase.environment,
+                            index: index,
+                            feature: testSuit.feature,
+                            assertionResult: finalAssertionResult,
+                            expectedValueFailures: expectedValueFailures,
+                            description: testCase.description,
+                            onlyFailures: onlyFailures
                         )
-                        expectedValuesFalses.forEach({ (key, value) in
-                            print("   => variable key: \(key)")
-                            print("          => expected: \(value.expected)")
-                            if let got = value.got {
-                                print("          => received: \(got)")
-                            }
-                            else {
-                                print("          => received: nil")
-                            }
-                        })
 
                     case .production:
 
@@ -187,38 +179,28 @@ struct FeaturevisorTestRunner {
                                     )
 
                                 if variable != variableExpectedValue {
-                                    expectedValuesFalses[variableKey] = (
+                                    expectedValueFailures[variableKey] = (
                                         variableExpectedValue, variable
                                     )
                                 }
                             })
 
                         let finalAssertionResult =
-                            isFeatureEnabledResult && expectedValuesFalses.isEmpty
-
-                        let resultMark = finalAssertionResult ? "✔" : "✘"
+                            isFeatureEnabledResult && expectedValueFailures.isEmpty
 
                         isTestSpecFailing = isTestSpecFailing || !finalAssertionResult
                         failedAssertionsCount =
                             finalAssertionResult ? failedAssertionsCount : failedAssertionsCount + 1
 
-                        //                    guard !result else { // Skip passing assertion
-                        //                        break
-                        //                    }
-
-                        print(
-                            " \(resultMark) Assertion #\(index): (production) \(testCase.description)"
+                        printAssertionResult(
+                            environment: testCase.environment,
+                            index: index,
+                            feature: testSuit.feature,
+                            assertionResult: finalAssertionResult,
+                            expectedValueFailures: expectedValueFailures,
+                            description: testCase.description,
+                            onlyFailures: onlyFailures
                         )
-                        expectedValuesFalses.forEach({ (key, value) in
-                            print("   => variable key: \(key)")
-                            print("          => expected: \(value.expected)")
-                            if let got = value.got {
-                                print("          => received: \(got)")
-                            }
-                            else {
-                                print("          => received: nil")
-                            }
-                        })
                 }
             }