result.yaml more detailed, option to only return critical errors (whi…

…ch prevent the tests to complete)

Signed-off-by: Matthias Büchse <matthias.buechse@cloudandheat.com>

Tests/iaas/flavor-naming/flavor-names-openstack.py

@@ -58,7 +58,7 @@ def main(argv):
                                        ("os-cloud=", "mand=", "verbose", "help", "quiet", "v2plus",
                                         "v3", "v1prefer", "accept-old-mandatory"))
     except getopt.GetoptError as exc:
-        print(f"{exc}", file=sys.stderr)
+        print(f"CRITICAL: {exc!r}", file=sys.stderr)
         usage(1)
     for opt in opts:
         if opt[0] == "-h" or opt[0] == "--help":
@@ -85,13 +85,14 @@ def main(argv):
         else:
             usage(2)
     if len(args) > 0:
-        print(f"Extra arguments {str(args)}", file=sys.stderr)
+        print(f"CRITICAL: Extra arguments {str(args)}", file=sys.stderr)
         usage(1)
 
     scsMandatory, scsRecommended = fnmck.readflavors(scsMandFile, v3mode, fnmck.prefer_old)
 
     if not cloud:
-        print("ERROR: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr)
+        print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr)
+        sys.exit(1)
     conn = openstack.connect(cloud=cloud, timeout=32)
     flavors = conn.compute.flavors()
 
@@ -187,7 +188,7 @@ def main(argv):
         except NameError as exc:
             errors += 1
             wrongFlv.append(flv.name)
-            print(f"Wrong flavor \"{flv.name}\": {exc}", file=sys.stderr)
+            print(f"ERROR: Wrong flavor \"{flv.name}\": {exc}", file=sys.stderr)
     # This makes the output more readable
     MSCSFlv.sort()
     RSCSFlv.sort()
@@ -198,6 +199,7 @@ def main(argv):
     # We have counted errors on the fly, add missing flavors to the final result
     if scsMandatory:
         errors += len(scsMandatory)
+        print(f"ERROR: Missing mandatory flavors: {', '.join(scsMandatory)}", file=sys.stderr)
     # Produce dicts for YAML reporting
     flvSCSList = {
         "MandatoryFlavorsPresent": MSCSFlv,

Tests/iaas/image-metadata/image-md-check.py

@@ -216,6 +216,7 @@ def main(argv):
         opts, args = getopt.gnu_getopt(argv[1:], "phvc:s",
                                        ("private", "help", "os-cloud=", "verbose", "skip-completeness"))
     except getopt.GetoptError:  # as exc:
+        print("CRITICAL: Command-line syntax error", file=sys.stderr)
         usage(1)
     for opt in opts:
         if opt[0] == "-h" or opt[0] == "--help":
@@ -230,18 +231,21 @@ def main(argv):
             cloud = opt[1]
     images = args
     if not cloud:
-        print("ERROR: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr)
+        print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr)
         usage(1)
-    conn = openstack.connect(cloud=cloud, timeout=24)
-    # Do work
-    if not images:
-        images = get_imagelist(private)
-    err = 0
-    # Analyse image metadata
-    for image in images:
-        err += validate_imageMD(image)
-    if not skip:
-        err += report_stdimage_coverage(images)
+    try:
+        conn = openstack.connect(cloud=cloud, timeout=24)
+        # Do work
+        if not images:
+            images = get_imagelist(private)
+        err = 0
+        # Analyse image metadata
+        for image in images:
+            err += validate_imageMD(image)
+        if not skip:
+            err += report_stdimage_coverage(images)
+    except BaseException as e:
+        print(f"CRITICAL: {e!r}")
     return err
 
 

Tests/scs-compliance-check.py

@@ -30,6 +30,7 @@
 import datetime
 import subprocess
 import copy
+from itertools import chain
 import yaml
 
 
@@ -39,16 +40,20 @@
 KEYWORDS_CHECKTOOL = ('executable', 'args', 'condition', 'classification')
 
 
-def usage():
-    "Output usage information"
-    print("Usage: scs-compliance-check.py [options] compliance-spec.yaml layer [layer [layer]]")
-    print("Options: -v/--verbose: More verbose output")
-    print(" -q/--quiet: Don't output anything but errors")
-    print(" -s/--single-layer: Don't perform required checks for dependant layers")
-    print(" -d/--date YYYY-MM-DD: Check standards valid on specified date instead of today")
-    print(" -V/--version VERS: Force version VERS of the standard (instead of deriving from date)")
-    print(" -c/--os-cloud CLOUD: Use specified cloud env (instead of OS_CLOUD env var)")
-    print(" -o/--output path: Generate yaml report of compliance check under given path")
+def usage(file=sys.stdout):
+    """Output usage information"""
+    print("""Usage: scs-compliance-check.py [options] compliance-spec.yaml
+Options: -v/--verbose: More verbose output
+ -q/--quiet: Don't output anything but errors
+ -s/--single-scope: Don't perform required checks for prerequisite scopes
+ -d/--date YYYY-MM-DD: Check standards valid on specified date instead of today
+ -V/--version VERS: Force version VERS of the standard (instead of deriving from date)
+ -c/--os-cloud CLOUD: Use specified cloud env (instead of OS_CLOUD env var)
+ -o/--output REPORT_PATH: Generate yaml report of compliance check under given path
+ -C/--critical-only: Only return critical errors in return code
+
+With -C, the return code will be nonzero precisely when the tests couldn't be run to completion.
+""".strip(), file=file)
 
 
 MYPATH = "."
@@ -97,59 +102,65 @@ def run_check_tool(executable, args, os_cloud, verbose=False, quiet=False):
         print(compl.stdout)
     if not quiet:
         print(compl.stderr, file=sys.stderr)
-    return compl.returncode
+    return compl
 
 
 def errcode_to_text(err):
     "translate error code to text"
     return f"{err} ERRORS" if err else "PASSED"
 
 
-def optparse(argv):
-    """
-    Parse options.
-    Return (args0, verbose, quiet, os_cloud, checkdate, version, output, classes).
-    """
-    verbose = False
-    quiet = False
-    os_cloud = os.environ.get("OS_CLOUD")
-    checkdate = datetime.date.today()
-    version = None
-    output = None
-    classes = ["light", "medium", "heavy"]
-    try:
-        opts, args = getopt.gnu_getopt(argv, "hvqd:V:sc:o:r:", (
-            "help", "verbose", "quiet", "date=", "version=",
-            "os-cloud=", "output=", "resource-usage=",
-        ))
-    except getopt.GetoptError as exc:
-        print(f"Option error: {exc}", file=sys.stderr)
-        usage()
-        sys.exit(1)
-    for opt in opts:
-        if opt[0] == "-h" or opt[0] == "--help":
+class Config:
+    def __init__(self):
+        self.arg0 = None
+        self.verbose = False
+        self.quiet = False
+        self.os_cloud = os.environ.get("OS_CLOUD")
+        self.checkdate = datetime.date.today()
+        self.version = None
+        self.output = None
+        self.classes = ["light", "medium", "heavy"]
+        self.critical_only = False
+
+    def apply_argv(self, argv):
+        """
+        Parse options. May exit the program.
+        """
+        try:
+            opts, args = getopt.gnu_getopt(argv, "hvqd:V:sc:o:r:C", (
+                "help", "verbose", "quiet", "date=", "version=",
+                "os-cloud=", "output=", "resource-usage=", "critical-only"
+            ))
+        except getopt.GetoptError as exc:
+            print(f"Option error: {exc}", file=sys.stderr)
             usage()
-            sys.exit(0)
-        elif opt[0] == "-v" or opt[0] == "--verbose":
-            verbose = True
-        elif opt[0] == "-q" or opt[0] == "--quiet":
-            quiet = True
-        elif opt[0] == "-d" or opt[0] == "--date":
-            checkdate = datetime.date.fromisoformat(opt[1])
-        elif opt[0] == "-V" or opt[0] == "--version":
-            version = opt[1]
-        elif opt[0] == "-c" or opt[0] == "--os-cloud":
-            os_cloud = opt[1]
-        elif opt[0] == "-o" or opt[0] == "--output":
-            output = opt[1]
-        elif opt[0] == "-r" or opt[0] == "--resource-usage":
-            classes = [x.strip() for x in opt[1].split(",")]
-        else:
-            print(f"Error: Unknown argument {opt[0]}", file=sys.stderr)
-    if len(args) < 1:
-        usage()
-        sys.exit(1)
-    return (args[0], verbose, quiet, os_cloud, checkdate, version, output, classes)
+            sys.exit(1)
+        for opt in opts:
+            if opt[0] == "-h" or opt[0] == "--help":
+                usage()
+                sys.exit(0)
+            elif opt[0] == "-v" or opt[0] == "--verbose":
+                self.verbose = True
+            elif opt[0] == "-q" or opt[0] == "--quiet":
+                self.quiet = True
+            elif opt[0] == "-d" or opt[0] == "--date":
+                self.checkdate = datetime.date.fromisoformat(opt[1])
+            elif opt[0] == "-V" or opt[0] == "--version":
+                self.version = opt[1]
+            elif opt[0] == "-c" or opt[0] == "--os-cloud":
+                self.os_cloud = opt[1]
+            elif opt[0] == "-o" or opt[0] == "--output":
+                self.output = opt[1]
+            elif opt[0] == "-r" or opt[0] == "--resource-usage":
+                self.classes = [x.strip() for x in opt[1].split(",")]
+            elif opt[0] == "-C" or opt[0] == "--critical-only":
+                self.critical_only = True
+            else:
+                print(f"Error: Unknown argument {opt[0]}", file=sys.stderr)
+        if len(args) < 1:
+            usage(file=sys.stderr)
+            sys.exit(1)
+        self.arg0 = args[0]
 
 
 def condition_optional(cond, default=False):
@@ -175,33 +186,35 @@ def check_keywords(d, ctx, valid=()):
 
 def main(argv):
     """Entry point for the checker"""
-    args0, verbose, quiet, os_cloud, checkdate, version, output, classes = optparse(argv)
-    if not os_cloud:
+    config = Config()
+    config.apply_argv(argv)
+    if not config.os_cloud:
         print("You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr)
         return 1
-    with open(args0, "r", encoding="UTF-8") as specfile:
+    with open(config.arg0, "r", encoding="UTF-8") as specfile:
         specdict = yaml.load(specfile, Loader=yaml.SafeLoader)
+    allaborts = 0
     allerrors = 0
     report = copy.deepcopy(specdict)
     check_keywords(report, 'spec', KEYWORDS_SPEC)
     run_report = report["run"] = {}
-    run_report["os_cloud"] = os_cloud
+    run_report["os_cloud"] = config.os_cloud
     # TODO: Add kubeconfig context as well
-    run_report["checked_at"] = checkdate
-    run_report["classes"] = classes
-    if version:
-        run_report["forced_version"] = version
-        report["versions"] = [vd for vd in report["versions"] if vd["version"] == version]
+    run_report["checked_at"] = config.checkdate
+    run_report["classes"] = config.classes
+    if config.version:
+        run_report["forced_version"] = config.version
+        report["versions"] = [vd for vd in report["versions"] if vd["version"] == config.version]
     if "prerequisite" in specdict:
         print("WARNING: prerequisite not yet implemented!", file=sys.stderr)
-    memo = {}  # memoize check tool results
+    memo = run_report["invokations"] = {}  # memoize check tool results
     matches = 0
     for vd in report["versions"]:
         check_keywords(vd, 'version', KEYWORDS_VERSION)
         stb_date = vd.get("stabilized_at")
         obs_date = vd.get("obsoleted_at")
-        futuristic = not stb_date or checkdate < stb_date
-        outdated = obs_date and obs_date < checkdate
+        futuristic = not stb_date or config.checkdate < stb_date
+        outdated = obs_date and obs_date < config.checkdate
         vd.update({
             "status": "n/a",
             "passed": False,
@@ -212,23 +225,24 @@ def main(argv):
             vd["status"] = "preview"
         else:
             vd["status"] = "valid"
-        if outdated and not version:
+        if outdated and not config.version:
             continue
         matches += 1
-        if version and outdated:
-            print(f"WARNING: Forced version {version} outdated", file=sys.stderr)
-        if version and futuristic:
-            print(f"INFO: Forced version {version} not (yet) stable", file=sys.stderr)
-        if not quiet:
+        if config.version and outdated:
+            print(f"WARNING: Forced version {config.version} outdated", file=sys.stderr)
+        if config.version and futuristic:
+            print(f"INFO: Forced version {config.version} not (yet) stable", file=sys.stderr)
+        if not config.quiet:
             print(f"Testing {specdict['name']} version {vd['version']}")
         if "standards" not in vd:
             print(f"WARNING: No standards defined yet for {specdict['name']} version {vd['version']}",
                   file=sys.stderr)
         errors = 0
+        aborts = 0
         for standard in vd["standards"]:
             check_keywords(standard, 'standard', KEYWORDS_STANDARD)
             optional = condition_optional(standard)
-            if not quiet:
+            if not config.quiet:
                 print("*******************************************************")
                 print(f"Testing {'optional ' * optional}standard {standard['name']} ...")
                 print(f"Reference: {standard['url']} ...")
@@ -237,34 +251,51 @@ def main(argv):
                 continue
             for check in standard["check_tools"]:
                 check_keywords(check, 'checktool', KEYWORDS_CHECKTOOL)
-                if check.get("classification", "light") not in classes:
+                if check.get("classification", "light") not in config.classes:
                     print(f"skipping check tool '{check['executable']}' because of resource classification")
                     continue
                 args = check.get('args', '')
-                memo_key = f"{check['executable']} {args}"
-                error = memo.get(memo_key)
-                if error is None:
-                    error = run_check_tool(check["executable"], args, os_cloud, verbose, quiet)
-                    memo[memo_key] = error
+                memo_key = f"{check['executable']} {args}".strip()
+                invokation = memo.get(memo_key)
+                if invokation is None:
+                    compl = run_check_tool(check["executable"], args, config.os_cloud, config.verbose, config.quiet)
+                    invokation = {
+                        "rc": compl.returncode,
+                        "stdout": compl.stdout.splitlines(),
+                        "stderr": compl.stderr.splitlines(),
+                    }
+                    for signal in ('info', 'warning', 'error', 'critical'):
+                        invokation[signal] = len([
+                            line
+                            for line in chain(compl.stderr.splitlines(), compl.stdout.splitlines())
+                            if line.lower().startswith(signal)
+                        ])
+                    memo[memo_key] = invokation
+                abort = invokation["critical"]
+                error = invokation["error"]
+                if not config.quiet:
+                    print(f"... returned {error} errors")
+                check["aborts"] = abort
                 check["errors"] = error
                 if not condition_optional(check, optional):
+                    aborts += abort
                     errors += error
-                if not quiet:
-                    print(f"... returned {error} errors")
+        vd["aborts"] = errors
         vd["errors"] = errors
-        vd["passed"] = not errors
-        if not quiet:
+        vd["passed"] = not (aborts + errors)
+        if not config.quiet:
             print("*******************************************************")
-            print(f"Verdict for os_cloud {os_cloud}, {specdict['name']}, "
-                  f"version {vd['version']}: {errcode_to_text(errors)}")
+            print(f"Verdict for os_cloud {config.os_cloud}, {specdict['name']}, "
+                  f"version {vd['version']}: {errcode_to_text(aborts + errors)}")
+        allaborts += aborts
+        allerrors += errors
     if not matches:
-        print(f"No valid standard found for {checkdate}", file=sys.stderr)
+        print(f"No valid standard found for {config.checkdate}", file=sys.stderr)
         return 2
-    if output:
-        with open(output, 'w', encoding='UTF-8') as file:
-            output = yaml.safe_dump(report, file, default_flow_style=False, sort_keys=False)
-    allerrors += errors
-    return allerrors
+    if config.output:
+        with open(config.output, 'w', encoding='UTF-8') as file:
+            yaml.safe_dump(report, file, default_flow_style=False, sort_keys=False)
+    return allaborts + (0 if config.critical_only else allerrors)
 
 
 if __name__ == "__main__":