Add the checks from the TODO list (issue #412)

Signed-off-by: Kurt Garloff <kurt@garloff.de>

Tests/iaas/image-metadata/image-md-check.py

@@ -14,6 +14,7 @@
 import os
 import sys
 import time
+import calendar
 import getopt
 import openstack
 
@@ -50,6 +51,22 @@ def usage(ret):
 rec2_images = ["SLES 15SP4", "RHEL 9", "RHEL 8", "Windows Server 2022", "Windows Server 2019"]
 sugg_images = ["openSUSE Leap 15.4", "Cirros 0.5.2", "Alpine", "Arch"]
 
+# Just for nice formatting of image naming hints -- otherwise we capitalize the 1st letter
+os_list = ("CentOS", "AlmaLinux", "Windows Server", "RHEL", "SLES", "openSUSE")
+
+
+def recommended_name(nm):
+    "Return capitalized name"
+    ln = len(nm)
+    for osnm in os_list:
+        osln = len(osnm)
+        if ln >= osln and nm[:osln].casefold() == osnm.casefold():
+            rest = ""
+            if ln > osln:
+                rest = nm[osln:]
+            return osnm + rest
+    return nm[0].upper() + nm[1:]
+
 
 def get_imagelist(priv):
     "Retrieve list of public images (optionally also private images)"
@@ -130,8 +147,72 @@ def is_url(stg):
     return False
 
 
+def is_date(stg, strict = False):
+    """Return time in Unix seconds or 0 if stg is not a valid date.
+       We recognize: %Y-%m-%dT%H:%M:%SZ, %Y-%m-%d %H:%M[:%S], and %Y-%m-%d
+    """
+    bdate = 0
+    if strict:
+        fmts = ("%Y-%m-%dT%H:%M:%SZ", )
+    else:
+        fmts = ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d") 
+    for fmt in fmts:
+        try:
+            tmdate = time.strptime(stg, fmt)
+            bdate = calendar.timegm(tmdate)
+            break
+        except ValueError as exc:
+            # print(f'date {stg} does not match {fmt}\n{exc}', file=sys.stderr)
+            pass
+    return bdate
+
+
+def freq2secs(stg):
+    "Convert frequency to seconds (round up a bit), return 0 if not applicable"
+    if stg == "never" or stg == "critical_bug":
+        return 0
+    if stg == "yearly":
+        return 365*24*3600
+    if stg == "quarterly":
+        return 92*24*3600
+    if stg == "monthly":
+        return 31*24*3600
+    if stg == "weekly":
+        return 7*25*3600
+    if stg == "daily":
+        return 25*3600
+
+
+OUTDATED_IMAGES = []
+
+
+def is_outdated(img, bdate):
+    "return 1 if img (with build/regdate bdate) is outdated, 2 if it's not hidden or marked"
+    max_age = 0
+    if "replace_frequency" in img.properties:
+        max_age = 1.1 * (freq2secs(img.properties["replace_frequency"]))
+    if not max_age or time.time() <= max_age + bdate:
+        return 0
+    # So we found an outdated image that should have been updated
+    # (5a1) Check whether we are past the provided_until date
+    until_str = img.properties["provided_until"]
+    until = is_date(img.properties["provided_until"])
+    if not until and not until_str == "none" and not until_str == "notice":
+        print(f'Error: Image "{img.name}" does not provide a valid provided until date')
+        return 3
+    if time.time() > until:
+        return 0
+    if img.is_hidden or img.name[-3:] == "old" or img.name[-4] == "prev" or img.name[-8:].isdecimal():
+        return 1
+    if is_date(img.name[-10:]):
+        return 1
+    print(f'Warning: Image "{img.name}" seems outdated (acc. to its repl freq) but is not hidden or otherwise marked')
+    return 2
+
+
 def validate_imageMD(imgnm):
     "Retrieve image properties and test for compliance with spec"
+    global OUTDATED_IMAGES
     try:
         img = conn.image.find_image(imgnm)
     except openstack.exceptions.DuplicateResource as exc:
@@ -155,23 +236,24 @@ def validate_imageMD(imgnm):
         warnings += 1
 
     # (4) image_build_date, image_original_user, image_source (opt image_description)
-    # (5) maintained_until, provided_until, uuid_validity, update_frequency
+    # (5) maintained_until, provided_until, uuid_validity, replace_frequency
     for prop in (*build_props, *maint_props):
         if not prop.is_ok(img.properties, imgnm):
             errors += 1
     # TODO: Some more sanity checks:
     #  - Dateformat for image_build_date
-    bdate = time.strptime(img.created_at, "%Y-%m-%dT%H:%M:%SZ")
+    rdate = is_date(img.created_at, True)
+    bdate = rdate
     if "image_build_date" in img.properties:
-        try:
-            bdate = time.strptime(img.properties["image_build_date"][:10], "%Y-%m-%d")
-            # This never evals to True, but makes bdate used for flake8
-            if verbose and False:
-                print(f'Info: Image "{imgnm}" with build date {bdate}')
-        except Exception:
+        bdate = is_date(img.properties["image_build_date"])
+        if bdate > rdate:
+            print(f'Error: Image "{imgnm}" with build date {img.properties["image_build_date"]} after registration date {img.created_at}')
+            errors += 1
+        if not bdate:
             print(f'Error: Image "{imgnm}": no valid image_build_date '
                   f'{img.properties["image_build_date"]}', file=sys.stderr)
             errors += 1
+            bdate = rdate
     # - image_source should be a URL
     if "image_source" in img.properties:
         if not is_url(img.properties["image_source"]):
@@ -182,18 +264,48 @@ def validate_imageMD(imgnm):
                 print(f'Error: Image "{imgnm}": image_source should be a URL or "private"', file=sys.stderr)
                 errors += 1
     #  - uuid_validity has a distinct set of options (none, last-X, DATE, notice, forever)
+    if "uuid_validity" in img.properties:
+        img_uuid_val = img.properties["uuid_validity"]
+        if img_uuid_val == "none" or img_uuid_val == "notice" or img_uuid_val == "forever":
+            pass
+        elif img_uuid_val[:5] == "last-" and img_uuid_val[5:].isdecimal():
+            pass
+        elif is_date(img_uuid_val):
+            pass
+        else:
+            print(f'Error: Image "{imgnm}": invalid uuid_validity {img_uuid_val}')
+            errors += 1
     #  - hotfix hours (if set!) should be numeric
-    # (5a) Sanity: Are we actually in violation of update_frequency?
+    if "hotfix_hours" in img.properties:
+        if not img.properties["hotfix_hours"].isdecimal():
+            print(f'Error: Image "{imgnm}" has non-numeric hotfix_hours set')
+            errors += 1
+    # (5a) Sanity: Are we actually in violation of replace_frequency?
     #  This is a bit tricky: We need to disregard images that have been rotated out
     #  - os_hidden = True is a safe sign for this
     #  - A name with a date stamp or old or prev (and a newer exists)
+    outd = is_outdated(img, bdate)
+    if outd == 3:
+        errors += 1
+    elif outd:
+        OUTDATED_IMAGES.append(imgnm)
+        warnings += (outd-1)
     # (2) sanity min_ram (>=64), min_disk (>= size)
+    if img.min_ram < 64:
+        print(f'Warning: Image "{imgnm}": min_ram == {img.min_ram} MB')
+        warnings += 1
+        # errors += 1
+    if img.min_disk < img.size/1073741824:
+        print(f'Warning: Image "{imgnm}" has img size of {img.size/1048576}MiB, but min_disk {img.min_disk*1024}MiB')
+        warnings += 1
+        # errors += 1 
     # (6) tags os:*, managed_by_*
-    #
+    # Nothing to do here ... we could do a warning if those are missing ...
+
     # (7) Recommended naming
     if imgnm[:len(constr_name)].casefold() != constr_name.casefold():  # and verbose
         # FIXME: There could be a more clever heuristic for displayed recommended names
-        rec_name = constr_name[0].upper()+constr_name[1:]
+        rec_name = recommended_name(constr_name)
         print(f'Warning: Image "{imgnm}" does not start with recommended name "{rec_name}"',
               file=sys.stderr)
         warnings += 1
@@ -217,11 +329,18 @@ def report_stdimage_coverage(imgs):
     return err
 
 
+def miss_replacement_images(images, outd_list):
+    "Go over list of images to find replacement imgs for outd_list, return the ones that are left missing"
+    return outd_list
+
+
 def main(argv):
     "Main entry point"
     # Option parsing
     global verbose, private, skip
     global cloud, conn
+    global OUTDATED_IMAGES
+    err = 0
     try:
         opts, args = getopt.gnu_getopt(argv[1:], "phvc:s",
                                        ("private", "help", "os-cloud=", "verbose", "skip-completeness"))
@@ -248,14 +367,23 @@ def main(argv):
         # Do work
         if not images:
             images = get_imagelist(private)
-        err = 0
         # Analyse image metadata
         for image in images:
             err += validate_imageMD(image)
         if not skip:
             err += report_stdimage_coverage(images)
+        if OUTDATED_IMAGES:
+            # TODO: Check whether we have replacements for outdated images with the same names
+            # except maybe stripped last word (which could be old, prev, datestamp)
+            if verbose:
+                print(f'Info: The following outdated images have been detected: {OUTDATED_IMAGES}')
+            rem_list = miss_replacement_images(images, OUTDATED_IMAGES)
+            if rem_list:
+                print(f'Error: Outdated images without replacement: {rem_list}')
+                err += len(rem_list)
     except BaseException as e:
         print(f"CRITICAL: {e!r}")
+        return 127
     return err