diff --git a/Tests/iaas/image-metadata/image-md-check.py b/Tests/iaas/image-metadata/image-md-check.py index 61adce136..cee2cbe1e 100755 --- a/Tests/iaas/image-metadata/image-md-check.py +++ b/Tests/iaas/image-metadata/image-md-check.py @@ -14,6 +14,7 @@ import os import sys import time +import calendar import getopt import openstack @@ -50,6 +51,22 @@ def usage(ret): rec2_images = ["SLES 15SP4", "RHEL 9", "RHEL 8", "Windows Server 2022", "Windows Server 2019"] sugg_images = ["openSUSE Leap 15.4", "Cirros 0.5.2", "Alpine", "Arch"] +# Just for nice formatting of image naming hints -- otherwise we capitalize the 1st letter +os_list = ("CentOS", "AlmaLinux", "Windows Server", "RHEL", "SLES", "openSUSE") + + +def recommended_name(nm): + "Return capitalized name" + ln = len(nm) + for osnm in os_list: + osln = len(osnm) + if ln >= osln and nm[:osln].casefold() == osnm.casefold(): + rest = "" + if ln > osln: + rest = nm[osln:] + return osnm + rest + return nm[0].upper() + nm[1:] + def get_imagelist(priv): "Retrieve list of public images (optionally also private images)" @@ -130,8 +147,72 @@ def is_url(stg): return False +def is_date(stg, strict = False): + """Return time in Unix seconds or 0 if stg is not a valid date. + We recognize: %Y-%m-%dT%H:%M:%SZ, %Y-%m-%d %H:%M[:%S], and %Y-%m-%d + """ + bdate = 0 + if strict: + fmts = ("%Y-%m-%dT%H:%M:%SZ", ) + else: + fmts = ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d") + for fmt in fmts: + try: + tmdate = time.strptime(stg, fmt) + bdate = calendar.timegm(tmdate) + break + except ValueError as exc: + # print(f'date {stg} does not match {fmt}\n{exc}', file=sys.stderr) + pass + return bdate + + +def freq2secs(stg): + "Convert frequency to seconds (round up a bit), return 0 if not applicable" + if stg == "never" or stg == "critical_bug": + return 0 + if stg == "yearly": + return 365*24*3600 + if stg == "quarterly": + return 92*24*3600 + if stg == "monthly": + return 31*24*3600 + if stg == "weekly": + return 7*25*3600 + if stg == "daily": + return 25*3600 + + +OUTDATED_IMAGES = [] + + +def is_outdated(img, bdate): + "return 1 if img (with build/regdate bdate) is outdated, 2 if it's not hidden or marked" + max_age = 0 + if "replace_frequency" in img.properties: + max_age = 1.1 * (freq2secs(img.properties["replace_frequency"])) + if not max_age or time.time() <= max_age + bdate: + return 0 + # So we found an outdated image that should have been updated + # (5a1) Check whether we are past the provided_until date + until_str = img.properties["provided_until"] + until = is_date(img.properties["provided_until"]) + if not until and not until_str == "none" and not until_str == "notice": + print(f'Error: Image "{img.name}" does not provide a valid provided until date') + return 3 + if time.time() > until: + return 0 + if img.is_hidden or img.name[-3:] == "old" or img.name[-4] == "prev" or img.name[-8:].isdecimal(): + return 1 + if is_date(img.name[-10:]): + return 1 + print(f'Warning: Image "{img.name}" seems outdated (acc. to its repl freq) but is not hidden or otherwise marked') + return 2 + + def validate_imageMD(imgnm): "Retrieve image properties and test for compliance with spec" + global OUTDATED_IMAGES try: img = conn.image.find_image(imgnm) except openstack.exceptions.DuplicateResource as exc: @@ -155,23 +236,24 @@ def validate_imageMD(imgnm): warnings += 1 # (4) image_build_date, image_original_user, image_source (opt image_description) - # (5) maintained_until, provided_until, uuid_validity, update_frequency + # (5) maintained_until, provided_until, uuid_validity, replace_frequency for prop in (*build_props, *maint_props): if not prop.is_ok(img.properties, imgnm): errors += 1 # TODO: Some more sanity checks: # - Dateformat for image_build_date - bdate = time.strptime(img.created_at, "%Y-%m-%dT%H:%M:%SZ") + rdate = is_date(img.created_at, True) + bdate = rdate if "image_build_date" in img.properties: - try: - bdate = time.strptime(img.properties["image_build_date"][:10], "%Y-%m-%d") - # This never evals to True, but makes bdate used for flake8 - if verbose and False: - print(f'Info: Image "{imgnm}" with build date {bdate}') - except Exception: + bdate = is_date(img.properties["image_build_date"]) + if bdate > rdate: + print(f'Error: Image "{imgnm}" with build date {img.properties["image_build_date"]} after registration date {img.created_at}') + errors += 1 + if not bdate: print(f'Error: Image "{imgnm}": no valid image_build_date ' f'{img.properties["image_build_date"]}', file=sys.stderr) errors += 1 + bdate = rdate # - image_source should be a URL if "image_source" in img.properties: if not is_url(img.properties["image_source"]): @@ -182,18 +264,48 @@ def validate_imageMD(imgnm): print(f'Error: Image "{imgnm}": image_source should be a URL or "private"', file=sys.stderr) errors += 1 # - uuid_validity has a distinct set of options (none, last-X, DATE, notice, forever) + if "uuid_validity" in img.properties: + img_uuid_val = img.properties["uuid_validity"] + if img_uuid_val == "none" or img_uuid_val == "notice" or img_uuid_val == "forever": + pass + elif img_uuid_val[:5] == "last-" and img_uuid_val[5:].isdecimal(): + pass + elif is_date(img_uuid_val): + pass + else: + print(f'Error: Image "{imgnm}": invalid uuid_validity {img_uuid_val}') + errors += 1 # - hotfix hours (if set!) should be numeric - # (5a) Sanity: Are we actually in violation of update_frequency? + if "hotfix_hours" in img.properties: + if not img.properties["hotfix_hours"].isdecimal(): + print(f'Error: Image "{imgnm}" has non-numeric hotfix_hours set') + errors += 1 + # (5a) Sanity: Are we actually in violation of replace_frequency? # This is a bit tricky: We need to disregard images that have been rotated out # - os_hidden = True is a safe sign for this # - A name with a date stamp or old or prev (and a newer exists) + outd = is_outdated(img, bdate) + if outd == 3: + errors += 1 + elif outd: + OUTDATED_IMAGES.append(imgnm) + warnings += (outd-1) # (2) sanity min_ram (>=64), min_disk (>= size) + if img.min_ram < 64: + print(f'Warning: Image "{imgnm}": min_ram == {img.min_ram} MB') + warnings += 1 + # errors += 1 + if img.min_disk < img.size/1073741824: + print(f'Warning: Image "{imgnm}" has img size of {img.size/1048576}MiB, but min_disk {img.min_disk*1024}MiB') + warnings += 1 + # errors += 1 # (6) tags os:*, managed_by_* - # + # Nothing to do here ... we could do a warning if those are missing ... + # (7) Recommended naming if imgnm[:len(constr_name)].casefold() != constr_name.casefold(): # and verbose # FIXME: There could be a more clever heuristic for displayed recommended names - rec_name = constr_name[0].upper()+constr_name[1:] + rec_name = recommended_name(constr_name) print(f'Warning: Image "{imgnm}" does not start with recommended name "{rec_name}"', file=sys.stderr) warnings += 1 @@ -217,11 +329,18 @@ def report_stdimage_coverage(imgs): return err +def miss_replacement_images(images, outd_list): + "Go over list of images to find replacement imgs for outd_list, return the ones that are left missing" + return outd_list + + def main(argv): "Main entry point" # Option parsing global verbose, private, skip global cloud, conn + global OUTDATED_IMAGES + err = 0 try: opts, args = getopt.gnu_getopt(argv[1:], "phvc:s", ("private", "help", "os-cloud=", "verbose", "skip-completeness")) @@ -248,14 +367,23 @@ def main(argv): # Do work if not images: images = get_imagelist(private) - err = 0 # Analyse image metadata for image in images: err += validate_imageMD(image) if not skip: err += report_stdimage_coverage(images) + if OUTDATED_IMAGES: + # TODO: Check whether we have replacements for outdated images with the same names + # except maybe stripped last word (which could be old, prev, datestamp) + if verbose: + print(f'Info: The following outdated images have been detected: {OUTDATED_IMAGES}') + rem_list = miss_replacement_images(images, OUTDATED_IMAGES) + if rem_list: + print(f'Error: Outdated images without replacement: {rem_list}') + err += len(rem_list) except BaseException as e: print(f"CRITICAL: {e!r}") + return 127 return err