From 4fd661cd8dfb52e38c5485be793b375f53e0f864 Mon Sep 17 00:00:00 2001 From: tonifinger <129007376+tonifinger@users.noreply.github.com> Date: Thu, 16 Nov 2023 07:48:46 +0100 Subject: [PATCH] Add tests for default storage class properties (#360) * Inital test script for standard "scs-0211-v1-kaas-default-storage-class.md" The script added in this commit holds the basic functionalities and tests used for checking this standard. Signed-off-by: Toni Finger * Apply k8s-default-storage-class as sonobuoy plugin Signed-off-by: Toni Finger * Update Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py Co-authored-by: anjastrunk <119566837+anjastrunk@users.noreply.github.com> Signed-off-by: Kurt Garloff * Move supporting functions to helper.py Update Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py Update Tests/kaas/k8s-default-storage-class/helper.py Signed-off-by: Toni Finger --------- Signed-off-by: Toni Finger Signed-off-by: Kurt Garloff Co-authored-by: Kurt Garloff Co-authored-by: anjastrunk <119566837+anjastrunk@users.noreply.github.com> --- .gitignore | 2 + .../Dockerfile_sonobuoy_plugin | 21 ++ .../kaas/k8s-default-storage-class/helper.py | 74 +++++ .../k8s-default-storage-class-check.py | 268 ++++++++++++++++++ .../k8s-default-storage-class-plugin.yaml | 13 + .../k8s-default-storage-class/run_checks.sh | 55 ++++ 6 files changed, 433 insertions(+) create mode 100644 Tests/kaas/k8s-default-storage-class/Dockerfile_sonobuoy_plugin create mode 100644 Tests/kaas/k8s-default-storage-class/helper.py create mode 100644 Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py create mode 100644 Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-plugin.yaml create mode 100755 Tests/kaas/k8s-default-storage-class/run_checks.sh diff --git a/.gitignore b/.gitignore index ad0a4485d..320bc8f1c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .idea .DS_Store node_modules +Tests/kaas/results/ +*.tar.gz diff --git a/Tests/kaas/k8s-default-storage-class/Dockerfile_sonobuoy_plugin b/Tests/kaas/k8s-default-storage-class/Dockerfile_sonobuoy_plugin new file mode 100644 index 000000000..bf15cfad0 --- /dev/null +++ b/Tests/kaas/k8s-default-storage-class/Dockerfile_sonobuoy_plugin @@ -0,0 +1,21 @@ +FROM ubuntu + +# Install kubectl +# Note: Latest version may be found on: +# https://aur.archlinux.org/packages/kubectl-bin/ +ADD https://storage.googleapis.com/kubernetes-release/release/v1.14.1/bin/linux/amd64/kubectl /usr/local/bin/kubectl + +ENV HOME=/config + +# Basic check it works. +RUN apt-get update && \ + apt-get -y install net-tools && \ + apt-get -y install curl && \ + chmod +x /usr/local/bin/kubectl && \ + kubectl version --client + + +COPY ./ ./ +RUN chmod +x ./run_checks.sh + +ENTRYPOINT ["./run_checks.sh"] diff --git a/Tests/kaas/k8s-default-storage-class/helper.py b/Tests/kaas/k8s-default-storage-class/helper.py new file mode 100644 index 000000000..dd5089298 --- /dev/null +++ b/Tests/kaas/k8s-default-storage-class/helper.py @@ -0,0 +1,74 @@ +import yaml +import sys +import logging +from kubernetes import client, config + +manual_result_file_template = { + "name": None, + "status": None, + "details": {"messages": None}, +} + +logger = logging.getLogger("helper") + + +def initialize_logging(): + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) + + +def print_usage(file=sys.stderr): + """Help output""" + print( + """Usage: k8s_storageclass_check.py [options] +This tool checks the requested k8s default storage class according to the SCS Standard 0211 "kaas-default-storage-class". +Options: + [-k/--kubeconfig PATH_TO_KUBECONFIG] sets kubeconfig file to access kubernetes api + [-d/--debug] enables DEBUG logging channel +""", + end="", + file=file, + ) + + +class SCSTestException(Exception): + """Raised when an Specific test did not pass""" + + def __init__(self, *args, return_code: int): + self.return_code = return_code + + +def setup_k8s_client(kubeconfigfile=None): + + if kubeconfigfile: + logger.debug(f"using kubeconfig file '{kubeconfigfile}'") + config.load_kube_config(kubeconfigfile) + else: + logger.debug(" using system kubeconfig") + config.load_kube_config() + + k8s_api_client = client.CoreV1Api() + k8s_storage_client = client.StorageV1Api() + + return ( + k8s_api_client, + k8s_storage_client, + ) + + +def gen_sonobuoy_result_file(error_n: int, error_msg: str, test_file_name: str): + + test_name = test_file_name.replace(".py", "") + + test_status = "passed" + + if error_n != 0: + test_status = test_name + "_" + str(error_n) + + result_file = manual_result_file_template + + result_file["name"] = test_name + result_file["status"] = test_status + result_file["details"]["messages"] = error_msg + + with open(f"./{test_name}.result.yaml", "w") as file: + yaml.dump(result_file, file) diff --git a/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py b/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py new file mode 100644 index 000000000..f8bd6c258 --- /dev/null +++ b/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-check.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 + +"""PersistentVolumeClaims checker + +Return codes: +0: Default StorageClass is available, and setup to SCS standard +1: Not able to connect to k8s api + +31: Default storage class has no provisioner +32: None or more then one default Storage Class is defined + +41: Not able to bind PersitantVolume to PersitantVolumeClaim +42: ReadWriteOnce is not a supported access mode + +All return codes between (and including) 1-19 as well as all return codes ending on 9 +can be seen as failures. + +Check given cloud for conformance with SCS standard regarding +Default StorageClass and PersistentVolumeClaims, to be found under /Standards/scs-0211-v1-kaas-default-storage-class.md + +""" + +import getopt +import sys +import time +import json +import logging + +from kubernetes import client +from helper import gen_sonobuoy_result_file +from helper import SCSTestException +from helper import initialize_logging +from helper import print_usage +from helper import setup_k8s_client + +import logging.config + +logger = logging.getLogger("k8s-default-storage-class-check") + + +def check_default_storageclass(k8s_client_storage): + + api_response = k8s_client_storage.list_storage_class(_preload_content=False) + storageclasses = api_response.read().decode("utf-8") + storageclasses_dict = json.loads(storageclasses) + + ndefault_class = 0 + + for item in storageclasses_dict["items"]: + storage_class_name = item["metadata"]["name"] + annotations = item["metadata"]["annotations"] + + if annotations["storageclass.kubernetes.io/is-default-class"] == "true": + ndefault_class += 1 + default_storage_class = storage_class_name + provisioner = item["provisioner"] + + if provisioner == "kubernetes.io/no-provisioner": + raise SCSTestException( + f"Provisioner is set to: {provisioner}.", + "This means the default storage class has no provisioner.", + return_code=31, + ) + + if ndefault_class != 1: + raise SCSTestException( + "More then one or none default StorageClass is defined! ", + f"Number of defined default StorageClasses = {ndefault_class} ", + return_code=32, + ) + + logger.info(f"One default Storage Class found:'{default_storage_class}'") + return default_storage_class + + +def check_default_persistentvolumeclaim_readwriteonce(k8s_api_instance, storage_class): + """ + 1. Create PersistantVolumeClaim + 2. Create pod which uses the PersitantVolumeClaim + 3. Check if PV got succesfully created using ReadWriteOnce + 4. Delete resources used for testing + """ + + namespace = "default" + pvc_name = "test-pvc" + pv_name = "test-pv" + pod_name = "test-pod" + + # 1. Create PersistantVolumeClaim + logger.debug(f"create pvc: {pvc_name}") + + pvc_meta = client.V1ObjectMeta(name=pvc_name) + pvc_resources = client.V1ResourceRequirements( + requests={"storage": "1Gi"}, + ) + pvc_spec = client.V1PersistentVolumeClaimSpec( + access_modes=["ReadWriteOnce"], + storage_class_name=storage_class, + resources=pvc_resources, + ) + body_pvc = client.V1PersistentVolumeClaim( + api_version="v1", kind="PersistentVolumeClaim", metadata=pvc_meta, spec=pvc_spec + ) + + api_response = k8s_api_instance.create_namespaced_persistent_volume_claim( + namespace, body_pvc + ) + + # 2. Create a pod which makes use of the PersitantVolumeClaim + logger.debug(f"create pod: {pod_name}") + + pod_vol = client.V1Volume( + name=pv_name, + persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(pvc_name), + ) + pod_con = client.V1Container( + name="nginx", + image="nginx", + ports=[client.V1ContainerPort(container_port=80)], + volume_mounts=[ + client.V1VolumeMount(name=pv_name, mount_path="/usr/share/nginx/html") + ], + ) + pod_spec = client.V1PodSpec(volumes=[pod_vol], containers=[pod_con]) + pod_body = client.V1Pod( + api_version="v1", + kind="Pod", + metadata=client.V1ObjectMeta(name=pod_name), + spec=pod_spec, + ) + + api_response = k8s_api_instance.create_namespaced_pod( + namespace, pod_body, _preload_content=False + ) + pod_info = json.loads(api_response.read().decode("utf-8")) + pod_status = pod_info["status"]["phase"] + + # Check if pod is up and running: + retries = 0 + while pod_status != "Running" and retries <= 30: + + api_response = k8s_api_instance.read_namespaced_pod( + pod_name, namespace, _preload_content=False + ) + pod_info = json.loads(api_response.read().decode("utf-8")) + pod_status = pod_info["status"]["phase"] + logger.debug(f"retries:{retries} status:{pod_status}") + time.sleep(1) + retries += 1 + + # assert pod_status == "Running" + if pod_status != "Running": + raise SCSTestException( + "pod is not Running not able to setup test Enviornment", + return_code=13, + ) + + # 3. Check if PV got succesfully created using ReadWriteOnce + logger.debug("check if the created PV supports ReadWriteOnce") + + api_response = k8s_api_instance.list_persistent_volume(_preload_content=False) + + pv_info = json.loads(api_response.read().decode("utf-8")) + pv_list = pv_info["items"] + + logger.debug("searching for corresponding pv") + for pv in pv_list: + logger.debug(f"parsing pv: {pv['metadata']['name']}") + if pv["spec"]["claimRef"]["name"] == pvc_name: + logger.debug(f"found pv to pvc: {pvc_name}") + + if pv["status"]["phase"] != "Bound": + raise SCSTestException( + "Not able to bind pv to pvc", + return_code=41, + ) + + if "ReadWriteOnce" not in pv["spec"]["accessModes"]: + raise SCSTestException( + "access mode 'ReadWriteOnce' is not supported", + return_code=42, + ) + + # 4. Delete resources used for testing + logger.debug(f"delete pod:{pod_name}") + api_response = k8s_api_instance.delete_namespaced_pod(pod_name, namespace) + logger.debug(f"delete pvc:{pvc_name}") + api_response = k8s_api_instance.delete_namespaced_persistent_volume_claim( + pvc_name, namespace + ) + + return 0 + + +def main(argv): + + initialize_logging() + return_code = 0 + return_message = "return_message: FAILED" + + try: + opts, args = getopt.gnu_getopt(argv, "k:h", ["kubeconfig=", "help"]) + except getopt.GetoptError as exc: + logger.debug(f"{exc}", file=sys.stderr) + print_usage() + return 1 + + kubeconfig = None + + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + print_usage() + return 0 + if opt[0] == "-k" or opt[0] == "--kubeconfig": + kubeconfig = opt[1] + else: + print_usage(kubeconfig) + return 2 + + # Setup kubernetes client + try: + logger.debug("setup_k8s_client(kubeconfig)") + k8s_core_api, k8s_storage_api = setup_k8s_client(kubeconfig) + except Exception as exception_message: + logger.info(f"{exception_message}") + return_message = f"{exception_message}" + return_code = 1 + + # Check if default storage class is defined (MENTETORY) + try: + logger.info("check_default_storageclass()") + default_class_name = check_default_storageclass(k8s_storage_api) + except SCSTestException as test_exception: + logger.info(f"{test_exception}") + return_message = f"{test_exception}" + return_code = test_exception.return_code + except Exception as exception_message: + logger.info(f"{exception_message}") + return_message = f"{exception_message}" + return_code = 1 + + # Check if default_persistent volume has ReadWriteOnce defined (MENTETORY) + try: + logger.info("check_default_persistentvolume_readwriteonce()") + return_code = check_default_persistentvolumeclaim_readwriteonce( + k8s_core_api, default_class_name + ) + except SCSTestException as test_exception: + logger.info(f"{test_exception}") + return_message = f"{test_exception}" + return_code = test_exception.return_code + except Exception as exception_message: + logger.info(f"{exception_message}") + return_message = f"{exception_message}" + return_code = 1 + + logger.debug(f"return_code:{return_code}") + + if return_code == 0: + return_message = "all tests passed" + + gen_sonobuoy_result_file(return_code, return_message, __file__) + + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-plugin.yaml b/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-plugin.yaml new file mode 100644 index 000000000..3a73f6159 --- /dev/null +++ b/Tests/kaas/k8s-default-storage-class/k8s-default-storage-class-plugin.yaml @@ -0,0 +1,13 @@ +sonobuoy-config: + driver: Job + plugin-name: k8s-default-storage-class + result-format: manual + resutl-file: k8s-default-storage-class-check.result.yaml + +spec: + args: + - k8s-default-storage-class-check + command: + - ./run_checks.sh + image: ghcr.io/sovereigncloudstack/standards/k8s-default-storage-class:latest + name: k8s-default-storage-class diff --git a/Tests/kaas/k8s-default-storage-class/run_checks.sh b/Tests/kaas/k8s-default-storage-class/run_checks.sh new file mode 100755 index 000000000..6205a8594 --- /dev/null +++ b/Tests/kaas/k8s-default-storage-class/run_checks.sh @@ -0,0 +1,55 @@ +#!/bin/sh + +############################################################################### +##### HELPERS ##### +############################################################################### + +set -x + +# This is the entrypoint for the image and meant to wrap the +# logic of gathering/reporting results to the Sonobuoy worker. + +results_dir="${RESULTS_DIR:-/tmp/results}" +mkdir -p ${results_dir} + +# saveResults prepares the results for handoff to the Sonobuoy worker. +# See: https://github.com/vmware-tanzu/sonobuoy/blob/main/site/content/docs/main/plugins.md +saveResults() { + cd ${results_dir} + echo ${results_dir} + + # Sonobuoy worker expects a tar file. + tar czf results.tar.gz * + + # Signal to the worker that we are done and where to find the results. + printf ${results_dir}/results.tar.gz > ${results_dir}/done +} + +# Ensure that we tell the Sonobuoy worker we are done regardless of results. +trap saveResults EXIT + + +############################################################################### +##### RUN TEST SCRIPTS ##### +############################################################################### + +# Each script name is expected to be given as an arg. If no args, error out +# but print one result file for clarity in the results. +if [ "$#" -eq "0" ]; then + echo "No arguments; expects each argument to be script name" > ${results_dir}/out + exit 1 +fi + +# Iterate through the python tests passed as arguments +i=0 +while [ "$1" != "" ]; do + # Run each arg as a command and save the output in the results directory. + echo "run testscript: [$1.py]" + + python $1.py > ${results_dir}/out_$1 + cp $1.result.yaml ${results_dir} + i=$((i + 1)) + + # Shift all the parameters down by one + shift +done