Run compliance checks on Zuul (resolves #316) (#362)

* Initial attempt at getting the compliance check to run via zuul * restructured to have secrets in a separate file * Simplify structure, pacify ansible-lint * Make sure pip is available * Allow restricting check tools according to resource classification * Master check tool doesn't fiddle with the environment * Reverse order in scs-compatible-iaas.yaml, add new, still-unstable version (will be skipped anyway) * Check all valid and upcoming versions if no version is forced * result.yaml more detailed, option to only return critical errors (which prevent the tests to complete) * Adjust compliance check cmd; will need to add artifact upload? * Notify zuul of artifact * Add task to copy artifact back * Added more cloud environments * Move compliance checks to daily, except for gx-scs for further debugging * Improved report format * Make number of ERROR lines match return code * Add Regiocloud RegionA * Fix missing quote marks, thanks @chess-knight for spotting * Comply with secret naming convention, use document start marker; thanks @berendt * Fix changed_when, thanks @berendt * Address ansible-lint recommendations Thanks @chess-knight and @berendt for your valuable insights! Signed-off-by: Matthias Büchse <matthias.buechse@cloudandheat.com>
SovereignCloudStack · Nov 30, 2023 · 8dc1b9b · 8dc1b9b
1 parent 4c26981
commit 8dc1b9b
Show file tree

Hide file tree

Showing 8 changed files with 597 additions and 235 deletions.
diff --git a/.zuul.d/config.yaml b/.zuul.d/config.yaml
@@ -0,0 +1,52 @@
+---
+- project:
+    name: SovereignCloudStack/standards
+    default-branch: main
+    merge-mode: "squash-merge"
+    periodic-daily:
+      jobs:
+        - scs-check-pco-prod1
+        - scs-check-pco-prod2
+        - scs-check-pco-prod3
+        - scs-check-regio-a
+        - scs-check-wavestack
+    periodic-hourly:
+      jobs:
+        - scs-check-gx-scs
+    post:
+      jobs:
+        - scs-check-gx-scs
+- job:
+    name: scs-check-gx-scs
+    parent: base
+    secrets:
+     - name: clouds_conf
+       secret: SECRET_STANDARDS
+    vars:
+      cloud: gx-scs
+    run: playbooks/compliance_check.yaml
+- job:
+    name: scs-check-pco-prod3
+    parent: scs-check-gx-scs
+    vars:
+      cloud: pco-prod3
+- job:
+    name: scs-check-pco-prod2
+    parent: scs-check-gx-scs
+    vars:
+      cloud: pco-prod2
+- job:
+    name: scs-check-pco-prod1
+    parent: scs-check-gx-scs
+    vars:
+      cloud: pco-prod1
+- job:
+    name: scs-check-regio-a
+    parent: scs-check-gx-scs
+    vars:
+      cloud: regio-a
+- job:
+    name: scs-check-wavestack
+    parent: scs-check-gx-scs
+    vars:
+      cloud: wavestack
diff --git a/.zuul.d/secure.yaml b/.zuul.d/secure.yaml
@@ -0,0 +1,136 @@
+---
+- secret:
+    name: SECRET_STANDARDS
+    data:
+      gx_scs_ac_id: !encrypted/pkcs1-oaep
+        - b/SnLk91ZRXkFj22EjOZk+vqqJIDySsGL9WzaR8nZntJOYfhkqVbp5AV9KDuqP/bj9MrR
+          yDc2hdkBesm1d62ynjQ94CjP8p7Lhs45FFyOcxGWQwMairD4YtnFsvKfvYtp3mz13n0gF
+          HOFGycm0CZwO1cETUJFB2O9ekbI9T5iO6PmQiwWbWbT/5EJu8bjAUaLV7ZyGsCZ22FQ8E
+          4dbs+2ShRsckitp7iBDWAzsPzX/aq8xuzoZ9Zf0DlHXuQrqENkx721QygsNLxb4dVC8e1
+          vT8R6Oy0MBGfn667Ob1yfquileryCu+eXmFPzKZwxn2IWpl3IdSYEs0ZSFkd9ZEuA4UIE
+          bolgg0hXSCzxoI9M+b0+FTvNmYQw7u4tFJ6YPLhs9QJHhJVj7oywwrZRumrD0XgPzHQgW
+          RhrL6OS8ChvZ5yjZdRK9L+pDM2MW4kKtKzmXZ5RpoMzNoh8Mkns5YlPrikrkQsYaiq0bx
+          540qoAZl+zL0SiV2Z7t8VdGwEroBDpEIrPfIboBPo9eyRbGUKRk9DqiJ+wqMhkEQ0Lu+T
+          whlE9WShj0BNs4mghjUg5WY9EMmH4IVFTxvbcr2UhfcnGxJJ0I+MfaAP4G8UZ0rRohYgi
+          i8YmbOQ7NCpyRqz0IniDODo2Cz76/NJ5e53nbTuGNBuoFoigwyGqMvI4vkN2Oc=
+      gx_scs_ac_secret: !encrypted/pkcs1-oaep
+        - ZJiujVLXDgWrPSOU6759BE23gzBGtT33c4ziUDNNFpZzW1Q8PwrrtMCBHbbAey1b8qF88
+          Yp8wk4EMLMNxB9SD+E1wVBAtHRubzFwOj4/4R1kza430BydqHr+8RE3kTVukj8Wv1B2ZC
+          SI1gaKJHhvLIVdmiDncI6MKf4AXC0pERIhg618xYNxmGjQT5vxSSZjvwflfJiFutXa0wa
+          5jFNo29T3TPQH7mA5SMzM2a0AENt/Ccp6/t+UlNLG3v3M1g/P+xednVOI1tdy+CappvHY
+          10u8Eln1L9ZfKKT6ebLLdoYws++numKGgzGkfyS0Ob7gOGaHC3EYyha5y//BqTkGsVmfE
+          8ed+zV9fO2BpaUB1h1rlO76SgXlygw/4r+JmNIbqG4b/KEseQeGL4NwR1wOJfG1LEbdsF
+          xE+zhSf3a3pzjt1q4qL+F4VEzIIJWS7Vn3BZaaKLovRz0iGc8PRnrQyl6arCgtInzbemu
+          mLv1l8bxsHvMEdb3l6wqqNvVPjo8pgZTH+1MK/2H2Zpg62NI6b+AGkqOPYIr6U3xllT23
+          ZLPVpoq1cuvY9Eh3i5OjE7AU7hGcy7cDy7lHSBNYPKv+ZERIneEciApgq6QYEEuf7v7rl
+          wxiW8mQgmQSsly/+7LrcBpwyv0ouHrg1VOx84dhYHEV++I0FDJGJsa/HcZ/8ZQ=
+      pco_prod1_ac_id: !encrypted/pkcs1-oaep
+        - jGScb1B/BfnuDdDnfsJoHnVRaeiTAX1fCB3eYBuUx6grQTQ2SorKWeUeVWqznfJJF0Pug
+          uE09n6oCwZE3hxzI2VxFA+o4wDBA3azasAs8N3vV+QyFYF5dl+5K1M0xwdkhqAyefw5n8
+          EhJvrqmzFk81TA4lDIOprK++16on4BsLnc1eUErYbeUv0AuKbBcq4RYU0AFLG47DA8BTH
+          9kMc+RkgAhB4+EtwTrGTITOrEVB/gfEnmbKanuopCFgmcdoa4GliHilHZBc24Ht64TfdN
+          HHuv8598r43SQX9B1l9ZzThWSs4VlfJEet4hAaSO9uJj2CSbRvC81Ep3UQBJBkSlE3r6g
+          XI+FIY5x/vtoPj7SFm89gyuwZ50rO3Gp8etJhqddyQjJUrGtZHNpS0LJ514EZ+ckDYuYG
+          iUd96mxVPVHukSSIfkFPhDrZLG0PhW94EuJjRi/qi9NgqHVKs0JtjwA/YI7tCVa5rq6cs
+          VfoZ0K9lG3r7Fh6Mzob+mZKMfaLMh73e0ZR8LwvBn/vFHmVTzaApaLJWnOepAWBLOWJyB
+          Hv5+vWbmohdwxaK6KuaiSENR9ssqerWjGFTeyG0dUSkW3f0N6nS2T9emIkelNLtANAwvw
+          7CA6nSrj8k1DNG+nH4RfeN5r+k8cCLCvVj4p/bIJuKSe5dLW6/+ooGyv+dfUDU=
+      pco_prod1_ac_secret: !encrypted/pkcs1-oaep
+        - htWKeIuB3Lracz52Zprq+WUe1y7OLEfR8GmrDKUYbUy5CN9gp+kxx4cmSjm/31Xz3wa9I
+          tssK7NJR5rAIgpzyXwCJaScxek/5d9Zfa/DFw6KJsnATq+VNEtQS/wDnptb2YJ5P5ff7A
+          zfBnAMIQf0tpRwzcupZPng3zLkYsqfS0ZnSo5RJBL1lH0gwHKbhq7XD4Vn6uCxClPelTT
+          iL90bk3zJUWB8qmtjVfcmWqepiaBb45Ve7kU5U6/iqoDo6itDRCaHMMmP7NI61Wlpxp3A
+          La6/X8a3/wMH4KLoWFaDxbirZEsMRHmTmHyYnQtko4WyYYz4s9q/vUEbCTecIiXBRBp8a
+          8GkoHO1XkrZuBsd2yFsA8fc2/d0D2gr9iMDdipsPmJbwFEtHM8KH5DmtIiXy2HRl2rO9f
+          HMCyUdEtH7UT7zPQDuY+jk4ExwL3SCuihVimLFQyWTqAM6r8L3MQAGNVWikxVlWokeQaX
+          VG+WGHBasBsBieTos/OujXFVqMDgiiyB5G6crr17AKMRldxAcgvPSTG279hdstNCNClQ9
+          xE2Yf0v8vXJzM/AWOMJgFanBZ35QDPIYMH6+aMh+BXj/Q0zYpWZ+y1vg358y8OYX4Q+NM
+          3AGCgXvUVG3wblRSlUWUac7enMNYxlvHXTeKNLYXwut+A1qPuYFWwE+EvjReEM=
+      pco_prod2_ac_id: !encrypted/pkcs1-oaep
+        - cjuaL7RFh1f4LPVP5nKv4eEOOIPKFVsK8vXtv390lX/MRDLxJGA/lb8xxRo77tKFOPlUA
+          65jEHYyeDMDn1RP+8Fqq6lEQGo9X3986zNj9+NlaINv/pRHfcNCMltEoLVsOTuBeg+DJC
+          XvcoAz+D2A9FFTIOaShE6iElUOmHnroSO1v6TrmOVcBdNmRYtuhL8ao7gHgHTnKdkZWPm
+          QjZfcME+rjftezOdRQWN6W7sLKP44miKXoglTpqxPFpko09MctP0F3YVNecoJzBtlOkMJ
+          dZlNhkaJeof2RCLN3xKRlwwGwhZA/YlIphE4YdzI2LXXpfHi+kD0DOx3jEixSdMJRdNdn
+          iHzdMyezpXzfZSGPhFVeCMb/lpVHsVGKn8sRfaH65WQX80t/vDP88k8biRbdg8eYbVHJw
+          Wtj4xjJbJyGG/Cc1VjBVUxP3YiVRHUGbcjOMytnDr94giMjz05fdOK//N0mmy3qgrRJIX
+          SYX40oU1gEMv/b1PyX3dsf7a7p4x8LVyvzUG+Kkr4+duVsqqJFJw9ejXSjhyf64oy2uhc
+          +GsHE6bmIraY+YTVmAI1l4T0g/bxL4ssuyZfyskJEnQrF9JEj1SBbW7JDZrqwYC6BqnSm
+          BDPhlbgxEfsNhxHk526f/rgG9IQIw4f0ntB+JTbQnAorNoN16IJHV3H7xVFTbc=
+      pco_prod2_ac_secret: !encrypted/pkcs1-oaep
+        - f5zoAfFxqVILvjPZWpOgAtpPFPexwPQg6wZMBAArsSY9fCaRSgh4/KyHhtojVf2kgXI2l
+          3We0O3HrwHrqItEWXsxGxYt5URRmQQIYbrfSQC5qMz5UdVPKZTK7jnk/ydsLlweeOZdCY
+          kEVrnmBUqJrvpYRMC2D1CCsXvoBaMLdEU1skPOesa1MlWT2T6bvl8tX+YW6lqEd823idX
+          m41SwDL/k7eRwqjdcUBs5T9C93j04vqgFB5vLCe7hoJfvdW0B1oHFeWGyz1NpDXgZGM2P
+          LToMIMCQ2KsRpmk8/gam1GocIce8gbUtOurppvBZFTH0ySnCgzqjf0MHVuxVxfZWjbmzl
+          5cqAcEslommZfDMwv6nASvPSH8EJL6wVuCj9QHqcgbILWO/z8VRB7g/yej1ZOjjBdw7Em
+          TxTcc1Kx2GtQtXLl3GDC6ib8gvJSsEys6nxVTlr1mr3+lq1g8rNl63PXlsk255ZI2Xm+x
+          FokGXgiJvYxZ+FroMlIVzJKrl3KN5luPM8FAj2CGwjoX2j6V5RlAl99lkKce8KSp76HVM
+          ItQVFGP94wfzMxeV1MFN5lYYJvOyApDgFLh9B35/Y2n2XcBu8KytvUR0zPojH3Q8YsGXR
+          y8MeCclG92oucn4huTRMTFdfuUuuAVRXapUMOP7M8HP8tqeYQkQjK1ipH2RyFE=
+      pco_prod3_ac_id: !encrypted/pkcs1-oaep
+        - Sxo7/4v13s46G/j45jkBrvI7puOww3VeKwc2wIaTGw7QIABH/wpcECcUqklXePd6iOWIU
+          apTY8h8uRd+3HyKn5z9b60L0QuSrkE7pawwN544CpFNMq2uImvfv4b/VeLk1VrJn/zO5j
+          keaIf/bTtg8eAPhakcnZLhDMbjprKsM2P6UpqQf473/n3jeNNoPBGf8lzkAqa2BTsQVRL
+          FYN1RIVviKUuEkHTY4U4ziw5i2SzHJq061qs+fYK3G4c9UWDEwKxCQilYHU3/6DgzQwzz
+          4n+nea2qUGuCc8B4q5qUbnMXVy1d9+pJZwSk3I3GkrROcsJ30C+MH1bsyHIxzZffVfjvT
+          NtjzX7KvnRRGhdL13HwODZOt6BYx5irN5TsEs4O4RgxaTv2tSQca3SLWW8uR6JsPXuOhl
+          lKK/dZPzVHWVXp+0bgRlhl/0E85eN2ivt+eIgje/S77tPSxAsxdri0gL/b2L5cT8spDqk
+          ZhV4VaoxEcgUhh605I09SToM+/V0fdWjIb1IySikj+jyOKTKsi11KUEA8yQKAEvnnI/Xo
+          vCu7SIuqy8yhPXqe7o+fzmbEsMA2ToQ779CBcxgwfHZk6iC/IFGFfXhHjgj0RzDlk+0yy
+          hiPzEAtU32Df/gzmo/hvVFiD2FrrI2RCou4kZjq+6xD42q1iupn8Wuzjgdne1M=
+      pco_prod3_ac_secret: !encrypted/pkcs1-oaep
+        - iYft4Sbh/hGulIyIur/cn0fJ9eAl97ah0d70/8NZecQMLoJBS9Lcfc9r4l0p/JItieGxE
+          tPd1qk4Zfx2a1kwIU6l49s+iveNAxxxfRrIckClk8TwwesjXhRjcjecTfDGZsUJ/ZeXa8
+          jnA/xe1Aj0oscME9oNpgu1z1ad02uzxuE2YgcONcx2P5boKoWGNovtBzGgBrUleaNQBsa
+          TLCgooK2es19l1Xh3EEtnI56Ps3vOKOuzbCVF1LzIedeAK7o7XnbTokMz3jydOPj9ebK4
+          8BhSJBJT9PIIVSyf0F0yDoFoWXQu/cItlRWjKnB19ZkyxKMAf+dHUwOqB9JKud7pXPL6x
+          TuslXjxiB3EXF29Ft+4q43HKV3CGKYi4xu09lwzlxx3QdRknY4GhSy2dB9LBoBhkIikO3
+          NACmZxg5z5eFZP+HQSky+uNRrnLKRBvHnc6i0QUYtVuxbprQlUgssvDTByQxQ9a8CVp+Q
+          riPv16TIXHG/VYteDxT4f/onB6xBBiZ7Bm7drM3nAsM+ZM3WwxUnu4luWZsZYAADV+S31
+          ODoxs6vEGmQgoOCCej1a71jkLMx+xdOeRN606H6Jrqfm2BfsYa1ZxUhX1Dk1dgpcVXZOK
+          gJLXG3zz2PJNa/Zl0/3aqrWJS6+A9lD5XuDHDdPxKfyhwo+R2+zhzScd/bbn4E=
+      regio_a_ac_id: !encrypted/pkcs1-oaep
+        - lBlWjvJ6RA3uniS1M4etvbdUxKB9KRNWm53gL3VlPyRkA7Ic2yFcAkZEGodHWH1iqNWfN
+          p/3B2iFYwuZRktllbc/Ro80pkg52vHOkNkBdXLQd7ZFKG8zNJOxRt2nhDQxQPS7PzcUgo
+          dLklwSAbAPJohxA1atqH5qw7pfk/EKsCAW8mu3eWbZWrYAcN9faNiDkm58XOKow4t5x9w
+          c+rG0zmKB4nsqZh+6Bt1iesSzJcUBZIX6eFYQRhj6ev70kCHU5NNYO4y2elNxPOLwSTrx
+          O2b9lYff/iY95ZiKHgyninMXOuoyQHntN7nKtReoBOuwf/1fFxrxEah5BAH5KDIpZ8NoX
+          5iArRHDoaL6L5oLLGs4Z6mojeypQfIHZmcaxkBvK6ZHel+cf4gCC05k1uAVS9005Qb/LH
+          pWJQoTF5ebCVSwU9JcBptaCss60NRkJP1bVLUuyvRTSbkbcK57BsXYnpXtCxsH3qR56QQ
+          nlxnrKT0IawjI0qra2QwUHjVJwmwY/U+Im00p2B2mXwJ2HCQoXD1pCcgBECeHkN3ENQeZ
+          SnE9w4NYo8UR1xjfMGb8xcCg5Lfj/19yxFtPTi/O6MthNmz8GYQMqcZ3x2knoEkt96SY8
+          9IjJxckOkduqBIQrL72CKk7Mk9+2Bb/9STPaMGkIFTGzVCIqtWF0uXs6EEUzWk=
+      regio_a_ac_secret: !encrypted/pkcs1-oaep
+        - r0hxDzPwGm9/Lvvp0KhXJt4sr+gISnEji91RLBt1bx+KRlsOZdZS50hO5Yp0bkWP1sWGx
+          ZphF6STyZmboeQn3+aeeLqhli0KfgAp9RLkQugaSIKNUm4ZF1MbBgBZ+kUrVrlI3JIFAG
+          su3mwmQnAeo6LvK6KdKbQsZm1/cLp9xQ0R6D24h5Y/OXAnfR7xV/Jb76/ZqSlgh65aqXE
+          xpYc+evJ7VcQZnqWPsNf/hC4y8tdjhdBR4T8lIi8+kBF8x7P8uFG7NKjnBFuXhXqauiih
+          HiFVt9oqs5diK8Ujvo+pL9K3kzwuh9v+1k3JyWw4KHeWyH6GcXvDnR0u/DpEnT8t7yACw
+          DRpfO/J5HLKPNKpRFD2oRx9gMQraslcyPXoA/pZzLsqbpsNzwy350cURl3fonUdA1hyyc
+          rnOfDsBXiS7omq79zM0/g8Fyarc7Nx0V+C39kn140LlbYfOpnjciJjmKAA5w5D0kP3SCf
+          VCsXjf0qBBMrzz6HP9z95Bk44fiJ3L/LkA3Iij961dYrQXbZKDrKOiX/QPwrcSrVmjmew
+          UbPexJFHgvTCqjadoLejSt9cUd9lVzhuzLJ8CS+CcCMbZOno6qathrd2B88riQaPNIGNu
+          gfkNT9R63ZzKB1qIA2n5RZi7SH9DPIUd0AwLMn2bhp3uok5pNAPP/4/1RkQiCA=
+      wavestack_ac_id: !encrypted/pkcs1-oaep
+        - NgtWt9AeOFCvfDaDtYdWAFO1oh+LVLMNi2gyK2N0IHkf5SK68DRkR8asKm10iOIaXVkN4
+          riQQqirjYHzIzWS2s/dKoLIH5DTpRHZUl4n8i7sdN5lhdoxjga5+Ep+FWTG8oSWN6ZJFP
+          jCEhvlPc9znUjZ1xNpLdNyLoRutuSXNmLajTFxvR7SNciAGOZdxHBcCwfL6fXO6UqCBb7
+          9iHVHdSLq8EMa3I3GaZ0M4VgoHfJ8XHC1D82fIQbzgZ1I4UJX6dIV+rkFQ4nhY+xUu4Sx
+          opWNT264qYejkqFz7a8VIc1X+bqBO/VDyyp7d25ZR8pRjZRJtpkdBcX+Rb0CUzQ8Gb3wo
+          FLqGxy17EUzB2m4l9+paDOPXnp1zOrSnCnFYKfHhWAkWQAJslubgjFn8tF9Afo1lZ4R83
+          veqqznWfPAtzy6Gn0WAgAjNV9VdK0lIpZ5JLLkhzrmjmhpN2dJIcJabfPabi9NwEtqAO/
+          4HDPZqmNbBYFaRGqhNd+ZwuTVEV38N4ZFv7sKDe0dM7IOjtL9zWeqeFI/raPXjyjE5W/P
+          +HG7rspYyh26JeiYOpNVxXbCn7IzZ48DB7keUZsiTyy6PFHEdzHRrYHxiT1LUZRrpCuLs
+          vPknbPwhA3KtKi7wKMvI2umXV+0zmzOk6Oq5OkIBv7aQp0eHAbpdL9TTr9vKPc=
+      wavestack_ac_secret: !encrypted/pkcs1-oaep
+        - Wd9fYe9nn6Dw4bY/WnQ6BBej8Mw59YWqBVH+zFyVmFIj6WpFpux7vLVVjujYsXIvNAcRA
+          t3+oky/N/AEINHP7XUczPt15P0/GJ8rAWd4L/NYN/p90hjsRE0QTHzp9GDpy+ZgyUuso6
+          HGUfDiykoWrSe/9Vl0BwqZJLzrOjBeK1HOccO8GvVkyRYk5WI2OksVtiQxKbcrffS/BJK
+          9vU+VlphqiRVPGe8cyAjtoLwm1HH6FQ5VqjXYi4R2uhgTJM66/ueRky2GFCuFThxUGLQl
+          YBA0+Kt/2W7ejtlII/7KA5Gy7Zqw5/lM6quVJzJE+jarMj/YbZnNbpUUQws9z88ZfMA0Q
+          Siix+3irhIwTWDgeZyO2G20pPLGoahj4LWDzi4xFeOY1w0xXB0vkNDpGY59BRBn/oJ5zW
+          Pap+BdSbstyqHpJDkwteplJxbh5hN510opGFw4D6PbtoYKlsXfD1GPprHB0tnTPot9IgL
+          wzWYpJQQ6gehh3OTs5i9fLetoaQ59HF01SVDQL44VKT3xi4P5Wr9u+hLCbGDeu+OMPNGM
+          UYLHSWwKv893lov4K6/e4jz00EikV17kd+tVW1qMKMRFxdrRgwKP+wr50do2zVxhBMu4H
+          Co5I7CeK4GeN3KURGCKfcm/IRm+8A9j1+ocJelQf4T5DZtUlLrkGBe2SGw/V2I=
diff --git a/Tests/iaas/flavor-naming/flavor-names-openstack.py b/Tests/iaas/flavor-naming/flavor-names-openstack.py
@@ -58,7 +58,7 @@ def main(argv):
                                        ("os-cloud=", "mand=", "verbose", "help", "quiet", "v2plus",
                                         "v3", "v1prefer", "accept-old-mandatory"))
     except getopt.GetoptError as exc:
-        print(f"{exc}", file=sys.stderr)
+        print(f"CRITICAL: {exc!r}", file=sys.stderr)
         usage(1)
     for opt in opts:
         if opt[0] == "-h" or opt[0] == "--help":
@@ -85,13 +85,14 @@ def main(argv):
         else:
             usage(2)
     if len(args) > 0:
-        print(f"Extra arguments {str(args)}", file=sys.stderr)
+        print(f"CRITICAL: Extra arguments {str(args)}", file=sys.stderr)
         usage(1)
 
     scsMandatory, scsRecommended = fnmck.readflavors(scsMandFile, v3mode, fnmck.prefer_old)
 
     if not cloud:
-        print("ERROR: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr)
+        print("CRITICAL: You need to have OS_CLOUD set or pass --os-cloud=CLOUD.", file=sys.stderr)
+        sys.exit(1)
     conn = openstack.connect(cloud=cloud, timeout=32)
     flavors = conn.compute.flavors()
 
@@ -187,7 +188,7 @@ def main(argv):
         except NameError as exc:
             errors += 1
             wrongFlv.append(flv.name)
-            print(f"Wrong flavor \"{flv.name}\": {exc}", file=sys.stderr)
+            print(f"ERROR: Wrong flavor \"{flv.name}\": {exc}", file=sys.stderr)
     # This makes the output more readable
     MSCSFlv.sort()
     RSCSFlv.sort()
@@ -196,8 +197,9 @@ def main(argv):
     wrongFlv.sort()
     warnFlv.sort()
     # We have counted errors on the fly, add missing flavors to the final result
-    if scsMandatory:
-        errors += len(scsMandatory)
+    for fn in scsMandatory:
+        errors += 1
+        print(f"ERROR: Missing mandatory flavor: {fn}", file=sys.stderr)
     # Produce dicts for YAML reporting
     flvSCSList = {
         "MandatoryFlavorsPresent": MSCSFlv,
@@ -250,4 +252,10 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
+    try:
+        sys.exit(main(sys.argv[1:]))
+    except SystemExit:
+        raise
+    except BaseException as exc:
+        print(f"CRITICAL: {exc!r}", file=sys.stderr)
+        sys.exit(1)
diff --git a/Tests/iaas/image-metadata/image-md-check.py b/Tests/iaas/image-metadata/image-md-check.py
@@ -216,6 +216,7 @@ def main(argv):
         opts, args = getopt.gnu_getopt(argv[1:], "phvc:s",
                                        ("private", "help", "os-cloud=", "verbose", "skip-completeness"))
     except getopt.GetoptError:  # as exc:
+        print("CRITICAL: Command-line syntax error", file=sys.stderr)
         usage(1)
     for opt in opts:
         if opt[0] == "-h" or opt[0] == "--help":
@@ -230,18 +231,21 @@ def main(argv):
             cloud = opt[1]
     images = args
     if not cloud:
-        print("ERROR: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr)
+        print("CRITICAL: Need to specify --os-cloud or set OS_CLOUD environment.", file=sys.stderr)
         usage(1)
-    conn = openstack.connect(cloud=cloud, timeout=24)
-    # Do work
-    if not images:
-        images = get_imagelist(private)
-    err = 0
-    # Analyse image metadata
-    for image in images:
-        err += validate_imageMD(image)
-    if not skip:
-        err += report_stdimage_coverage(images)
+    try:
+        conn = openstack.connect(cloud=cloud, timeout=24)
+        # Do work
+        if not images:
+            images = get_imagelist(private)
+        err = 0
+        # Analyse image metadata
+        for image in images:
+            err += validate_imageMD(image)
+        if not skip:
+            err += report_stdimage_coverage(images)
+    except BaseException as e:
+        print(f"CRITICAL: {e!r}")
     return err
 
 

diff --git a/Tests/scs-compatible-iaas.yaml b/Tests/scs-compatible-iaas.yaml
@@ -1,24 +1,48 @@
 name: SCS Compatible IaaS
 url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-iaas.yaml
 versions:
-  - version: v1
-    stabilized_at: 2021-01-01
-    obsoleted_at: 2023-10-31
+  - version: v4
     standards:
+      - name: Standard flavors
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0103-v1-standard-flavors.md
+        check_tools:
+          - executable: ./iaas/standard-flavors/flavors-openstack.py
+            args: "./iaas/scs-0103-v1-flavors.yaml"
+      - name: Standard images
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0104-v1-standard-images.md
+        check_tools:
+          - executable: ./iaas/standard-images/images-openstack.py
+            args: "./iaas/scs-0104-v1-images.yaml"
       - name: Flavor naming
-        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v1-flavor-naming.md
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md
         check_tools:
           - executable: ./iaas/flavor-naming/flavor-names-openstack.py
-            args: "--v1prefer"
+            args: "--mand=./iaas/scs-0100-v3-flavors.yaml"
       - name: Image metadata
         url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md
         check_tools:
           - executable: ./iaas/image-metadata/image-md-check.py
-            args: -v
-      - name: OpenStack Powered Compute v2020.11
-        url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2020.11.json
-        condition: mandatory
+            args: -s -v
+      - name: OpenStack Powered Compute v2022.11
+        url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json
         # Unfortunately, no wrapper to run refstack yet, needs to be added
+  - version: v3
+    stabilized_at: 2023-06-15
+    obsoleted_at: 2024-04-30
+    standards:
+      - name: Flavor naming
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md
+        check_tools:
+          - executable: ./iaas/flavor-naming/flavor-names-openstack.py
+            args: "--v3"
+            # Note: "--v3 --v2plus" would outlaw the v1 flavor names. Don't do this yet.
+      - name: Image metadata
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md
+        check_tools:
+          - executable: ./iaas/image-metadata/image-md-check.py
+            args: -v
+      - name: OpenStack Powered Compute v2022.11
+        url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json
   - version: v2
     stabilized_at: 2023-03-23
     obsoleted_at: 2023-11-30
@@ -34,24 +58,19 @@ versions:
             args: -v
       - name: OpenStack Powered Compute v2022.11
         url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json
-        condition: mandatory
-        # Unfortunately, no wrapper to run refstack yet, needs to be added
-  - version: v3
-    stabilized_at: 2023-06-15
-    obsoleted_at: 2024-04-30
+  - version: v1
+    stabilized_at: 2021-01-01
+    obsoleted_at: 2023-10-31
     standards:
       - name: Flavor naming
-        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v3-flavor-naming.md
+        url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0100-v1-flavor-naming.md
         check_tools:
           - executable: ./iaas/flavor-naming/flavor-names-openstack.py
-            args: "--v3"
-            # Note: "--v3 --v2plus" would outlaw the v1 flavor names. Don't do this yet.
+            args: "--v1prefer"
       - name: Image metadata
         url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0102-v1-image-metadata.md
         check_tools:
           - executable: ./iaas/image-metadata/image-md-check.py
             args: -v
-      - name: OpenStack Powered Compute v2022.11
-        url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2022.11.json
-        condition: mandatory
-        # Unfortunately, no wrapper to run refstack yet, needs to be added
+      - name: OpenStack Powered Compute v2020.11
+        url: https://opendev.org/openinfra/interop/src/branch/master/guidelines/2020.11.json