Add CoreDNS deployment to ostests pre-flight checks

Sonobuoy expects the DNS pods to be running during its pre-flight
tests. In addition to check for konnectivity, also check for the CoreDNS
deployment, so that there are less CI failures due to bad timing.

Signed-off-by: Tom Wieczorek <twieczorek@mirantis.com>

hack/ostests/modules/k0sctl/main.tf

@@ -27,10 +27,10 @@ locals {
   num_workers     = length([for h in terraform_data.k0sctl_apply.output.hosts : h if h.is_worker])
 }
 
-resource "terraform_data" "konnectivity_available" {
+resource "terraform_data" "pre_flight_checks" {
   triggers_replace = [
     sha256(jsonencode(terraform_data.k0sctl_apply.output.k0sctl_config)),
-    sha256(file("${path.module}/wait-for-konnectivity.sh")),
+    sha256(file("${path.module}/pre-flight-checks.sh")),
   ]
 
   input = {
@@ -51,7 +51,7 @@ resource "terraform_data" "konnectivity_available" {
     inline = [
       "#!/usr/bin/env sh",
       format("set -- %d %d", local.num_controllers, local.num_workers),
-      file("${path.module}/wait-for-konnectivity.sh"),
+      file("${path.module}/pre-flight-checks.sh"),
     ]
   }
 }

hack/ostests/modules/k0sctl/outputs.tf

@@ -1,15 +1,15 @@
 output "hosts" {
-  value       = terraform_data.konnectivity_available.output.hosts
+  value       = terraform_data.pre_flight_checks.output.hosts
   description = "The hosts that have been provisioned by k0sctl."
 }
 
 output "ssh_private_key_filename" {
-  value       = terraform_data.konnectivity_available.output.ssh_private_key_filename
+  value       = terraform_data.pre_flight_checks.output.ssh_private_key_filename
   description = "The name of the private key file that has been used to authenticate via SSH."
 }
 
 output "k0sctl_config" {
-  value       = terraform_data.konnectivity_available.output.k0sctl_config
+  value       = terraform_data.pre_flight_checks.output.k0sctl_config
   description = "The k0sctl config that has been used."
 }
 

hack/ostests/modules/k0sctl/wait-for-konnectivity.sh → hack/ostests/modules/k0sctl/pre-flight-checks.sh

@@ -37,7 +37,7 @@ countSeenControllers() {
 }
 
 testKonnectivityPods() {
-  pods="$(kubectl -n kube-system get po -l k8s-app=konnectivity-agent -oname)"
+  pods="$(kubectl -n kube-system get po -l k8s-app=konnectivity-agent --field-selector=status.phase=Running -oname)"
 
   seenPods=0
   for pod in $pods; do
@@ -50,6 +50,10 @@ testKonnectivityPods() {
   [ $seenPods -eq "$expectedWorkers" ]
 }
 
+testKubeDnsPods() {
+  kubectl -n kube-system wait --for=condition=Available --timeout=9s deploy/coredns
+}
+
 main() {
   {
     [ $# -eq 2 ] \
@@ -67,8 +71,7 @@ main() {
   echo Expecting "$expectedWorkers" pods with "$expectedControllers" controller connections each ... >&2
 
   failedAttempts=0
-  while :; do
-    ! testKonnectivityPods || return 0
+  while ! testKonnectivityPods; do
     failedAttempts=$((failedAttempts + 1))
     if [ $failedAttempts -gt 30 ]; then
       echo Giving up after $failedAttempts failed attempts ... >&2
@@ -77,6 +80,19 @@ main() {
     echo Attempt $failedAttempts failed, retrying in ten seconds ... >&2
     sleep 10
   done
+
+  echo Waiting for CoreDNS deployment to become available ... >&2
+
+  failedAttempts=0
+  while ! testKubeDnsPods; do
+    failedAttempts=$((failedAttempts + 1))
+    if [ $failedAttempts -gt 30 ]; then
+      echo Giving up after $failedAttempts failed attempts ... >&2
+      return 1
+    fi
+    echo Attempt $failedAttempts failed, retrying in a second ... >&2
+    sleep 1
+  done
 }
 
 main "$@"