Update to Kubernetes 1.14.8 (#254)

README.rst

@@ -157,7 +157,7 @@ to initialize the required cloud resources.
     -e LOCATION={an azure location like eastus} \
     -e RESOURCE_GROUP_NAME={the name of the resource group to create or reuse} \
     -v ${PWD}/secrets:/secrets \
-    setup
+    setup ./setup.sh
 
 The secrets to access the Azure resources created by the setup script will be
 stored in files in the :code:`secrets` directory. Other parts of the

docker/setup/Dockerfile

@@ -1,16 +1,19 @@
 FROM microsoft/azure-cli:2.0.32
 
-ARG HELM_VERSION="v2.9.1"
-ARG KUBECTL_VERSION="v1.10.3"
+ARG HELM_VERSION="2.16.1"
+ARG KUBECTL_VERSION="1.14.8"
+ENV KUBERNETES_VERSION="1.14.8"
+ENV CERT_MANAGER_VERSION="0.8.0"
+ENV NGINX_INGRESS_VERSION="0.3.7"
 
 RUN apk add -q --no-cache \
     jq=1.5-r2 \
     curl=7.59.0-r0 && \
-  curl -sLfO "https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz" && \
-  tar xf "helm-${HELM_VERSION}-linux-amd64.tar.gz" && \
+  curl -sLfO "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \
+  tar xf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \
   mv "linux-amd64/helm" /usr/local/bin/helm && \
   chmod +x /usr/local/bin/helm && \
-  rm -rf "linux-amd64" "helm-${HELM_VERSION}-linux-amd64.tar.gz" && \
+  rm -rf "linux-amd64" "helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \
   az aks install-cli --client-version "${KUBECTL_VERSION}" && \
   mkdir /secrets
 

docker/setup/renew-cert.sh

@@ -16,17 +16,20 @@ scriptname="${BASH_SOURCE[0]}"
 required_file "${scriptname}" "/secrets/kube-config"
 
 #
-# delete kube-lego pod: the pod will be automatically re-created which will
+# delete cert-manager pod: the pod will be automatically re-created which will
 # force a refresh of the https certificate if required
 #
 
 export KUBECONFIG="/secrets/kube-config"
 
-log "Looking up current kube-lego pods"
-kubectl get pod -l app=kube-lego
+log "Looking up current cert-manager pods"
+kubectl get pod -l certmanager.k8s.io/acme-http01-solver=true
+kubectl get pod -n cert-manager
 
-log "Re-creating kube-lego pod"
-kubectl delete pod -l app=kube-lego
+log "Re-creating cert-manager pod"
+kubectl delete pod -l certmanager.k8s.io/acme-http01-solver=true
+kubectl delete pod -n cert-manager -l app=cert-manager
 
-log "Looking up new kube-lego pods"
-kubectl get pod -l app=kube-lego
+log "Looking up new cert-manager pods"
+kubectl get pod -l certmanager.k8s.io/acme-http01-solver=true
+kubectl get pod -n cert-manager

docker/setup/setup.sh

@@ -11,10 +11,10 @@
 ##   SP_TENANT
 ##   SUBSCRIPTION_ID
 ##   LOCATION
-##   RESOURCE_GROUP_NAME
 ##
 ## Optional environment variables:
 ##
+##   RESOURCE_GROUP_NAME
 ##   SERVICE_BUS_SKU
 ##   STORAGE_ACCOUNT_SKU
 ##
@@ -23,6 +23,7 @@
 ##   KUBERNETES_DOCKER_TAG
 ##   KUBERNETES_NODE_SKU
 ##   KUBERNETES_NODE_COUNT
+##   KUBERNETES_VERSION
 ##   LOKOLE_DNS_NAME
 ##
 
@@ -45,7 +46,6 @@ required_env "${scriptname}" "SP_PASSWORD"
 required_env "${scriptname}" "SP_TENANT"
 required_env "${scriptname}" "SUBSCRIPTION_ID"
 required_env "${scriptname}" "LOCATION"
-required_env "${scriptname}" "RESOURCE_GROUP_NAME"
 
 #
 # connect to azure
@@ -61,6 +61,8 @@ az configure --defaults location="${LOCATION}"
 #
 if [[ "${DEPLOY_SERVICES}" != "no" ]]; then
 
+required_env "${scriptname}" "RESOURCE_GROUP_NAME"
+
 use_resource_group "${RESOURCE_GROUP_NAME}"
 
 storageaccountsku="${STORAGE_ACCOUNT_SKU:-Standard_GRS}"
@@ -103,15 +105,15 @@ fi
 #
 if [[ "${DEPLOY_COMPUTE}" != "no" ]]; then
 
-if [[ -z "${KUBERNETES_RESOURCE_GROUP_NAME}" ]] || [[ -z "${KUBERNETES_NODE_COUNT}" ]] || [[ -z "${KUBERNETES_NODE_SKU}" ]]; then
-  log "Skipping production deployment to kubernetes since KUBERNETES_RESOURCE_GROUP_NAME, KUBERNETES_NODE_COUNT, or KUBERNETES_NODE_SKU are not set"
+if [[ -z "${KUBERNETES_RESOURCE_GROUP_NAME}" ]] || [[ -z "${KUBERNETES_NODE_COUNT}" ]] || [[ -z "${KUBERNETES_NODE_SKU}" ]] || [[ -z "${KUBERNETES_VERSION}" ]]; then
+  log "Skipping production deployment to kubernetes since KUBERNETES_RESOURCE_GROUP_NAME, KUBERNETES_NODE_COUNT, or KUBERNETES_NODE_SKU, or KUBERNETES_VERSION are not set"
   exit 0
 fi
 
 k8sname="opwencluster$(generate_identifier 8)"
 helmname="opwenserver$(generate_identifier 8)"
 
-log "Creating kubernetes cluster ${k8sname}"
+log "Creating kubernetes v${KUBERNETES_VERSION} cluster ${k8sname}"
 
 use_resource_group "${KUBERNETES_RESOURCE_GROUP_NAME}"
 
@@ -121,6 +123,7 @@ az provider register --wait --namespace Microsoft.Compute
 az provider register --wait --namespace Microsoft.ContainerService
 
 az aks create \
+  --kubernetes-version "${KUBERNETES_VERSION}" \
   --service-principal "${SP_APPID}" \
   --client-secret "${SP_PASSWORD}" \
   --name "${k8sname}" \
@@ -130,17 +133,53 @@ az aks create \
 
 az aks get-credentials --name "${k8sname}"
 
+log "Setting up helm in cluster ${k8sname}"
+
+kubectl apply -f- <<< "
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tiller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: tiller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: tiller
+    namespace: kube-system
+"
+helm_init
+
+log "Setting up cert-manager v${CERT_MANAGER_VERSION} in cluster ${k8sname}"
+
+kubectl apply -f "https://raw.githubusercontent.com/jetstack/cert-manager/v${CERT_MANAGER_VERSION}/deploy/manifests/00-crds.yaml"
+kubectl create namespace cert-manager
+kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install --name cert-manager --namespace cert-manager --version "v${CERT_MANAGER_VERSION}" jetstack/cert-manager --wait
+
+log "Setting up nginx-ingress in cluster ${k8sname}"
+
+helm repo add nginx-stable https://helm.nginx.com/stable
+helm repo update
+helm install --name nginx-ingress --version "${NGINX_INGRESS_VERSION}" nginx-stable/nginx-ingress --set controller.replicaCount=3
+
 log "Setting up kubernetes secrets for ${k8sname}"
 
 kubectl create secret generic "azure" --from-env-file "/secrets/azure.env"
 kubectl create secret generic "cloudflare" --from-env-file "/secrets/cloudflare.env"
 kubectl create secret generic "users" --from-env-file "/secrets/users.env"
 kubectl create secret generic "sendgrid" --from-env-file "/secrets/sendgrid.env"
 
-log "Setting up helm chart in cluster ${k8sname}"
-
-helm init --wait
-helm dependency update "${scriptdir}/helm"
+log "Installing application in ${k8sname}"
 
 k8simageregistry="${KUBERNETES_IMAGE_REGISTRY:-ascoderu}"
 k8sdockertag="${KUBERNETES_DOCKER_TAG:-latest}"
@@ -159,7 +198,7 @@ while :; do
 done
 
 while :; do
-  ingressip="$(kubectl get service --selector app=nginx-ingress,component=controller --output jsonpath={..ip})"
+  ingressip="$(kubectl get service --selector app.kubernetes.io/instance=nginx-ingress --output jsonpath={..ip})"
   if [[ -z "${ingressip}" ]]; then log "Waiting for ${k8sname} public IP"; sleep 30s; else break; fi
 done
 
@@ -193,6 +232,8 @@ else
     -d '{"type":"A","name":"'"${lokole_dns_name}"'","content":"'"${ingressip}"'","ttl":1,"proxied":false}'
 fi
 
+./renew-cert.sh
+
 cat > /secrets/kubedeployment.env << EOF
 RESOURCE_GROUP=${KUBERNETES_RESOURCE_GROUP_NAME}
 HELM_NAME=${helmname}
@@ -207,7 +248,7 @@ fi
 
 storage_account="$(get_dotenv '/secrets/azure.env' 'LOKOLE_EMAIL_SERVER_AZURE_BLOBS_NAME')"
 storage_key="$(get_dotenv '/secrets/azure.env' 'LOKOLE_EMAIL_SERVER_AZURE_BLOBS_KEY')"
-container_name="secrets"
+container_name="secrets-${k8sname}"
 
 log "Backing up secrets to ${storage_account}/${container_name}"
 

docker/setup/upgrade.sh

@@ -34,9 +34,7 @@ log "Upgrading helm deployment ${HELM_NAME}"
 
 export KUBECONFIG="/secrets/kube-config"
 
-helm init --client-only --wait
-
-helm dependency update "${scriptdir}/helm"
+helm_init
 
 helm upgrade "${HELM_NAME}" \
   --set domain="${LOKOLE_DNS_NAME}" \

docker/setup/utils.sh

@@ -69,3 +69,7 @@ use_resource_group() {
 
   az configure --defaults group="${name}"
 }
+
+helm_init() {
+  helm init --history-max 200 --service-account tiller --node-selectors "beta.kubernetes.io/os=linux" --wait
+}

helm/templates/api-autoscaler.yaml

@@ -7,7 +7,7 @@ spec:
   maxReplicas: 9
   minReplicas: 3
   scaleTargetRef:
-    apiVersion: extensions/v1beta1
+    apiVersion: apps/v1
     kind: Deployment
     name: {{ .Release.Name }}-api
   targetCPUUtilizationPercentage: 75

helm/templates/api-deployment.yaml

@@ -1,18 +1,21 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   creationTimestamp: null
   labels:
-    io.kompose.service: {{ .Release.Name }}-api
+    app: {{ .Release.Name }}-api
   name: {{ .Release.Name }}-api
 spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-api
   replicas: 1
   strategy: {}
   template:
     metadata:
       creationTimestamp: null
       labels:
-        io.kompose.service: {{ .Release.Name }}-api
+        app: {{ .Release.Name }}-api
     spec:
       containers:
       - name: api
@@ -29,11 +32,6 @@ spec:
             secretKeyRef:
               name: cloudflare
               key: LOKOLE_CLOUDFLARE_USER
-        - name: LOKOLE_CLOUDFLARE_ZONE
-          valueFrom:
-            secretKeyRef:
-              name: cloudflare
-              key: LOKOLE_CLOUDFLARE_ZONE
         - name: LOKOLE_CLOUDFLARE_KEY
           valueFrom:
             secretKeyRef:

helm/templates/api-service.yaml

@@ -3,14 +3,14 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    io.kompose.service: {{ .Release.Name }}-api
+    app: {{ .Release.Name }}-api
   name: {{ .Release.Name }}-api
 spec:
   ports:
   - name: "8080"
     port: 8080
     targetPort: 8080
   selector:
-    io.kompose.service: {{ .Release.Name }}-api
+    app: {{ .Release.Name }}-api
 status:
   loadBalancer: {}

helm/templates/cluster-issuer.yaml

@@ -0,0 +1,11 @@
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: ClusterIssuer
+metadata:
+  name: {{ .Release.Name }}-cluster-issuer
+spec:
+  acme:
+    server: {{ .Values.letsencrypt.url }}
+    email: {{ .Values.letsencrypt.email }}
+    privateKeySecretRef:
+      name: {{ .Release.Name }}-tls-secret
+    http01: {}

helm/templates/ingress.yaml

@@ -1,15 +1,15 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: {{ .Release.Name }}-ingress
   annotations:
     kubernetes.io/ingress.class: nginx
-    kubernetes.io/tls-acme: "true"
+    certmanager.k8s.io/cluster-issuer: {{ .Release.Name }}-cluster-issuer
 spec:
   tls:
   - hosts:
     - {{.Values.domain}}
-    secretName: lego-secret
+    secretName: {{ .Release.Name }}-tls-secret
   rules:
   - host: {{.Values.domain}}
     http:

helm/templates/nginx-autoscaler.yaml

@@ -7,7 +7,7 @@ spec:
   maxReplicas: 5
   minReplicas: 3
   scaleTargetRef:
-    apiVersion: extensions/v1beta1
+    apiVersion: apps/v1
     kind: Deployment
     name: {{ .Release.Name }}-nginx
   targetCPUUtilizationPercentage: 75

helm/templates/nginx-deployment.yaml

@@ -1,18 +1,21 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   creationTimestamp: null
   labels:
-    io.kompose.service: {{ .Release.Name }}-nginx
+    app: {{ .Release.Name }}-nginx
   name: {{ .Release.Name }}-nginx
 spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-nginx
   replicas: 1
   strategy: {}
   template:
     metadata:
       creationTimestamp: null
       labels:
-        io.kompose.service: {{ .Release.Name }}-nginx
+        app: {{ .Release.Name }}-nginx
     spec:
       containers:
       - name: nginx

helm/templates/nginx-service.yaml

@@ -3,14 +3,14 @@ kind: Service
 metadata:
   creationTimestamp: null
   labels:
-    io.kompose.service: {{ .Release.Name }}-nginx
+    app: {{ .Release.Name }}-nginx
   name: {{ .Release.Name }}-nginx
 spec:
   ports:
   - name: "8888"
     port: 8888
     targetPort: 8888
   selector:
-    io.kompose.service: {{ .Release.Name }}-nginx
+    app: {{ .Release.Name }}-nginx
 status:
   loadBalancer: {}

helm/templates/worker-autoscaler.yaml

@@ -7,7 +7,7 @@ spec:
   maxReplicas: 9
   minReplicas: 3
   scaleTargetRef:
-    apiVersion: extensions/v1beta1
+    apiVersion: apps/v1
     kind: Deployment
     name: {{ .Release.Name }}-worker
   targetCPUUtilizationPercentage: 75