Add VM deployment option (#336)

* Switch to moustache templating

* Add option to increase number of nginx workers

* Add option to use LetsEncrypt certificates

* Make nginx config compatible with system setup

* Add deployment scripts for VM

.env

@@ -4,6 +4,7 @@ BUILD_TARGET=builder
 DOCKER_REPO=ascoderu
 SERVER_WORKERS=1
 QUEUE_WORKERS=1
+NGINX_WORKERS=1
 LOKOLE_LOG_LEVEL=INFO
 LOKOLE_QUEUE_BROKER_SCHEME=amqp
 LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME=

README.rst

@@ -174,13 +174,6 @@ stored in files in the :code:`secrets` directory. Other parts of the
 project's tooling (e.g. docker-compose) depend on these files so make sure to
 not delete them.
 
-To run the project using the Azure resources created by the setup, use the
-following command:
-
-.. sourcecode :: sh
-
-  make start-azure
-
 ---------------------
 Production deployment
 ---------------------

docker-compose.yml

@@ -49,6 +49,7 @@ services:
     ports:
       - ${APP_PORT}:8888
     environment:
+      NGINX_WORKERS: ${NGINX_WORKERS}
       DNS_RESOLVER: 127.0.0.11
       HOSTNAME_WEBAPP: webapp:8080
       HOSTNAME_CLIENT_METRICS: api:8080

docker/docker-compose.prod.yml

@@ -0,0 +1,41 @@
+version: '3.4'
+
+x-shared-secret-environment:
+  &shared-secret-environment
+  environment:
+    PORT: 8888
+    LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS
+    LOKOLE_QUEUE_BROKER_SCHEME: azureservicebus
+    CONNEXION_SPEC: dir:/app/opwen_email_server/swagger
+    CELERY_QUEUE_NAMES: all
+    TESTING_UI: "False"
+    LOKOLE_LOG_LEVEL: INFO
+    SERVER_WORKERS: 4
+    QUEUE_WORKERS: 5
+  env_file:
+    - ../secrets/azure.env
+    - ../secrets/cloudflare.env
+    - ../secrets/users.env
+    - ../secrets/sendgrid.env
+  volumes:
+    - /tmp:/tmp
+
+services:
+
+  webapp:
+    image: ascoderu/opwenwebapp:latest
+    <<: *shared-secret-environment
+    ports:
+      - 8080:8080
+
+  api:
+    image: ascoderu/opwenserver_app:latest
+    command: ["/app/docker/app/run-gunicorn.sh"]
+    <<: *shared-secret-environment
+    ports:
+      - 8888:8888
+
+  worker:
+    image: ascoderu/opwenserver_app:latest
+    command: ["/app/docker/app/run-celery.sh"]
+    <<: *shared-secret-environment

docker/nginx/Dockerfile

@@ -1,14 +1,25 @@
+ARG PYTHON_VERSION=3.7
+FROM python:${PYTHON_VERSION} AS builder
+
+RUN curl -sSL https://git.io/get-mo -o /usr/bin/mo \
+  && chmod +x /usr/bin/mo
+
 FROM nginx:stable
 
+COPY --from=builder /usr/bin/mo /usr/bin/mo
 COPY docker/nginx/static /static
 COPY docker/nginx/nginx.conf.template /app/nginx.conf.template
+COPY docker/nginx/server.conf.template /app/server.conf.template
 COPY docker/nginx/run-nginx.sh /app/run-nginx.sh
 
-RUN mkdir -p /var/cache/nginx \
+RUN mkdir -p /var/cache/nginx /etc/nginx/modules-enabled /etc/nginx/sites-enabled \
   && rm /etc/nginx/conf.d/default.conf \
-  && chown -R nginx:nginx \
+  && chown -R www-data:www-data \
     /app \
     /static \
+    /run \
+    /etc/nginx/modules-enabled \
+    /etc/nginx/sites-enabled \
     /var/cache/nginx
 
 ENV DNS_RESOLVER=""
@@ -21,7 +32,7 @@ ENV HOSTNAME_CLIENT_REGISTER="SET_ME"
 ENV PORT=8888
 
 EXPOSE ${PORT}
-USER nginx
+USER www-data
 WORKDIR /static
 
 CMD ["/app/run-nginx.sh"]

docker/nginx/nginx.conf.template

@@ -1,7 +1,7 @@
-worker_processes 1;
-
-error_log /var/log/nginx/error.log warn;
-pid /app/nginx.pid;
+user www-data;
+worker_processes {{NGINX_WORKERS}};
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
 
 events {
   worker_connections 1024;
@@ -16,65 +16,19 @@ http {
     '$status $body_bytes_sent "$http_referer" '
     '"$http_user_agent" "$http_x_forwarded_for"';
 
-  access_log /var/log/nginx/access.log main;
-
   sendfile on;
-
+  tcp_nopush on;
+  tcp_nodelay on;
   keepalive_timeout 65;
+  types_hash_max_size 2048;
 
-  upstream healthcheck_hosts {
-    server ${HOSTNAME_EMAIL_RECEIVE};
-    server ${HOSTNAME_CLIENT_METRICS};
-    server ${HOSTNAME_CLIENT_WRITE};
-    server ${HOSTNAME_CLIENT_READ};
-    server ${HOSTNAME_CLIENT_REGISTER};
-  }
-
-  server {
-    listen ${PORT};
-
-    resolver ${DNS_RESOLVER};
-
-    client_max_body_size 50M;
+  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+  ssl_prefer_server_ciphers on;
 
-    location = /favicon.ico {
-      root /static;
-    }
-
-    location = /robots.txt {
-      root /static;
-    }
-
-    location /healthcheck {
-      proxy_pass http://healthcheck_hosts;
-    }
-
-    location /web {
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_pass http://${HOSTNAME_WEBAPP};
-    }
-
-    location /api/email/sendgrid {
-      proxy_pass http://${HOSTNAME_EMAIL_RECEIVE};
-    }
-
-    location /api/email/metrics {
-      proxy_pass http://${HOSTNAME_CLIENT_METRICS};
-    }
-
-    location /api/email/upload {
-      proxy_pass http://${HOSTNAME_CLIENT_WRITE};
-    }
+  access_log /var/log/nginx/access.log main;
+  error_log /var/log/nginx/error.log warn;
 
-    location /api/email/download {
-      proxy_pass http://${HOSTNAME_CLIENT_READ};
-    }
+  gzip on;
 
-    location /api/email/register {
-      proxy_pass http://${HOSTNAME_CLIENT_REGISTER};
-    }
-  }
+  include /etc/nginx/sites-enabled/*;
 }

docker/nginx/run-nginx.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
-envsubst < /app/nginx.conf.template > /app/nginx.conf "$(env | sed -e 's/=.*//' -e 's/^/\$/g')"
+mo < /app/nginx.conf.template > /app/nginx.conf
+mo < /app/server.conf.template > /etc/nginx/sites-enabled/server.conf
 
 nginx -c "/app/nginx.conf" -p "${PWD}" -g "daemon off;"

docker/nginx/server.conf.template

@@ -0,0 +1,68 @@
+upstream healthcheck_hosts {
+  server {{HOSTNAME_EMAIL_RECEIVE}};
+  server {{HOSTNAME_CLIENT_METRICS}};
+  server {{HOSTNAME_CLIENT_WRITE}};
+  server {{HOSTNAME_CLIENT_READ}};
+  server {{HOSTNAME_CLIENT_REGISTER}};
+}
+
+server {
+  listen {{PORT}};
+
+  {{#LETSENCRYPT_DOMAIN}}
+  server_name {{LETSENCRYPT_DOMAIN}};
+  listen [::]:443 ssl ipv6only=on; # managed by Certbot
+  listen 443 ssl; # managed by Certbot
+  ssl_certificate /etc/letsencrypt/live/mailserver.lokole.ca/fullchain.pem; # managed by Certbot
+  ssl_certificate_key /etc/letsencrypt/live/mailserver.lokole.ca/privkey.pem; # managed by Certbot
+  include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+  ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+  if ($scheme != "https") { return 301 https://$host$request_uri; } # managed by Certbot
+  {{/LETSENCRYPT_DOMAIN}}
+
+  {{#DNS_RESOLVER}}
+  resolver {{DNS_RESOLVER}};
+  {{/DNS_RESOLVER}}
+
+  client_max_body_size 50M;
+
+  location = /favicon.ico {
+    root {{STATIC_ROOT}}/static;
+  }
+
+  location = /robots.txt {
+    root {{STATIC_ROOT}}/static;
+  }
+
+  location /healthcheck {
+    proxy_pass http://healthcheck_hosts;
+  }
+
+  location /web {
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_pass http://{{HOSTNAME_WEBAPP}};
+  }
+
+  location /api/email/sendgrid {
+    proxy_pass http://{{HOSTNAME_EMAIL_RECEIVE}};
+  }
+
+  location /api/email/metrics {
+    proxy_pass http://{{HOSTNAME_CLIENT_METRICS}};
+  }
+
+  location /api/email/upload {
+    proxy_pass http://{{HOSTNAME_CLIENT_WRITE}};
+  }
+
+  location /api/email/download {
+    proxy_pass http://{{HOSTNAME_CLIENT_READ}};
+  }
+
+  location /api/email/register {
+    proxy_pass http://{{HOSTNAME_CLIENT_REGISTER}};
+  }
+}

docker/setup/Dockerfile

@@ -8,6 +8,7 @@ ENV NGINX_INGRESS_VERSION="0.3.7"
 
 RUN apk add -q --no-cache \
     jq=1.5-r2 \
+    sshpass=1.05-r0 \
     curl=7.59.0-r0 && \
   curl -sLfO "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \
   tar xf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \

docker/setup/upgrade-helm.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+##
+## This script upgrades an existing production deployment.
+## The script assumes that a kubernetes secret exists at /secrets/kube-config.
+##
+## Required environment variables:
+##
+##   DOCKER_TAG
+##   HELM_NAME
+##   IMAGE_REGISTRY
+##   LOKOLE_DNS_NAME
+##
+
+scriptdir="$(dirname "$0")"
+scriptname="${BASH_SOURCE[0]}"
+# shellcheck disable=SC1090
+. "${scriptdir}/utils.sh"
+
+#
+# verify inputs
+#
+
+required_env "${scriptname}" "DOCKER_TAG"
+required_env "${scriptname}" "HELM_NAME"
+required_env "${scriptname}" "IMAGE_REGISTRY"
+required_env "${scriptname}" "LOKOLE_DNS_NAME"
+required_file "${scriptname}" "/secrets/kube-config"
+
+#
+# upgrade production deployment
+#
+
+log "Upgrading helm deployment ${HELM_NAME}"
+
+export KUBECONFIG="/secrets/kube-config"
+
+helm_init
+
+helm upgrade "${HELM_NAME}" \
+  --set domain="${LOKOLE_DNS_NAME}" \
+  --set version.imageRegistry="${IMAGE_REGISTRY}" \
+  --set version.dockerTag="${DOCKER_TAG}" \
+  "${scriptdir}/helm/opwen_cloudserver"

docker/setup/upgrade-vm.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+##
+## This script upgrades an production VM.
+##
+## Required environment variables:
+##
+##   LOKOLE_VM_PASSWORD
+##   LOKOLE_DNS_NAME
+##
+
+scriptdir="$(dirname "$0")"
+scriptname="${BASH_SOURCE[0]}"
+# shellcheck disable=SC1090
+. "${scriptdir}/utils.sh"
+
+#
+# verify inputs
+#
+
+required_env "${scriptname}" "LOKOLE_VM_PASSWORD"
+required_env "${scriptname}" "LOKOLE_DNS_NAME"
+
+#
+# upgrade production deployment
+#
+
+log "Upgrading VM ${LOKOLE_DNS_NAME}"
+
+exec sshpass -p "${LOKOLE_VM_PASSWORD}" ssh -o StrictHostKeyChecking=no "opwen@${LOKOLE_DNS_NAME}" < "${scriptdir}/vm.sh"