diff --git a/.env b/.env index 606adf81..82b0c9ab 100644 --- a/.env +++ b/.env @@ -4,6 +4,7 @@ BUILD_TARGET=builder DOCKER_REPO=ascoderu SERVER_WORKERS=1 QUEUE_WORKERS=1 +NGINX_WORKERS=1 LOKOLE_LOG_LEVEL=INFO LOKOLE_QUEUE_BROKER_SCHEME=amqp LOKOLE_EMAIL_SERVER_QUEUES_SAS_NAME= diff --git a/README.rst b/README.rst index 0dcd3255..3a6e4290 100644 --- a/README.rst +++ b/README.rst @@ -174,13 +174,6 @@ stored in files in the :code:`secrets` directory. Other parts of the project's tooling (e.g. docker-compose) depend on these files so make sure to not delete them. -To run the project using the Azure resources created by the setup, use the -following command: - -.. sourcecode :: sh - - make start-azure - --------------------- Production deployment --------------------- diff --git a/docker-compose.yml b/docker-compose.yml index c97cbf34..61ec5c8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ services: ports: - ${APP_PORT}:8888 environment: + NGINX_WORKERS: ${NGINX_WORKERS} DNS_RESOLVER: 127.0.0.11 HOSTNAME_WEBAPP: webapp:8080 HOSTNAME_CLIENT_METRICS: api:8080 diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 00000000..2d26cb68 --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,41 @@ +version: '3.4' + +x-shared-secret-environment: + &shared-secret-environment + environment: + PORT: 8888 + LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS + LOKOLE_QUEUE_BROKER_SCHEME: azureservicebus + CONNEXION_SPEC: dir:/app/opwen_email_server/swagger + CELERY_QUEUE_NAMES: all + TESTING_UI: "False" + LOKOLE_LOG_LEVEL: INFO + SERVER_WORKERS: 4 + QUEUE_WORKERS: 5 + env_file: + - ../secrets/azure.env + - ../secrets/cloudflare.env + - ../secrets/users.env + - ../secrets/sendgrid.env + volumes: + - /tmp:/tmp + +services: + + webapp: + image: ascoderu/opwenwebapp:latest + <<: *shared-secret-environment + ports: + - 8080:8080 + + api: + image: ascoderu/opwenserver_app:latest + command: ["/app/docker/app/run-gunicorn.sh"] + <<: *shared-secret-environment + ports: + - 8888:8888 + + worker: + image: ascoderu/opwenserver_app:latest + command: ["/app/docker/app/run-celery.sh"] + <<: *shared-secret-environment diff --git a/docker/docker-compose.secrets.yml b/docker/docker-compose.secrets.yml deleted file mode 100644 index 87c5a9d4..00000000 --- a/docker/docker-compose.secrets.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: '3.4' - -x-shared-secret-environment: - &shared-secret-environment - environment: - DOTENV_SECRETS: azure;cloudflare;users;sendgrid - LOKOLE_STORAGE_PROVIDER: AZURE_BLOBS - LOKOLE_QUEUE_BROKER_SCHEME: azureservicebus - LOKOLE_EMAIL_SERVER_AZURE_BLOBS_HOST: - LOKOLE_EMAIL_SERVER_AZURE_BLOBS_SECURE: "True" - LOKOLE_EMAIL_SERVER_AZURE_TABLES_HOST: - LOKOLE_EMAIL_SERVER_AZURE_TABLES_SECURE: "True" - LOKOLE_CLIENT_AZURE_STORAGE_HOST: - LOKOLE_CLIENT_AZURE_STORAGE_SECURE: "True" - LOKOLE_EMAIL_SERVER_APPINSIGHTS_HOST: - secrets: - - azure - - cloudflare - - users - - sendgrid - -services: - - api: - <<: *shared-secret-environment - - worker: - <<: *shared-secret-environment - -secrets: - azure: - file: ./secrets/azure.env - cloudflare: - file: ./secrets/cloudflare.env - users: - file: ./secrets/users.env - sendgrid: - file: ./secrets/sendgrid.env diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 4de7237c..4f2f92ee 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -1,14 +1,25 @@ +ARG PYTHON_VERSION=3.7 +FROM python:${PYTHON_VERSION} AS builder + +RUN curl -sSL https://git.io/get-mo -o /usr/bin/mo \ + && chmod +x /usr/bin/mo + FROM nginx:stable +COPY --from=builder /usr/bin/mo /usr/bin/mo COPY docker/nginx/static /static COPY docker/nginx/nginx.conf.template /app/nginx.conf.template +COPY docker/nginx/server.conf.template /app/server.conf.template COPY docker/nginx/run-nginx.sh /app/run-nginx.sh -RUN mkdir -p /var/cache/nginx \ +RUN mkdir -p /var/cache/nginx /etc/nginx/modules-enabled /etc/nginx/sites-enabled \ && rm /etc/nginx/conf.d/default.conf \ - && chown -R nginx:nginx \ + && chown -R www-data:www-data \ /app \ /static \ + /run \ + /etc/nginx/modules-enabled \ + /etc/nginx/sites-enabled \ /var/cache/nginx ENV DNS_RESOLVER="" @@ -21,7 +32,7 @@ ENV HOSTNAME_CLIENT_REGISTER="SET_ME" ENV PORT=8888 EXPOSE ${PORT} -USER nginx +USER www-data WORKDIR /static CMD ["/app/run-nginx.sh"] diff --git a/docker/nginx/nginx.conf.template b/docker/nginx/nginx.conf.template index 38c052b1..9e34f182 100644 --- a/docker/nginx/nginx.conf.template +++ b/docker/nginx/nginx.conf.template @@ -1,7 +1,7 @@ -worker_processes 1; - -error_log /var/log/nginx/error.log warn; -pid /app/nginx.pid; +user www-data; +worker_processes {{NGINX_WORKERS}}; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; events { worker_connections 1024; @@ -16,65 +16,19 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - + tcp_nopush on; + tcp_nodelay on; keepalive_timeout 65; + types_hash_max_size 2048; - upstream healthcheck_hosts { - server ${HOSTNAME_EMAIL_RECEIVE}; - server ${HOSTNAME_CLIENT_METRICS}; - server ${HOSTNAME_CLIENT_WRITE}; - server ${HOSTNAME_CLIENT_READ}; - server ${HOSTNAME_CLIENT_REGISTER}; - } - - server { - listen ${PORT}; - - resolver ${DNS_RESOLVER}; - - client_max_body_size 50M; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; - location = /favicon.ico { - root /static; - } - - location = /robots.txt { - root /static; - } - - location /healthcheck { - proxy_pass http://healthcheck_hosts; - } - - location /web { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://${HOSTNAME_WEBAPP}; - } - - location /api/email/sendgrid { - proxy_pass http://${HOSTNAME_EMAIL_RECEIVE}; - } - - location /api/email/metrics { - proxy_pass http://${HOSTNAME_CLIENT_METRICS}; - } - - location /api/email/upload { - proxy_pass http://${HOSTNAME_CLIENT_WRITE}; - } + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; - location /api/email/download { - proxy_pass http://${HOSTNAME_CLIENT_READ}; - } + gzip on; - location /api/email/register { - proxy_pass http://${HOSTNAME_CLIENT_REGISTER}; - } - } + include /etc/nginx/sites-enabled/*; } diff --git a/docker/nginx/run-nginx.sh b/docker/nginx/run-nginx.sh index de8c5155..13de7094 100755 --- a/docker/nginx/run-nginx.sh +++ b/docker/nginx/run-nginx.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -envsubst < /app/nginx.conf.template > /app/nginx.conf "$(env | sed -e 's/=.*//' -e 's/^/\$/g')" +mo < /app/nginx.conf.template > /app/nginx.conf +mo < /app/server.conf.template > /etc/nginx/sites-enabled/server.conf nginx -c "/app/nginx.conf" -p "${PWD}" -g "daemon off;" diff --git a/docker/nginx/server.conf.template b/docker/nginx/server.conf.template new file mode 100644 index 00000000..277bcea3 --- /dev/null +++ b/docker/nginx/server.conf.template @@ -0,0 +1,68 @@ +upstream healthcheck_hosts { + server {{HOSTNAME_EMAIL_RECEIVE}}; + server {{HOSTNAME_CLIENT_METRICS}}; + server {{HOSTNAME_CLIENT_WRITE}}; + server {{HOSTNAME_CLIENT_READ}}; + server {{HOSTNAME_CLIENT_REGISTER}}; +} + +server { + listen {{PORT}}; + + {{#LETSENCRYPT_DOMAIN}} + server_name {{LETSENCRYPT_DOMAIN}}; + listen [::]:443 ssl ipv6only=on; # managed by Certbot + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/mailserver.lokole.ca/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/mailserver.lokole.ca/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + if ($scheme != "https") { return 301 https://$host$request_uri; } # managed by Certbot + {{/LETSENCRYPT_DOMAIN}} + + {{#DNS_RESOLVER}} + resolver {{DNS_RESOLVER}}; + {{/DNS_RESOLVER}} + + client_max_body_size 50M; + + location = /favicon.ico { + root {{STATIC_ROOT}}/static; + } + + location = /robots.txt { + root {{STATIC_ROOT}}/static; + } + + location /healthcheck { + proxy_pass http://healthcheck_hosts; + } + + location /web { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://{{HOSTNAME_WEBAPP}}; + } + + location /api/email/sendgrid { + proxy_pass http://{{HOSTNAME_EMAIL_RECEIVE}}; + } + + location /api/email/metrics { + proxy_pass http://{{HOSTNAME_CLIENT_METRICS}}; + } + + location /api/email/upload { + proxy_pass http://{{HOSTNAME_CLIENT_WRITE}}; + } + + location /api/email/download { + proxy_pass http://{{HOSTNAME_CLIENT_READ}}; + } + + location /api/email/register { + proxy_pass http://{{HOSTNAME_CLIENT_REGISTER}}; + } +} diff --git a/docker/setup/Dockerfile b/docker/setup/Dockerfile index baf0117d..ac8cd7dd 100644 --- a/docker/setup/Dockerfile +++ b/docker/setup/Dockerfile @@ -8,6 +8,7 @@ ENV NGINX_INGRESS_VERSION="0.3.7" RUN apk add -q --no-cache \ jq=1.5-r2 \ + sshpass=1.05-r0 \ curl=7.59.0-r0 && \ curl -sLfO "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \ tar xf "helm-v${HELM_VERSION}-linux-amd64.tar.gz" && \ diff --git a/docker/setup/upgrade-helm.sh b/docker/setup/upgrade-helm.sh new file mode 100755 index 00000000..a993c8da --- /dev/null +++ b/docker/setup/upgrade-helm.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +## +## This script upgrades an existing production deployment. +## The script assumes that a kubernetes secret exists at /secrets/kube-config. +## +## Required environment variables: +## +## DOCKER_TAG +## HELM_NAME +## IMAGE_REGISTRY +## LOKOLE_DNS_NAME +## + +scriptdir="$(dirname "$0")" +scriptname="${BASH_SOURCE[0]}" +# shellcheck disable=SC1090 +. "${scriptdir}/utils.sh" + +# +# verify inputs +# + +required_env "${scriptname}" "DOCKER_TAG" +required_env "${scriptname}" "HELM_NAME" +required_env "${scriptname}" "IMAGE_REGISTRY" +required_env "${scriptname}" "LOKOLE_DNS_NAME" +required_file "${scriptname}" "/secrets/kube-config" + +# +# upgrade production deployment +# + +log "Upgrading helm deployment ${HELM_NAME}" + +export KUBECONFIG="/secrets/kube-config" + +helm_init + +helm upgrade "${HELM_NAME}" \ + --set domain="${LOKOLE_DNS_NAME}" \ + --set version.imageRegistry="${IMAGE_REGISTRY}" \ + --set version.dockerTag="${DOCKER_TAG}" \ + "${scriptdir}/helm/opwen_cloudserver" diff --git a/docker/setup/upgrade-vm.sh b/docker/setup/upgrade-vm.sh new file mode 100755 index 00000000..809f2cb8 --- /dev/null +++ b/docker/setup/upgrade-vm.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +## +## This script upgrades an production VM. +## +## Required environment variables: +## +## LOKOLE_VM_PASSWORD +## LOKOLE_DNS_NAME +## + +scriptdir="$(dirname "$0")" +scriptname="${BASH_SOURCE[0]}" +# shellcheck disable=SC1090 +. "${scriptdir}/utils.sh" + +# +# verify inputs +# + +required_env "${scriptname}" "LOKOLE_VM_PASSWORD" +required_env "${scriptname}" "LOKOLE_DNS_NAME" + +# +# upgrade production deployment +# + +log "Upgrading VM ${LOKOLE_DNS_NAME}" + +exec sshpass -p "${LOKOLE_VM_PASSWORD}" ssh -o StrictHostKeyChecking=no "opwen@${LOKOLE_DNS_NAME}" < "${scriptdir}/vm.sh" diff --git a/docker/setup/upgrade.sh b/docker/setup/upgrade.sh index a993c8da..3c26520e 100755 --- a/docker/setup/upgrade.sh +++ b/docker/setup/upgrade.sh @@ -1,43 +1,10 @@ #!/usr/bin/env bash -## -## This script upgrades an existing production deployment. -## The script assumes that a kubernetes secret exists at /secrets/kube-config. -## -## Required environment variables: -## -## DOCKER_TAG -## HELM_NAME -## IMAGE_REGISTRY -## LOKOLE_DNS_NAME -## scriptdir="$(dirname "$0")" -scriptname="${BASH_SOURCE[0]}" -# shellcheck disable=SC1090 -. "${scriptdir}/utils.sh" -# -# verify inputs -# - -required_env "${scriptname}" "DOCKER_TAG" -required_env "${scriptname}" "HELM_NAME" -required_env "${scriptname}" "IMAGE_REGISTRY" -required_env "${scriptname}" "LOKOLE_DNS_NAME" -required_file "${scriptname}" "/secrets/kube-config" - -# -# upgrade production deployment -# - -log "Upgrading helm deployment ${HELM_NAME}" - -export KUBECONFIG="/secrets/kube-config" - -helm_init - -helm upgrade "${HELM_NAME}" \ - --set domain="${LOKOLE_DNS_NAME}" \ - --set version.imageRegistry="${IMAGE_REGISTRY}" \ - --set version.dockerTag="${DOCKER_TAG}" \ - "${scriptdir}/helm/opwen_cloudserver" +if [[ "$1" = "vm" ]]; then + shift + exec "${scriptdir}/upgrade-vm.sh" "$@" +else + exec "${scriptdir}/upgrade-helm.sh" "$@" +fi diff --git a/docker/setup/vm.sh b/docker/setup/vm.sh new file mode 100755 index 00000000..1eca127f --- /dev/null +++ b/docker/setup/vm.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +if [[ "$1" != "install" ]]; then + cd /home/opwen/opwen-cloudserver || exit 99 + git pull || exit 1 + cd - || exit 99 + docker-compose -f /home/opwen/opwen-cloudserver/docker/docker-compose.prod.yml pull || exit 2 + docker-compose -f /home/opwen/opwen-cloudserver/docker/docker-compose.prod.yml up -d || exit 3 + exit 0 +fi + +# +# note: the rest of this script is not intended to be run automatically +# the following steps are merely provided for illustrative purposes for +# setting up a virtual machine to run the system: all steps should be +# run interactively and verified thoroughly +# +hostname="mailserver.lokole.ca" +contact_email="ascoderu.opwen@gmail.com" + +# +# set up system +# +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y git curl fail2ban +sudo tee /etc/fail2ban/jail.conf << EOM +[DEFAULT] +ignoreip = 127.0.0.1 +bantime = 300 +findtime = 60 +maxretry = 3 +[sshd] +enabled = true +EOM +sudo service fail2ban restart + +# +# set up docker +# +curl -fsSL https://get.docker.com | sudo bash +sudo usermod -aG docker opwen +sudo curl -fsSL "https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose + +# +# set up letsencrypt +# +sudo add-apt-repository -y ppa:certbot/certbot +sudo apt-get update +sudo apt-get install -y nginx python-certbot-nginx +sudo sed -i "s/server_name _/server_name ${hostname}/" /etc/nginx/sites-available/default +sudo systemctl reload nginx +sudo certbot --nginx -d "${hostname}" --agree-tos -m "${contact_email}" +echo "0 0 1,15 * * certbot renew 2>&1 | /usr/bin/logger -t update_letsencrypt_renewal" | sudo crontab +sudo chmod 0755 /etc/letsencrypt/{live,archive} + +# +# set up app +# important: remember to scp the secrets to the vm manually +# +git clone https://github.com/ascoderu/opwen-cloudserver.git +docker-compose -f opwen-cloudserver/docker/docker-compose.prod.yml pull +docker-compose -f opwen-cloudserver/docker/docker-compose.prod.yml up -d + +# +# set up nginx +# +cat > opwen-cloudserver/secrets/nginx.env << EOM +NGINX_WORKERS=auto +HOSTNAME_WEBAPP=localhost:8080 +HOSTNAME_EMAIL_RECEIVE=localhost:8888 +HOSTNAME_CLIENT_METRICS=localhost:8888 +HOSTNAME_CLIENT_WRITE=localhost:8888 +HOSTNAME_CLIENT_READ=localhost:8888 +HOSTNAME_CLIENT_REGISTER=localhost:8888 +PORT=80 +STATIC_ROOT=/home/opwen/opwen-cloudserver/docker/nginx +LETSENCRYPT_DOMAIN=${hostname} +EOM +docker pull ascoderu/opwenserver_nginx +docker run --env-file opwen-cloudserver/secrets/nginx.env --rm ascoderu/opwenserver_nginx sh -c 'mo < /app/nginx.conf.template' | sudo tee /etc/nginx/nginx.conf +docker run --env-file opwen-cloudserver/secrets/nginx.env --rm ascoderu/opwenserver_nginx sh -c 'mo < /app/server.conf.template' | sudo tee /etc/nginx/sites-available/default +sudo systemctl reload nginx diff --git a/makefile b/makefile index 11b41eab..f5c493d2 100644 --- a/makefile +++ b/makefile @@ -74,7 +74,6 @@ build: docker-compose \ -f docker-compose.yml \ -f docker/docker-compose.dev.yml \ - -f docker/docker-compose.secrets.yml \ -f docker/docker-compose.test.yml \ -f docker/docker-compose.tools.yml \ build @@ -86,9 +85,6 @@ start: docker-compose -f docker-compose.yml -f docker/docker-compose.dev.yml up -d --remove-orphans; \ fi -start-azure: - docker-compose -f docker-compose.yml -f docker/docker-compose.secrets.yml up -d --remove-orphans - start-devtools: docker-compose -f docker-compose.yml -f docker/docker-compose.tools.yml up -d --remove-orphans @@ -107,7 +103,6 @@ stop: docker-compose \ -f docker-compose.yml \ -f docker/docker-compose.dev.yml \ - -f docker/docker-compose.secrets.yml \ -f docker/docker-compose.test.yml \ -f docker/docker-compose.tools.yml \ down --volumes --timeout=5 @@ -142,7 +137,7 @@ kubeconfig: curl -sSfL "$(KUBECONFIG_URL)" -o "$(PWD)/kube-config"; \ fi -renew-cert: kubeconfig +renew-cert-k8s: kubeconfig docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml build setup && \ docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml run --rm \ -v "$(PWD)/kube-config:/secrets/kube-config" \ @@ -150,7 +145,7 @@ renew-cert: kubeconfig /app/renew-cert.sh && \ rm -f "$(PWD)/kube-config" -deploy: kubeconfig +deploy-k8s: kubeconfig docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml build setup && \ docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml run --rm \ -e IMAGE_REGISTRY="$(DOCKER_USERNAME)" \ @@ -161,3 +156,14 @@ deploy: kubeconfig setup \ /app/upgrade.sh && \ rm -f "$(PWD)/kube-config" + +renew-cert: + echo "Skipping: handled by cron on the VM" + +deploy: + docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml build setup && \ + docker-compose -f docker-compose.yml -f docker/docker-compose.setup.yml run --rm \ + -e LOKOLE_VM_PASSWORD="$(LOKOLE_VM_PASSWORD)" \ + -e LOKOLE_DNS_NAME="$(LOKOLE_DNS_NAME)" \ + setup \ + /app/upgrade.sh vm