From ca05e9f2214fcf9897ef5378102afc2ce473f2a7 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Sun, 22 Dec 2019 01:32:10 -0500 Subject: [PATCH] Improve development experience with docker (#267) * Make paths consistent between builder and runtime * Avoid reliance on Dockerfile default ENV values * Fix issue with pickling memoryview in librabbitmq * Use dev environment when running locally * Reload api and worker containers on app changes --- .env | 2 +- .travis.yml | 1 + README.rst | 2 +- docker-compose.yml | 14 +++++++++++-- docker/app/Dockerfile | 29 +++++++++++---------------- docker/docker-entrypoint-dev.sh | 15 ++++++++++++++ helm/templates/api-deployment.yaml | 6 +++++- helm/templates/nginx-deployment.yaml | 2 ++ helm/templates/worker-deployment.yaml | 2 +- requirements-dev.txt | 5 ++++- 10 files changed, 54 insertions(+), 24 deletions(-) create mode 100755 docker/docker-entrypoint-dev.sh diff --git a/.env b/.env index 97713788..92691734 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ APP_PORT=8080 BUILD_TAG=development -BUILD_TARGET=runtime +BUILD_TARGET=builder DOCKER_REPO=ascoderu SERVER_WORKERS=1 QUEUE_WORKERS=1 diff --git a/.travis.yml b/.travis.yml index 79d528fc..24f1f1ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ script: if [[ "$TRAVIS_PULL_REQUEST" = "false" ]] && [[ "$TEST_MODE" = "live" ]]; then echo "Skipping live service test for branch build" else + export BUILD_TARGET=runtime make build verify-build make start make integration-tests diff --git a/README.rst b/README.rst index ba113658..9f7ac298 100644 --- a/README.rst +++ b/README.rst @@ -86,7 +86,7 @@ running the unit tests and other CI steps such as linting: make build -You can now run the application stack: +You can now run the application stack; code changes will be hot reloaded: .. sourcecode :: sh diff --git a/docker-compose.yml b/docker-compose.yml index c1e689fb..da431790 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,9 +59,10 @@ services: api: <<: *shared-app-build - command: ["/app/run-gunicorn.sh"] + command: ["/app/docker/docker-entrypoint-dev.sh", "/app/docker/app/run-gunicorn.sh"] environment: <<: *shared-app-environment + PORT: "8080" CONNEXION_SPEC: dir:/app/opwen_email_server/swagger SERVER_WORKERS: ${SERVER_WORKERS} TESTING_UI: "True" @@ -70,10 +71,13 @@ services: depends_on: - appinsights - azurite + volumes: + - .:/app + - venv:/app/venv worker: <<: *shared-app-build - command: ["/app/run-celery.sh"] + command: ["/app/docker/docker-entrypoint-dev.sh", "/app/docker/app/run-celery.sh"] environment: <<: *shared-app-environment CELERY_QUEUE_NAMES: all @@ -82,6 +86,9 @@ services: - appinsights - azurite - rabbitmq + volumes: + - .:/app + - venv:/app/venv setup: image: ${DOCKER_REPO}/opwenserver_setup:${BUILD_TAG} @@ -133,3 +140,6 @@ services: image: mcr.microsoft.com/azure-storage/azurite:latest ports: - ${AZURITE_PORT}:10000 + +volumes: + venv: diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 6fe87b44..1a63c02a 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -12,7 +12,7 @@ RUN wget -q -O /tmp/shellcheck.tar.xz "https://storage.googleapis.com/shellcheck && rm /tmp/shellcheck.tar.xz \ && shellcheck --version -WORKDIR /src +WORKDIR /app COPY requirements*.txt ./ COPY makefile . @@ -24,6 +24,9 @@ COPY . . ARG LOKOLE_SENDGRID_KEY="" RUN make ci clean +ENV PY_ENV=/app/venv +ENV HOT_RELOAD="True" + FROM python:${PYTHON_VERSION}-slim AS runtime RUN groupadd -r opwen \ @@ -36,19 +39,11 @@ RUN pip --no-cache-dir -q install /deps/*.whl USER opwen WORKDIR /app -COPY --from=builder /src/docker/docker-entrypoint.sh . -COPY --from=builder /src/docker/app/run-celery.sh . -COPY --from=builder /src/docker/app/run-gunicorn.sh . -COPY --from=builder /src/opwen_email_server ./opwen_email_server - -ENV PY_ENV="/usr/local" -ENV TESTING_UI="False" -ENV CONNEXION_SPEC="SET_ME" -ENV SERVER_WORKERS="1" -ENV QUEUE_WORKERS="1" -ENV LOKOLE_LOG_LEVEL="INFO" -ENV PORT=8080 - -EXPOSE ${PORT} -ENTRYPOINT ["/app/docker-entrypoint.sh"] -CMD ["bash"] +COPY --from=builder /app/docker/docker-entrypoint.sh ./docker/docker-entrypoint.sh +COPY --from=builder /app/docker/app/run-celery.sh ./docker/app/run-celery.sh +COPY --from=builder /app/docker/app/run-gunicorn.sh ./docker/app/run-gunicorn.sh +COPY --from=builder /app/opwen_email_server ./opwen_email_server + +ENV PY_ENV=/usr/local + +ENTRYPOINT ["/app/docker/docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint-dev.sh b/docker/docker-entrypoint-dev.sh new file mode 100755 index 00000000..8de13d14 --- /dev/null +++ b/docker/docker-entrypoint-dev.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +if [[ "${HOT_RELOAD}" != "True" ]]; then + exec "$@" +fi + +rootdir="$(readlink -f "$(dirname "$0")/..")" + +exec "${PY_ENV}/bin/watchmedo" \ + auto-restart \ + --debug-force-polling \ + --directory="${rootdir}/opwen_email_server" \ + --pattern='*.py' \ + --recursive \ + -- "$@" diff --git a/helm/templates/api-deployment.yaml b/helm/templates/api-deployment.yaml index f66adbde..1b57533c 100644 --- a/helm/templates/api-deployment.yaml +++ b/helm/templates/api-deployment.yaml @@ -20,12 +20,16 @@ spec: containers: - name: api image: {{.Values.version.imageRegistry}}/opwenserver_app:{{.Values.version.dockerTag}} - command: ["/app/run-gunicorn.sh"] + command: ["/app/docker/app/run-gunicorn.sh"] env: + - name: PORT + value: "8080" - name: CONNEXION_SPEC value: dir:/app/opwen_email_server/swagger - name: SERVER_WORKERS value: "{{.Values.server.serverWorkers}}" + - name: TESTING_UI + value: "False" {{ include "opwen.environment.shared" . }} - name: REGISTRATION_USERNAME valueFrom: diff --git a/helm/templates/nginx-deployment.yaml b/helm/templates/nginx-deployment.yaml index 71bc4363..37b9b169 100644 --- a/helm/templates/nginx-deployment.yaml +++ b/helm/templates/nginx-deployment.yaml @@ -21,6 +21,8 @@ spec: - name: nginx image: {{.Values.version.imageRegistry}}/opwenserver_nginx:{{.Values.version.dockerTag}} env: + - name: PORT + value: "8888" - name: DNS_RESOLVER value: 127.0.0.1:53 ipv6=off - name: HOSTNAME_CLIENT_METRICS diff --git a/helm/templates/worker-deployment.yaml b/helm/templates/worker-deployment.yaml index f19c2468..625fe84d 100644 --- a/helm/templates/worker-deployment.yaml +++ b/helm/templates/worker-deployment.yaml @@ -20,7 +20,7 @@ spec: containers: - name: worker image: {{.Values.version.imageRegistry}}/opwenserver_app:{{.Values.version.dockerTag}} - command: ["/app/run-celery.sh"] + command: ["/app/docker/app/run-celery.sh"] env: - name: CELERY_QUEUE_NAMES value: all diff --git a/requirements-dev.txt b/requirements-dev.txt index c9e12a15..cbc820c6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ flake8==3.7.9 flex==6.14.1 isort==4.3.21 -librabbitmq==2.0.0 +PyAMQP==0.0.8.11 lockfile==0.12.2 mypy==0.740 nose2[coverage_plugin]==0.9.1 @@ -10,3 +10,6 @@ responses==0.10.6 bandit==1.6.2 yamllint==1.18.0 yapf==0.28.0 + +# replace with pypi when watchdog > 0.9.0 is released +https://github.com/gorakhargosh/watchdog/archive/6793588d8ab8e2cd45aa2151b0b0bd7da5dd2617.zip#egg=watchdog[watchmedo]