From c6ca50e19a72d0758d22e1913415d79188ac6679 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Thu, 12 Oct 2023 20:06:17 -0700 Subject: [PATCH] feat: add Docker compose, use in Makefile, try in CI --- .github/workflows/ci.yaml | 45 +++--------- dev/local/Makefile | 132 +++++++--------------------------- dev/local/docker-compose.yaml | 90 +++++++++++++++++++++++ dev/local/install.sh | 9 ++- 4 files changed, 127 insertions(+), 149 deletions(-) create mode 100644 dev/local/docker-compose.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd2d5fb4b..17055c677 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,7 @@ jobs: build: runs-on: ubuntu-latest steps: + # This imitates the dev/docker/local/install.sh script. - run: mkdir -p repos/undefx - name: Checkout undefx/py3tester uses: actions/checkout@v2 @@ -18,7 +19,6 @@ jobs: path: repos/undefx/undef-analysis - run: mkdir -p repos/delphi - - name: Checkoutcmu-delphi/operations uses: actions/checkout@v2 with: @@ -49,47 +49,20 @@ jobs: repository: cmu-delphi/nowcast path: repos/delphi/nowcast - - name: Build docker images - run: | - docker build -t delphi_database_epidata -f ./repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . - docker build -t delphi_web_python -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . - sudo docker build -t delphi_redis -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile . - cd ./repos/delphi/delphi-epidata - docker build -t delphi_web_epidata -f ./devops/Dockerfile . - cd ../../../ - - # MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run. - # see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name - - name: Start database and Redis services - run: | - docker network create --driver bridge delphi-net - docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata - docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis - - - run: | - wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for - chmod +x wait-for - ./wait-for localhost:13306 -- echo 'ready' - sleep 10s - - - name: Start delphi_web_epidata - run: | - docker run --rm -d -p 10080:80 --env "MODULE_NAME=delphi.epidata.server.main" --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "REDIS_HOST=delphi_redis" --env "REDIS_PASSWORD=1234" --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" --env "API_KEY_ADMIN_PASSWORD=test_admin_password" --env "TESTING_MODE=True" --network delphi-net --name delphi_web_epidata delphi_web_epidata - docker ps - - - name: Run Unit Tests - run: | - docker run --rm --network delphi-net --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/tests + ln -s repos/delphi/delphi-epidata/dev/local/Makefile + ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml + ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore + ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/ + ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/ - - name: Run Integration Tests + - name: Run tests run: | - docker run --rm --network delphi-net delphi_web_python python -m pytest --import-mode importlib repos/delphi/delphi-epidata/integrations + make test - name: Clean Up run: | - docker stop delphi_database_epidata delphi_web_epidata delphi_redis - docker network remove delphi-net + docker compose down build_js_client: runs-on: ubuntu-latest diff --git a/dev/local/Makefile b/dev/local/Makefile index fb02668ee..f473ecdb2 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -25,7 +25,7 @@ # # all: Runs the commands 'web' 'db' and 'python'. # -# test: Runs test and integrations in delphi-epidata. If test +# test: Runs test and integrations in delphi-epidata. If test # optional arg is provided, then only the tests in that subdir # are run. # @@ -65,9 +65,6 @@ NOW:=$(shell date "+%Y-%m-%d") LOG_WEB:=delphi_web_epidata_$(NOW).log LOG_DB:=delphi_database_epidata_$(NOW).log LOG_REDIS:=delphi_redis_instance_$(NOW).log -WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata') -DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata') -REDIS_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_redis') M1= ifeq ($(shell uname -smp), Darwin arm64 arm) @@ -75,130 +72,49 @@ $(info M1 system detected, changing docker platform to linux/amd64.) override M1 =--platform linux/amd64 endif -.PHONY=web -web: - @# Stop container if running - @if [ $(WEB_CONTAINER_ID) ]; then\ - docker stop $(WEB_CONTAINER_ID);\ - fi - - @# Setup virtual network if it doesn't exist - @docker network ls | grep delphi-net || docker network create --driver bridge delphi-net - - @# Build the web_epidata image - @cd repos/delphi/delphi-epidata;\ - docker build -t delphi_web_epidata\ - $(M1) \ - -f ./devops/Dockerfile .;\ - cd - - - @# Run the web server - @# MODULE_NAME specifies the location of the `app` variable, the actual WSGI application object to run. - @# see https://github.com/tiangolo/meinheld-gunicorn-docker#module_name - @docker run --rm -p 127.0.0.1:10080:80 \ - $(M1) \ - --env "MODULE_NAME=delphi.epidata.server.main" \ - --env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \ - --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --env "LOG_DEBUG" \ - --env "REDIS_HOST=delphi_redis" \ - --env "REDIS_PASSWORD=1234" \ - --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ - --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ - --env "TESTING_MODE=True" \ - --network delphi-net --name delphi_web_epidata \ - delphi_web_epidata >$(LOG_WEB) 2>&1 & - .PHONY=db db: - @# Stop container if running - @if [ $(DATABASE_CONTAINER_ID) ]; then\ - docker stop $(DATABASE_CONTAINER_ID);\ - fi - - @# Setup virtual network if it doesn't exist - @docker network ls | grep delphi-net || docker network create --driver bridge delphi-net - - @# Build the database_epidata image - @docker build -t delphi_database_epidata \ - $(M1) \ - -f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile . - - @# Run the database - @docker run --rm -p 127.0.0.1:13306:3306 \ - $(M1) \ - --network delphi-net --name delphi_database_epidata \ - --cap-add=sys_nice \ - delphi_database_epidata >$(LOG_DB) 2>&1 & - - @# Block until DB is ready - @while true; do \ - sed -n '/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \ - tail -1 $(LOG_DB); \ - sleep 1; \ - done - -.PHONY=py -py: - @docker build -t delphi_web_python \ - $(M1) \ - -f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile . + @docker compose up delphi_database_epidata $(M1) --detach + @docker logs -f delphi_database_epidata >$(LOG_DB) 2>&1 & + +.PHONY=web +web: + @SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose up delphi_web_epidata $(M1) --detach + @docker logs -f delphi_web_epidata >$(LOG_WEB) 2>&1 & .PHONY=redis redis: - @# Stop container if running - @if [ $(REDIS_CONTAINER_ID) ]; then\ - docker stop $(REDIS_CONTAINER_ID);\ - fi - - @docker build -t delphi_redis \ - $(M1) \ - -f repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile . - - @docker run --rm -d -p 127.0.0.1:6379:6379 \ - $(M1) \ - --network delphi-net \ - --env "REDIS_PASSWORD=1234" \ - --name delphi_redis delphi_redis >$(LOG_REDIS) 2>&1 & - -.PHONY=all -all: db web py redis + @docker compose up delphi_redis $(M1) --detach + @docker logs -f delphi_redis >$(LOG_REDIS) 2>&1 & .PHONY=test test: - @docker run -i --rm --network delphi-net \ - $(M1) \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - --env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \ - --env "FLASK_SECRET=abc" \ - delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log + @SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \ + python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log .PHONY=r-test r-test: - @docker run -i --rm --network delphi-net \ - $(M1) \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - --env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \ - --env "FLASK_SECRET=abc" \ - delphi_web_python Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log - + @SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) \ + Rscript repos/delphi/delphi-epidata/integrations/client/test_delphi_epidata.R | tee r-test_output_$(NOW).log .PHONY=bash bash: - @docker run -it --rm --network delphi-net \ - $(M1) \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \ - --mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \ - --env "SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri)" \ - --env "FLASK_SECRET=abc" \ - delphi_web_python bash + @SQLALCHEMY_DATABASE_URI=$(sqlalchemy_uri) docker compose run delphi_web_python $(M1) bash .PHONY=sql sql: @docker run --rm -it --network delphi-net --cap-add=sys_nice \ percona mysql --user=user --password=pass --port 3306 --host delphi_database_epidata epidata +.PHONY=down +down: + @docker compose down + +.PHONY=down-force-rebuild +down-force-rebuild: + @docker compose down --rmi all --volumes + @docker compose up --build --force-recreate --detach + .PHONY=clean clean: @docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1 diff --git a/dev/local/docker-compose.yaml b/dev/local/docker-compose.yaml new file mode 100644 index 000000000..b8a6be261 --- /dev/null +++ b/dev/local/docker-compose.yaml @@ -0,0 +1,90 @@ +networks: + delphi-net: + driver: bridge + +services: + delphi_database_epidata: + container_name: delphi_database_epidata + build: + context: . + dockerfile: repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile + cap_add: + - SYS_NICE + ports: + - "127.0.0.1:13306:3306" + networks: + - delphi-net + healthcheck: + test: "mysql --user=user --password=pass --execute \"SHOW DATABASES;\"" + start_period: 5s + timeout: 10s + retries: 5 + + delphi_web_epidata: + container_name: delphi_web_epidata + build: + context: repos/delphi/delphi-epidata + dockerfile: ./devops/Dockerfile + environment: + MODULE_NAME: delphi.epidata.server.main + SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata} + FLASK_SECRET: abc + FLASK_PREFIX: /epidata + LOG_DEBUG: "1" + REDIS_HOST: delphi_redis + REDIS_PASSWORD: 1234 + API_KEY_ADMIN_PASSWORD: test_admin_password + API_KEY_REGISTER_WEBHOOK_TOKEN: abc + TESTING_MODE: True + ports: + - "127.0.0.1:10080:80" + networks: + - delphi-net + healthcheck: + test: "curl -f http://localhost/epidata/covidcast || exit 1" + timeout: 10s + retries: 5 + + delphi_redis: + container_name: delphi_redis + build: + context: . + dockerfile: repos/delphi/delphi-epidata/dev/docker/redis/Dockerfile + environment: + REDIS_PASSWORD: 1234 + ports: + - "127.0.0.1:6379:6379" + networks: + - delphi-net + depends_on: + delphi_database_epidata: + condition: service_healthy + delphi_web_epidata: + condition: service_healthy + + delphi_web_python: + container_name: delphi_web_python + build: + context: . + dockerfile: repos/delphi/delphi-epidata/dev/docker/python/Dockerfile + environment: + SQLALCHEMY_DATABASE_URI: ${SQLALCHEMY_DATABASE_URI:-mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata} + FLASK_SECRET: abc + volumes: + - type: bind + source: repos/delphi/delphi-epidata + target: /usr/src/app/delphi/epidata + read_only: true + - type: bind + source: repos/delphi/delphi-epidata/src + target: /usr/src/app/delphi/epidata + read_only: true + networks: + - delphi-net + depends_on: + delphi_database_epidata: + condition: service_healthy + delphi_web_epidata: + condition: service_healthy + delphi_redis: + condition: service_started diff --git a/dev/local/install.sh b/dev/local/install.sh index 9fd262e2e..6634857db 100644 --- a/dev/local/install.sh +++ b/dev/local/install.sh @@ -1,7 +1,7 @@ #!/bin/bash # Bootstrap delphi-epidata development # -# Downloads the repos needed for local delphi-epidata development into current dir +# Downloads the repos needed for local delphi-epidata development into current dir # and provides a Makefile with Docker control commands # as well as pyproject/setup.cfg files for IDE mappings. # @@ -44,9 +44,8 @@ git clone https://github.com/undefx/py3tester git clone https://github.com/undefx/undef-analysis cd ../../ +ln -s repos/delphi/delphi-epidata/dev/local/docker-compose.yaml ln -s repos/delphi/delphi-epidata/dev/local/Makefile ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore -cd repos -ln -s delphi/delphi-epidata/dev/local/pyproject.toml -ln -s delphi/delphi-epidata/dev/local/setup.cfg -cd - +ln -s repos/delphi/delphi-epidata/dev/local/pyproject.toml repos/ +ln -s repos/delphi/delphi-epidata/dev/local/setup.cfg repos/