Update local development to run all tasks in containers (cabotage#81)

* restructure dependencies, add make target to pip-compile

* move reformat/lint into container for local development

* remove extraneous file

* remove vault database secrets engine in dev

this isn't used in practice

.dockerignore

@@ -1,6 +1,6 @@
 /.git
-/.tox
 /.ruff_cache
+/dev
 __pycache__
 .swp
 buildanddeploy

.github/workflows/ci.yml

@@ -8,7 +8,7 @@ jobs:
       matrix:
         include:
           - name: Lint
-            command: make lint
+            command: bin/lint
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -18,7 +18,6 @@ jobs:
         python-version: 3.11
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install tox
+        python -m pip install -r requirements/dev.txt
     - name: Run ${{ matrix.name }}
       run: ${{ matrix.command }}

.gitignore

@@ -1,6 +1,6 @@
 *.swp
 __pycache__
 .DS_Store
-.tox
 /buildanddeploy
 /celerybeat-schedule
+/dev

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.11-slim-bullseye
 
+ARG DEVEL=no
+
 # By default, Docker has special steps to avoid keeping APT caches in the layers, which
 # is good, but in our case, we're going to mount a special cache volume (kept between
 # builds), so we WANT the cache to persist.
@@ -27,12 +29,12 @@ ENV PATH="/opt/cabotage-app/bin:${PATH}"
 
 RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
 
-WORKDIR /opt/cabotage-app/src/
+COPY requirements /tmp/requirements
 
-COPY requirements.txt requirements.txt
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install -r requirements.txt
+    pip install -r /tmp/requirements/base.txt \
+    $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/dev.txt'; fi)
 
 COPY . /opt/cabotage-app/src/
+WORKDIR /opt/cabotage-app/src/
 
-USER nobody

Makefile

@@ -14,13 +14,6 @@ default:
 	@echo
 	@exit 1
 
-lint:
-	tox -e black
-	tox -e ruff
-
-reformat:
-	tox -e reformat
-
 start:
 	docker-compose up --build --detach
 
@@ -40,3 +33,12 @@ create-admin:
 
 routes:
 	docker-compose exec cabotage-app python3 -m flask routes
+
+requirements/%.txt: requirements/%.in
+	docker compose run --build --rm base pip-compile --generate-hashes --output-file=$@ $(F) $<
+
+reformat:
+	docker compose run --build --rm base black .
+
+lint:
+	docker compose run --build --rm base bin/lint

README.md

@@ -58,6 +58,16 @@ base application image. For example, changing the dependencies in the
 after making such changes to rebuild images and restart containers.
 
 
+## `make requirements/{base,dev}.txt`
+
+Recompiles dependency files.
+Additional arguments can be passed to pip-compile with `F=`:
+
+```shell
+make requirements/base.txt F='--upgrade-package flask'
+```
+
+
 ### `make stop`
 
 Stops all running containers, but keeps any volumes containing database files,

bin/lint

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+black --check --diff .
+ruff check .

cabotage/scripts/create_admin.py

@@ -10,7 +10,14 @@
     exit(1)
 
 with app.app_context():
-    user = User(email="ad@min.com", password="admin", username="admin", admin=True, active=True, fs_uniquifier="admin")
+    user = User(
+        email="ad@min.com",
+        password="admin",
+        username="admin",
+        admin=True,
+        active=True,
+        fs_uniquifier="admin",
+    )
     db.session.add(user)
     db.session.flush()
 

docker-compose.yml

@@ -94,62 +94,57 @@ services:
       - "9001:9001"
     volumes:
       - miniodata:/export
-  cabotage-app:
-    build: .
-    image: cabotage-app-web:docker-compose
-    command: hupper -m gunicorn.app.wsgiapp -c gunicorn.conf -w 4 --threads 100 -b 0.0.0.0:8000 cabotage.server.wsgi:app
-    environment:
-      - FLASK_APP=cabotage.server.wsgi
-      - CABOTAGE_DEBUG=True
-      - CABOTAGE_TESTING=True
+  base:
+    build:
+      context: .
+      args:
+        DEVEL: yes
+    image: cabotage-app:docker-compose
+    environment: &base_environment
+      # Runs as `nobody`, override the home directory
+      - PIP_CACHE_DIR=/opt/cabotage-app/src/dev/.pip-cache
+      - PIP_TOOLS_CACHE_DIR=/opt/cabotage-app/src/dev/.pip-tools-cache
+      # Application configuration
       - CABOTAGE_BCRYPT_LOG_ROUNDS=4
-      - CABOTAGE_WTF_CSRF_ENABLED=False
-      - CABOTAGE_VAULT_DB_CREDS_PATH=database/creds/cabotage
-      - CABOTAGE_VAULT_DB_DATABASE_URI=postgresql://postgres@db/cabotage_dev
-      - CABOTAGE_VAULT_TOKEN=deadbeef-dead-beef-dead-beefdeadbeef
+      - CABOTAGE_DEBUG=True
       - CABOTAGE_DEBUG_TB_ENABLED=True
-      - CABOTAGE_SECURITY_CONFIRMABLE=False
-      - CABOTAGE_GITHUB_APP_URL=https://github.com/apps/cabotage-local
       - CABOTAGE_GITHUB_APP_ID=
       - CABOTAGE_GITHUB_APP_PRIVATE_KEY=
+      - CABOTAGE_GITHUB_APP_URL=https://github.com/apps/cabotage-local
       - CABOTAGE_GITHUB_WEBHOOK_SECRET=
-      - KUBECONFIG=/var/run/kube/config
-      - CABOTAGE_KUBERNETES_ENABLED=False
       - CABOTAGE_KUBERNETES_CONTEXT=minikube
+      - CABOTAGE_KUBERNETES_ENABLED=False
+      - CABOTAGE_SECURITY_CONFIRMABLE=False
+      - CABOTAGE_SQLALCHEMY_DATABASE_URI=postgresql://postgres@db/cabotage_dev
+      - CABOTAGE_TESTING=True
+      - CABOTAGE_VAULT_TOKEN=deadbeef-dead-beef-dead-beefdeadbeef
+      - CABOTAGE_WTF_CSRF_ENABLED=False
+      - C_FORCE_ROOT=1
+      - FLASK_APP=cabotage.server.wsgi
+      - KUBECONFIG=/var/run/kube/config
     volumes:
-      - .:/opt/cabotage-app/src
+      - .:/opt/cabotage-app/src:z
+
+  cabotage-app:
+    image: cabotage-app:docker-compose
+    command: hupper -m gunicorn.app.wsgiapp -c gunicorn.conf -w 4 --threads 100 -b 0.0.0.0:8000 cabotage.server.wsgi:app
+    environment: *base_environment
+    volumes:
+      - .:/opt/cabotage-app/src:z
       - $HOME/.kube:/var/run/kube
     ports:
       - "8000:8000"
     links:
       - db
       - redis
       - vault
+
   cabotage-app-worker:
-    image: cabotage-app-web:docker-compose
+    image: cabotage-app:docker-compose
     command: hupper -m celery -A cabotage.celery.worker.celery_app worker -E --loglevel=INFO
-    environment:
-      - C_FORCE_ROOT=1
-      - FLASK_APP=cabotage.server.wsgi
-      - CABOTAGE_DEBUG=True
-      - CABOTAGE_TESTING=True
-      - CABOTAGE_BCRYPT_LOG_ROUNDS=4
-      - CABOTAGE_WTF_CSRF_ENABLED=False
-      - CABOTAGE_SQLALCHEMY_DATABASE_URI=postgresql://postgres@db/cabotage_dev
-      - CABOTAGE_VAULT_DB_CREDS_PATH=database/creds/cabotage
-      - CABOTAGE_VAULT_DB_DATABASE_URI=postgresql://postgres@db/cabotage_dev
-      - CABOTAGE_VAULT_TOKEN=deadbeef-dead-beef-dead-beefdeadbeef
-      - CABOTAGE_DEBUG_TB_ENABLED=True
-      - CABOTAGE_SECURITY_CONFIRMABLE=False
-      - CABOTAGE_GITHUB_APP_URL=https://github.com/apps/cabotage-local
-      - CABOTAGE_GITHUB_APP_ID=
-      - CABOTAGE_GITHUB_APP_PRIVATE_KEY=
-      - CABOTAGE_GITHUB_WEBHOOK_SECRET=
-      - KUBECONFIG=/var/run/kube/config
-      - CABOTAGE_KUBERNETES_ENABLED=False
-      - CABOTAGE_KUBERNETES_CONTEXT=minikube
+    environment: *base_environment
     volumes:
-      - .:/opt/cabotage-app/src
+      - .:/opt/cabotage-app/src:z
       - $HOME/.kube:/var/run/kube
     links:
       - db

docker-compose/vault/entry.sh

@@ -2,8 +2,10 @@
 
 export VAULT_ADDR=http://127.0.0.1:8200
 if [ -f /vault/file/unseal ]; then
+
     echo "starting vault!"
     vault server -dev -dev-skip-init -dev-listen-address=$VAULT_DEV_LISTEN_ADDRESS -dev-root-token-id=$VAULT_DEV_ROOT_TOKEN_ID -config /etc/vault/config.hcl &
+
     echo "unsealing!"
     while true; do
         vault status 2>&1 >/dev/null
@@ -17,7 +19,9 @@ if [ -f /vault/file/unseal ]; then
     export UNSEAL_TOKEN=`cat /vault/file/unseal | tr -d '[:space:]'`
     vault operator unseal "$UNSEAL_TOKEN"
     wait
+
 else
+
     echo "starting vault!"
     vault server -dev -dev-listen-address=$VAULT_DEV_LISTEN_ADDRESS -dev-root-token-id=$VAULT_DEV_ROOT_TOKEN_ID -config /etc/vault/config.hcl 2>&1 | tee $HOME/logfile &
     while true; do
@@ -29,15 +33,14 @@ else
         echo "vault not up and initialized yet..."
         sleep .5
     done
+
     echo "bootstrapping cabotage secret storage"
     VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault secrets enable -path=cabotage-secrets -version=1 kv
     echo "bootstrapping our transit key"
     VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault secrets enable transit
     VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault write transit/restore/cabotage-app backup=`cat /etc/vault/cabotage-vault-key.backup`
-    echo "bootstrapping postgres stufffff"
-    VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault secrets enable database
-    VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault write database/config/cabotage plugin_name=postgresql-database-plugin allowed_roles="cabotage" connection_url="postgresql://postgres@db/cabotage_dev?sslmode=disable" verify_connection=false
-    VAULT_TOKEN=$VAULT_DEV_ROOT_TOKEN_ID vault write database/roles/cabotage db_name=cabotage default_ttl="60s" max_ttl="120s" creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}' IN ROLE cabotage;" revocation_statements="REASSIGN OWNED BY \"{{name}}\" TO cabotage" renew_statements="ALTER ROLE \"{{name}}\" VALID UNTIL '{{expiration}}';"
+
+    echo "storing vault unseal token"
     cat $HOME/logfile | echo -n `grep 'Unseal Key: ' | awk '{print $NF}' | sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"` > /vault/file/unseal
     wait
 fi