WIP

.github/workflows/ci.yaml

@@ -16,9 +16,9 @@ jobs:
     name: Python Unit Test with Postgres
     uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
     with:
-       python-version: '3.9'
-       test-script: 'tests/ci_commands_script.sh'
-       run-coveralls: true
+      python-version: '3.9'
+      test-script: 'tests/ci_commands_script.sh'
+      run-coveralls: true
 
   BuildImageAndPush:
     name: Build Image and Push

DockerfileMcrypt

@@ -1 +1,67 @@
-# old Dockerfile
+# old Dockerfile -- specifically for the use of Mcrypt, since Al2 image does not support Mcrypt (which is used in usersyncjob)
+# To run: docker run --rm -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
+# To check running container do: docker exec -it fence /bin/bash
+
+FROM quay.io/cdis/python:python3.9-buster-2.0.0
+
+ENV appname=fence
+WORKDIR /$appname
+
+
+RUN pip install --upgrade pip
+RUN pip install --upgrade poetry
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl bash git \
+    && apt-get -y install vim \
+    libmcrypt4 mcrypt \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/
+
+RUN mkdir -p /var/www/$appname \
+    && mkdir -p /var/www/.cache/Python-Eggs/ \
+    && mkdir /run/nginx/ \
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && chown nginx -R /var/www/.cache/Python-Eggs/ \
+    && chown nginx /var/www/$appname
+
+# aws cli v2 - needed for storing files in s3 during usersync k8s job
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && /bin/rm -rf awscliv2.zip ./aws
+
+
+# copy ONLY poetry artifact, install the dependencies but not fence
+# this will make sure than the dependencies is cached
+COPY poetry.lock pyproject.toml /$appname/
+RUN pwd
+
+RUN sed -i 's/psycopg2-binary = "<3"/psycopg2 = "<3"/g' /$appname/pyproject.toml
+
+RUN poetry lock -vv --no-update \
+    && poetry config virtualenvs.create false \
+    && poetry install -vv --no-root --no-dev --no-interaction \
+    && poetry show -v
+
+# copy source code ONLY after installing dependencies
+COPY . /$appname
+RUN sed -i 's/psycopg2-binary = "<3"/psycopg2 = "<3"/g' /$appname/pyproject.toml
+
+# Changing the ownership of gunicorn to root for this dockerfile to run
+RUN sed -i 's/gen3/root/g' /$appname/deployment/wsgi/gunicorn.conf.py
+
+COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
+COPY ./deployment/wsgi/wsgi.py /$appname/wsgi.py
+COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc
+
+# install fence
+RUN poetry config virtualenvs.create false \
+    && poetry lock -vv --no-update \
+    && poetry install -vv --no-dev --no-interaction \
+    && poetry show -v
+
+RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
+
+CMD ["poetry", "run", "gunicorn", "-c", "deployment/wsgi/gunicorn.conf.py"]

deployment/uwsgi/uwsgi.ini

@@ -0,0 +1,37 @@
+[uwsgi]
+protocol = uwsgi
+socket = /var/run/gen3/uwsgi.sock
+buffer-size = 32768
+uid = nginx
+gid = nginx
+chown-socket = nginx:nginx
+chmod-socket = 666
+master = true
+harakiri-verbose = true
+# No global HARAKIRI, using only user HARAKIRI, because export overwrites it
+# Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD
+# harakiri = 45
+http-timeout = 45
+socket-timeout = 45
+worker-reload-mercy = 45
+reload-mercy = 45
+mule-reload-mercy = 45
+disable-logging = true
+wsgi-file=/fence/wsgi.py
+plugins = python3
+vacuum = true
+pythonpath = /var/www/fence/
+pythonpath = /fence/
+# poetry installs git dependencies at /usr/local/src
+pythonpath = /usr/local/src/*
+
+# metrics setup
+stats = 127.0.0.1:9191
+stats-http = true
+env = prometheus_multiproc_dir=/var/tmp/uwsgi_flask_metrics
+exec-asap = /fence/clear_prometheus_multiproc /var/tmp/uwsgi_flask_metrics
+
+# Initialize application in worker processes, not master. This prevents the
+# workers from all trying to open the same database connections at startup.
+lazy = true
+lazy-apps = true

deployment/wsgi/gunicorn.conf.py

@@ -2,8 +2,8 @@
 bind = "0.0.0.0:8000"
 workers = 1
 preload_app = True
-user = "gen3"
-group = "gen3"
+user = "root"
+group = "root"
 timeout = 300
 keepalive = 2
 keepalive_timeout = 5

dockerrun.bash

@@ -1,10 +1,5 @@
 #!/bin/bash
 
-#
-# Update certificate authority index -
-# environment may have mounted more authorities
-#
-# update-ca-certificates
 #
 # Kubernetes may mount jwt-keys as a tar ball
 #
@@ -15,9 +10,9 @@ if [ -f /fence/jwt-keys.tar ]; then
     if [ -d jwt-keys ]; then
       mkdir -p keys
       mv jwt-keys/* keys/
-      rm -rf /fence/keys/key/
     fi
   )
 fi
+
 nginx
 gunicorn -c /fence/deployment/wsgi/gunicorn.conf.py