workflow.yml

name: Testing Workflow
on: [workflow_dispatch,push]

jobs:
  clean:
    strategy:
      matrix:
        package: ['neurosynth_compose', 'compose_nginx', 'compose_pgsql', 'compose_worker', 'neurostore', 'store_nginx', 'store_pgsql']
    runs-on: ubuntu-latest

    steps:
      - uses: actions/delete-package-versions@v5
        with: 
          package-name: ${{ matrix.package }}
          package-type: 'container'
          min-versions-to-keep: 10
          delete-only-pre-release-versions: "true"

  build_neurosynth_compose:
   runs-on: ubuntu-latest
   defaults:
      run:
        working-directory: compose
   steps:
      -
        name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive
      -
        name: Configuration
        run: |
          cp .env.example .env
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Build and push
        uses: docker/bake-action@v6.0.0
        with:
          files: docker-compose.yml,docker-compose.dev.yml
          push: true
          load: false
          workdir: compose
          source: "{{defaultContext}}:compose"
          set: |
              compose.tags=ghcr.io/${{ github.repository_owner }}/neurosynth_compose:${{ hashFiles('**/compose/neurosynth_compose/**') }}
              compose.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/neurosynth_compose:${{ hashFiles('**/compose/neurosynth_compose/**') }}
              compose.cache-from=type=gha,scope=cached-compose
              compose.cache-to=type=gha,scope=cached-compose,mode=max,ignore-error=true
              compose_nginx.tags=ghcr.io/${{ github.repository_owner }}/compose_nginx:${{ hashFiles('**/compose/nginx/**') }}
              compose_nginx.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_nginx:${{ hashFiles('**/compose/nginx/**') }}
              compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
              compose_nginx.cache-to=type=gha,scope=cached-compose-nginx,mode=max,ignore-error=true
              compose_pgsql.tags=ghcr.io/${{ github.repository_owner }}/compose_pgsql:${{ hashFiles('**/compose/postgres/**') }}
              compose_pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_pgsql:${{ hashFiles('**/compose/postgres/**') }}
              compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
              compose_pgsql.cache-to=type=gha,scope=cached-compose-pgsql,mode=max,ignore-error=true
              compose_worker.tags=ghcr.io/${{ github.repository_owner }}/compose_worker:${{ hashFiles('**/compose/neurosynth_compose/**') }}
              compose_worker.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/compose_worker:${{ hashFiles('**/compose/neurosynth_compose/**') }}
              compose_worker.cache-from=type=gha,scope=cached-compose-worker
              compose_worker.cache-to=type=gha,scope=cached-compose-worker,mode=max,ignore-error=true

  build_neurostore:
   runs-on: ubuntu-latest
   defaults:
      run:
        working-directory: store
   steps:
      -
        name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive
      -
        name: Configuration
        run: |
          cp .env.example .env
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      -
        name: Build and push
        uses: docker/bake-action@v6.0.0
        with:
          files: docker-compose.yml,docker-compose.dev.yml
          push: true
          load: false
          workdir: store
          source: "{{defaultContext}}:store"
          set: |
              neurostore.tags=ghcr.io/${{ github.repository_owner }}/neurostore:${{ hashFiles('**/store/neurostore/**') }}
              neurostore.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/neurostore:${{ hashFiles('**/store/neurostore/**') }}
              neurostore.cache-from=type=gha,scope=cached-neurostore
              neurostore.cache-to=type=gha,scope=cached-neurostore,mode=max,ignore-error=true
              store_nginx.tags=ghcr.io/${{ github.repository_owner }}/store_nginx:${{ hashFiles('**/store/nginx/**') }}
              store_nginx.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store_nginx:${{ hashFiles('**/store/nginx/**') }}
              store_nginx.cache-from=type=gha,scope=cached-store-nginx
              store_nginx.cache-to=type=gha,scope=cached-store-nginx,mode=max,ignore-error=true
              store_pgsql.tags=ghcr.io/${{ github.repository_owner }}/store_pgsql:${{ hashFiles('**/store/postgres/**') }}
              store_pgsql.cache-from=type=registry,ref=ghcr.io/${{ github.repository_owner }}/store_pgsql:${{ hashFiles('**/store/postgres/**') }}
              store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
              store_pgsql.cache-to=type=gha,scope=cached-store-pgsql,mode=max,ignore-error=true


  neurostore_backend_tests:
    runs-on: ubuntu-latest
    needs: build_neurostore
    defaults:
      run:
        working-directory: store
    steps:
      - 
        name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Configuration
        run: |
          cp .env.example .env
      -
        name: load images
        uses: docker/bake-action@v6.0.0
        with:
          files: docker-compose.yml,docker-compose.dev.yml
          push: false
          load: true
          workdir: store
          source: "{{defaultContext}}:store"
          set: |
              neurostore.cache-from=type=gha,scope=cached-neurostore
              store_nginx.cache-from=type=gha,scope=cached-store-nginx
              store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
      - 
        name: spin up backend
        run: |
          docker network create nginx-proxy
          docker compose pull --ignore-buildable 
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            up -d --no-build
      - 
        name: Create Test Database
        run: |
          until docker compose exec -T \
          store_pgsql pg_isready -U postgres; do sleep 1; done

          docker compose exec -T \
          store_pgsql \
          psql -U postgres -c "create database test_db"
      -
        name: Backend Tests
        env:
          AUTH0_CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }}
          AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
          AUTH0_BASE_URL: ${{ secrets.AUTH0_BASE_URL }}
          AUTH0_ACCESS_TOKEN_URL: ${{ secrets.AUTH0_ACCESS_TOKEN_URL }}
          AUTH0_AUTH_URL: ${{ secrets.AUTH0_AUTH_URL }}
        run: |
          docker compose run \
            -e "APP_SETTINGS=neurostore.config.DockerTestConfig" \
            -e "AUTH0_CLIENT_ID=${AUTH0_CLIENT_ID}" \
            -e "AUTH0_CLIENT_SECRET=${AUTH0_CLIENT_SECRET}" \
            -e "AUTH0_BASE_URL=${AUTH0_BASE_URL}" \
            -e "AUTH0_ACCESS_TOKEN_URL=${AUTH0_ACCESS_TOKEN_URL}" \
            -e "AUTH0_AUTH_URL=${AUTH0_AUTH_URL}" \
            --rm -w /neurostore \
            neurostore \
            bash -c "python -m pytest neurostore/tests && python -m pytest --auth neurostore/tests/test_auth.py"

  neurosynth_compose_backend_tests:
    runs-on: ubuntu-latest
    needs: build_neurosynth_compose
    defaults:
      run:
        working-directory: compose
    steps:
    - 
      name: Checkout
      uses: actions/checkout@v4
      with:
        submodules: recursive
    -
      name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    -
      name: Configuration
      run: |
        cp .env.example .env
    -
      name: load images
      uses: docker/bake-action@v6.0.0
      with:
        files: docker-compose.yml,docker-compose.dev.yml
        push: false
        load: true
        workdir: compose
        source: "{{defaultContext}}:compose"
        set: |
            compose.cache-from=type=gha,scope=cached-compose
            compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
            compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
            compose_worker.cache-from=type=gha,scope=cached-compose-worker
    -
      name: Spin up backend
      run: |
        docker network create nginx-proxy
        docker compose pull --ignore-buildable 
        docker compose \
          -f docker-compose.yml \
          -f docker-compose.dev.yml \
          up -d --no-build
    -
      name: Create Test Database
      run: |
        until docker compose exec -T \
        compose_pgsql pg_isready -U postgres; do sleep 1; done

        docker compose exec -T \
        compose_pgsql \
        psql -U postgres -c "create database test_db"
    -
      name: Backend Tests
      env:
        AUTH0_CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }}
        AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
        AUTH0_BASE_URL: ${{ secrets.AUTH0_BASE_URL }}
        AUTH0_ACCESS_TOKEN_URL: ${{ secrets.AUTH0_ACCESS_TOKEN_URL }}
        AUTH0_AUTH_URL: ${{ secrets.AUTH0_AUTH_URL }}
        NEUROVAULT_ACCESS_TOKEN: ${{ secrets.NEUROVAULT_ACCESS_TOKEN }}
      run: |
        docker compose run \
          -e "APP_SETTINGS=neurosynth_compose.config.DockerTestConfig" \
          -e "AUTH0_CLIENT_ID=${AUTH0_CLIENT_ID}" \
          -e "AUTH0_CLIENT_SECRET=${AUTH0_CLIENT_SECRET}" \
          -e "AUTH0_BASE_URL=${AUTH0_BASE_URL}" \
          -e "AUTH0_ACCESS_TOKEN_URL=${AUTH0_ACCESS_TOKEN_URL}" \
          -e "AUTH0_AUTH_URL=${AUTH0_AUTH_URL}" \
          -e "NEUROVAULT_ACCESS_TOKEN=${NEUROVAULT_ACCESS_TOKEN}" \
          --rm -w /compose \
          compose \
          bash -c "python -m pytest neurosynth_compose/tests && python -m pytest --celery neurosynth_compose/tests/api/celery_tests && python -m pytest --auth neurosynth_compose/tests/api/test_auth.py"

  style_check:
    runs-on: ubuntu-latest
    steps:
    -
      name: Checkout
      uses: actions/checkout@v4
      with:
        submodules: recursive
    -
      name: run flake8
      run: |
        pip install flake8
        cd ./store
        flake8 ./neurostore
        cd ../compose
        flake8 ./neurosynth_compose


  neurosynth_compose_JUNIT_tests:
    runs-on: ubuntu-latest
    needs: build_neurosynth_compose
    defaults:
      run:
        working-directory: compose
    steps:
    - 
      name: Checkout
      uses: actions/checkout@v4
      with:
        submodules: recursive
    -
      name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    -
      name: Configuration
      run: |
        cp .env.example .env
    -
      name: load images
      uses: docker/bake-action@v6.0.0
      with:
        files: docker-compose.yml,docker-compose.dev.yml
        push: false
        load: true
        workdir: compose
        source: "{{defaultContext}}:compose"
        set: |
            compose.cache-from=type=gha,scope=cached-compose
            compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
            compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
            compose_worker.cache-from=type=gha,scope=cached-compose-worker
    -
      name: Spin up backend
      run: |
        docker network create nginx-proxy
        docker compose pull --ignore-buildable 
        docker compose \
          -f docker-compose.yml \
          -f docker-compose.dev.yml \
          up -d --no-build
    -
      name: Create Test Database
      run: |
        until docker compose exec -T \
        compose_pgsql pg_isready -U postgres; do sleep 1; done

        docker compose exec -T \
        compose_pgsql \
        psql -U postgres -c "create database test_db"
    -
      name: Frontend Jest Unit Tests
      env:
        AUTH0_CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }}
        AUTH0_CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }}
        AUTH0_BASE_URL: ${{ secrets.AUTH0_BASE_URL }}
        AUTH0_ACCESS_TOKEN_URL: ${{ secrets.AUTH0_ACCESS_TOKEN_URL }}
        AUTH0_AUTH_URL: ${{ secrets.AUTH0_AUTH_URL }}
        VITE_APP_AUTH0_CLIENT_ID: ${{ secrets.VITE_APP_AUTH0_CLIENT_ID }}
        VITE_APP_AUTH0_DOMAIN: ${{ secrets.VITE_APP_AUTH0_DOMAIN }}
        VITE_APP_AUTH0_CLIENT_SECRET: ${{ secrets.VITE_APP_AUTH0_CLIENT_SECRET }}
      run: |
        cp neurosynth-frontend/.env.example neurosynth-frontend/.env.dev && \
        docker compose run \
          -e "APP_SETTINGS=neurosynth_compose.config.DockerTestConfig" \
          -e "AUTH0_CLIENT_ID=${AUTH0_CLIENT_ID}" \
          -e "AUTH0_CLIENT_SECRET=${AUTH0_CLIENT_SECRET}" \
          -e "AUTH0_BASE_URL=${AUTH0_BASE_URL}" \
          -e "AUTH0_ACCESS_TOKEN_URL=${AUTH0_ACCESS_TOKEN_URL}" \
          -e "AUTH0_AUTH_URL=${AUTH0_AUTH_URL}" \
          -e "VITE_APP_AUTH0_DOMAIN=${VITE_APP_AUTH0_DOMAIN}" \
          -e "VITE_APP_AUTH0_CLIENT_ID=${VITE_APP_AUTH0_CLIENT_ID}" \
          -e "VITE_APP_AUTH0_AUDIENCE=https://neurostore.xyz/api/" \
          -e "VITE_APP_AUTH0_CLIENT_SECRET=${VITE_APP_AUTH0_CLIENT_SECRET}" \
          -e "VITE_APP_ENV=STAGING" \
          -e "VITE_APP_NEUROSTORE_API_DOMAIN=http://localhost/api" \
          -e "CI=true" \
          -e "VITE_APP_NEUROSYNTH_API_DOMAIN=http://localhost:81/api" \
          --rm -w /compose/neurosynth-frontend \
          compose \
          bash -c "cd /compose/neurosynth-frontend && \
          npm install && npm run test"

  neurosynth_compose_cypress_tests:
    runs-on: ubuntu-latest
    needs:
      - build_neurosynth_compose
      - build_neurostore
    steps:
    - 
      name: Checkout
      uses: actions/checkout@v4
      with:
        submodules: recursive
    -
      name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    -
      name: Configuration
      run: |
        cp compose/.env.example compose/.env
        cp store/.env.example store/.env
    -
      name: load compose images
      uses: docker/bake-action@v6.0.0
      with:
        files: docker-compose.yml,docker-compose.dev.yml
        push: false
        load: true
        workdir: compose
        source: "{{defaultContext}}:compose"
        set: |
            compose.cache-from=type=gha,scope=cached-compose
            compose_nginx.cache-from=type=gha,scope=cached-compose-nginx
            compose_pgsql.cache-from=type=gha,scope=cached-compose-pgsql
            compose_worker.cache-from=type=gha,scope=cached-compose-worker
    -
      name: load store images
      uses: docker/bake-action@v6.0.0
      with:
        files: docker-compose.yml,docker-compose.dev.yml
        push: false
        load: true
        workdir: store
        source: "{{defaultContext}}:store"
        set: |
            neurostore.cache-from=type=gha,scope=cached-neurostore
            store_nginx.cache-from=type=gha,scope=cached-store-nginx
            store_pgsql.cache-from=type=gha,scope=cached-store-pgsql
    -
      name: Spin up compose backend
      run: |
        cd compose
        docker network create nginx-proxy
        docker compose pull --ignore-buildable 
        docker compose \
          -f docker-compose.yml \
          -f docker-compose.dev.yml \
          up -d --no-build
    -
      name: Spin up store backend
      run: |
        cd store
        docker compose pull --ignore-buildable 
        docker compose \
          -f docker-compose.yml \
          -f docker-compose.dev.yml \
          up -d --no-build
    -
      name: Create Compose Database
      run: |
        cd compose
        until docker compose exec -T \
        compose_pgsql pg_isready -U postgres; do sleep 1; done

        docker compose exec -T \
        compose_pgsql \
        psql -U postgres -c "create database compose"
    -
      name: Create Store Database
      run: |
        cd store
        until docker compose exec -T \
        store_pgsql pg_isready -U postgres; do sleep 1; done

        docker compose exec -T \
        store_pgsql \
        psql -U postgres -c "create database neurostore"
    -
      name: Initialize Compose Database
      run: |
        cd compose
        docker compose exec -T compose \
        bash -c \
            "flask db merge heads && \
             flask db stamp head && \
             flask db migrate && \
             flask db upgrade"
    -
      name: Initialize Store Database
      run: |
        cd store
        docker compose exec -T neurostore \
        bash -c \
            "flask db merge heads && \
             flask db stamp head && \
             flask db migrate && \
             flask db upgrade"
    - 
      name: Ingest data into Store
      run: |
        cd store
        docker compose exec -T neurostore bash -c "flask ingest-neurosynth --max-rows 100"
    -
      name: Ingest data into Compose
      run: |
        cd compose
        ipaddr=$(docker inspect neurostore | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['NetworkSettings']['Networks']['nginx-proxy']['IPAddress'])")
        docker compose exec -T compose bash -c "flask create-meta-analyses --n-studysets 1 --neurostore-url http://${ipaddr}:8000"
    -
      name: Frontend Cypress E2E Tests
      uses: cypress-io/github-action@v6
      env:
        CYPRESS_auth0ClientId: ${{ secrets.VITE_APP_AUTH0_CLIENT_ID }}
        CYPRESS_auth0ClientSecret: ${{ secrets.VITE_APP_AUTH0_CLIENT_SECRET }}
        CYPRESS_: ${{ secrets.VITE_APP_AUTH0_DOMAIN }}
        CYPRESS_auth0Audience: https://neurostore.xyz/api/
        CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
        VITE_APP_AUTH0_AUDIENCE: https://neurostore.xyz/api/
        VITE_APP_AUTH0_CLIENT_ID: ${{ secrets.VITE_APP_AUTH0_CLIENT_ID }}
        VITE_APP_AUTH0_DOMAIN: ${{ secrets.VITE_APP_AUTH0_DOMAIN }}
        VITE_APP_AUTH0_CLIENT_SECRET: ${{ secrets.VITE_APP_AUTH0_CLIENT_SECRET }}
        VITE_APP_ENV: STAGING
        VITE_APP_NEUROSTORE_API_DOMAIN: ${{ secrets.VITE_APP_NEUROSTORE_API_DOMAIN }}
        VITE_APP_NEUROSYNTH_API_DOMAIN: ${{ secrets.VITE_APP_NEUROSYNTH_API_DOMAIN }}
      with:
        build: npm run build:staging
        record: true
        start: npm run serve-build
        browser: chrome
        wait-on: http://localhost:3000
        working-directory: /home/runner/work/neurostore/neurostore/compose/neurosynth-frontend