crawler-workflow.yml

# CI for NVIP

name: Crawler Workflow

on:
  pull_request:
    branches:
      - main
      - dev
  push:
    branches:
      - main
      - dev
      - 'cicd/**'
    paths-ignore:
      - '.github/workflows/patchfinder-workflow.yml'
      - '.github/workflows/pne-workflow.yml'
      - '.github/workflows/reconciler-workflow.yml'
      - 'productnameextractor/**'
      - 'exploitfinder/**'
      - 'patchfinder/**'
      - 'reconciler/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  AWS_REGION: 'us-east-1'
  DB_TYPE: 'mysql'
  DB_NAME: 'nvip'
  HIKARI_URL: 'jdbc:mysql://localhost:3306/nvip?useSSL=false&allowPublicKeyRetrieval=true'
  HIKARI_USER: 'root'
  HIKARI_PASSWORD: 'root'
  OUTPUT_DIR: 'output'
  RESOURCE_DIR: 'nvip_data'
  DATA_DIR: 'data'
  NLP_DIR: 'nlp'
  CHAR_2_VEC_CONFIG: 'c2v_model_config_50.json'
  CHAR_2_VEC_WEIGHTS: 'c2v_model_weights_50.h5'
  WORD_2_VEC: 'w2v_model_250.bin'
  NER_MODEL: 'NERallModel.bin'
  NER_MODEL_NORMALIZER: 'NERallNorm.bin'
  SENTENCE_MODEL: 'en-sent.bin'
  PRODUCT_DETECTOR_MODEL: 'en-pos-perceptron.bin'

jobs:
  # Build and Test Crawler Component
  build-test-crawler:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          lfs: 'true'

      - uses: actions/setup-java@v3.10.0
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'maven'

      # Setup MySQL DB for tests that require connection (Might not need this)
      - name: Initialize Database
        env:
          LIQUIBASE_COMMAND_URL: ${{ env.HIKARI_URL }}
          LIQUIBASE_COMMAND_USERNAME: ${{ env.HIKARI_USER }}
          LIQUIBASE_COMMAND_PASSWORD: ${{ env.HIKARI_PASSWORD }}
        run: |
          sudo /etc/init.d/mysql start
          mysql -e 'CREATE DATABASE ${{ env.DB_NAME }};' -u${{ env.HIKARI_USER }} -p${{ env.HIKARI_PASSWORD }}

      - name: Run Liquibase Update
        run: |
          mvn liquibase:update --projects=db

      - name: Test Crawler
        run: |
          mvn test --projects=db,crawler

      - name: Publish Test Report
        if: success() || failure()
        uses: scacap/action-surefire-report@v1
        with:
          check_name: Crawler Test Report

  # Deploy crawler component project to container registry
  staging-deploy:
    runs-on: ubuntu-latest
    environment: staging
    needs: build-test-crawler
    if: ${{ success() && github.ref == 'refs/heads/main' }}
    steps:
      - uses: actions/checkout@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and Push Docker Image for Crawler
        run: |
          docker build . -f crawler/Dockerfile -t nvip-crawler:staging -t ghcr.io/softwaredesignlab/nvip-crawler:staging
          docker push ghcr.io/softwaredesignlab/nvip-crawler:staging

  # Deploy crawler component project to container registry
  production-deploy:
    runs-on: ubuntu-latest
    environment: production
    needs: staging-deploy
    steps:
      - uses: actions/checkout@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and Push Docker Image for Crawler
        run: |
          docker build . -f crawler/Dockerfile -t nvip-crawler -t ghcr.io/softwaredesignlab/nvip-crawler
          docker push ghcr.io/softwaredesignlab/nvip-crawler