Merge branch 'crawler-added-sources' of https://github.com/SoftwareDe…

…signLab/nvip-crawler into crawler-added-sources
SoftwareDesignLab · Sep 27, 2023 · 28cf9b0 · 28cf9b0
2 parents 8420748 + eab4167
commit 28cf9b0
Show file tree

Hide file tree

Showing 568 changed files with 3,908 additions and 459,850 deletions.
diff --git a/.github/workflows/crawler-workflow.yml b/.github/workflows/crawler-workflow.yml
@@ -53,16 +53,16 @@ jobs:
       # Setup MySQL DB for tests that require connection (Might not need this)
       - name: Initialize Database
         env:
-          LIQUIBASE_URL: ${{ env.HIKARI_URL }}
-          LIQUIBASE_USER: ${{ env.HIKARI_USER }}
-          LIQUIBASE_PASS: ${{ env.HIKARI_PASSWORD }}
+          LIQUIBASE_COMMAND_URL: ${{ env.HIKARI_URL }}
+          LIQUIBASE_COMMAND_USERNAME: ${{ env.HIKARI_USER }}
+          LIQUIBASE_COMMAND_PASSWORD: ${{ env.HIKARI_PASSWORD }}
         run: |
           sudo /etc/init.d/mysql start
           mysql -e 'CREATE DATABASE ${{ env.DB_NAME }};' -u${{ env.HIKARI_USER }} -p${{ env.HIKARI_PASSWORD }}
 
       - name: Run Liquibase Update
         run: |
-          mvn liquibase:update
+          mvn liquibase:update --projects=crawler
 
       - name: Test Crawler
         run: |
@@ -93,6 +93,5 @@ jobs:
 
       - name: Build and Push Docker Image for Crawler
         run: |
-          cd crawler
-          docker build . -t nvip-crawler-sub:dev -t ghcr.io/softwaredesignlab/nvip-crawler-sub:dev
+          docker build . -f crawler/Dockerfile -t nvip-crawler-sub:dev -t ghcr.io/softwaredesignlab/nvip-crawler-sub:dev
           docker push ghcr.io/softwaredesignlab/nvip-crawler-sub:dev     
diff --git a/.github/workflows/patchfinder-workflow.yml b/.github/workflows/patchfinder-workflow.yml
@@ -57,12 +57,12 @@ jobs:
 
       - name: Initialize DB
         env:
-          LIQUIBASE_URL: ${{ env.HIKARI_URL }}
-          LIQUIBASE_USER: ${{ env.HIKARI_USER }}
-          LIQUIBASE_PASS: ${{ env.HIKARI_PASSWORD }}
+          LIQUIBASE_COMMAND_URL: ${{ env.HIKARI_URL }}
+          LIQUIBASE_COMMAND_USERNAME: ${{ env.HIKARI_USER }}
+          LIQUIBASE_COMMAND_PASSWORD: ${{ env.HIKARI_PASSWORD }}
         run: |
           mysql -e 'CREATE DATABASE nvip;' -u${{ env.HIKARI_USER }} -p${{ env.HIKARI_PASSWORD }} 
-          mvn liquibase:update --no-transfer-progress
+          mvn liquibase:update --no-transfer-progress --projects=patchfinder
 
       - name: Test and Build PatchFinder with Maven
         run: |
@@ -101,5 +101,5 @@ jobs:
       - name: Build and Push Docker Image for Patchfinder
         run: |
           cd patchfinder
-          docker build . -t nvip-patchfinder:dev -t ghcr.io/softwaredesignlab/nvip-patchfinder:dev
+          docker build . -f patchfinder/Dockerfile -t nvip-patchfinder:dev -t ghcr.io/softwaredesignlab/nvip-patchfinder:dev
           docker push ghcr.io/softwaredesignlab/nvip-patchfinder:dev
diff --git a/.github/workflows/pne-workflow.yml b/.github/workflows/pne-workflow.yml
@@ -69,12 +69,12 @@ jobs:
 
       - name: Initialize DB
         env:
-          LIQUIBASE_URL: ${{ env.HIKARI_URL }}
-          LIQUIBASE_USER: ${{ env.HIKARI_USER }}
-          LIQUIBASE_PASS: ${{ env.HIKARI_PASSWORD }}
+          LIQUIBASE_COMMAND_URL: ${{ env.HIKARI_URL }}
+          LIQUIBASE_COMMAND_USERNAME: ${{ env.HIKARI_USER }}
+          LIQUIBASE_COMMAND_PASSWORD: ${{ env.HIKARI_PASSWORD }}
         run: |
           mysql -e 'CREATE DATABASE nvip;' -u${{ env.HIKARI_USER }} -p${{ env.HIKARI_PASSWORD }} 
-          mvn liquibase:update --no-transfer-progress
+          mvn liquibase:update --no-transfer-progress --projects=productnameextractor
 
       - name: Test and Build Product Extractor with Maven
         run: |
@@ -119,5 +119,5 @@ jobs:
       - name: Build and Push Docker Image for Product Extractor
         run: |
           cd productnameextractor
-          docker build . -t nvip-product-extractor:dev -t ghcr.io/softwaredesignlab/nvip-product-extractor:dev
+          docker build . -f productnameextractor/Dockerfile -t nvip-product-extractor:dev -t ghcr.io/softwaredesignlab/nvip-product-extractor:dev
           docker push ghcr.io/softwaredesignlab/nvip-product-extractor:dev
diff --git a/.github/workflows/reconciler-workflow.yml b/.github/workflows/reconciler-workflow.yml
@@ -70,5 +70,5 @@ jobs:
       - name: Build and Push Docker Image for Reconciler
         run: |
           cd reconciler
-          docker build . -t nvip-reconciler:dev -t ghcr.io/softwaredesignlab/nvip-reconciler:dev
+          docker build . -f reconciler/Dockerfile -t nvip-reconciler:dev -t ghcr.io/softwaredesignlab/nvip-reconciler:dev
           docker push ghcr.io/softwaredesignlab/nvip-reconciler:dev  
diff --git a/.gitignore b/.gitignore
@@ -6,10 +6,7 @@ logs
 *.iml
 
 # Class outputs
-/target/
-/crawler/target/
-/reconciler/target/
-/productnameextractor/target/
+*/target/*
 /productnameextractor/nvip_data/data/test_results.csv
 
 # Large files
@@ -36,4 +33,4 @@ env.list
 # Git Repos
 nvip_data/pypa-repo/
 exploitfinder/src/main/resources/exploit-repo
-patchfinder/src/main/resources/patch-repos
+patchfinder/src/main/resources/patch-repos
diff --git a/Dockerfile b/Dockerfile
diff --git a/crawler/Dockerfile b/crawler/Dockerfile
@@ -1,15 +1,22 @@
-FROM maven:3.8-jdk-11-slim AS builder
+FROM maven:3.8.7-openjdk-18-slim AS builder
 
 WORKDIR /home/app
 
 ADD pom.xml .
+ADD crawler/pom.xml crawler/pom.xml
+ADD exploitfinder/pom.xml exploitfinder/pom.xml
+ADD patchfinder/pom.xml patchfinder/pom.xml
+ADD productnameextractor/pom.xml productnameextractor/pom.xml
+ADD reconciler/pom.xml reconciler/pom.xml
+
 RUN mvn dependency:go-offline
-ADD src/main src/main
 
-RUN mvn package -Dmaven.test.skip=true
+ADD crawler/src/main crawler/src/main
+
+RUN mvn package -Dmaven.test.skip=true --projects=crawler
 
-### Run Stage
-FROM openjdk:11-jre-slim
+### Crawler Run Stage
+FROM openjdk:11-jre-slim AS crawler
 
 RUN apt-get update \
     && apt-get install -y libglib2.0-0 libnss3 libxcb1
@@ -19,13 +26,13 @@ RUN wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd6
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
 
 VOLUME /usr/local/lib/resources
-ADD resources /usr/local/lib/resources
+COPY crawler/resources /usr/local/lib/resources
 
 VOLUME /usr/local/lib/output
-ADD output /usr/local/lib/output
+COPY crawler/output /usr/local/lib/output
 
-COPY --from=builder /home/app/target/nvip_lib /usr/local/lib/nvip_lib
-COPY --from=builder /home/app/target/crawler-1.0.jar /usr/local/lib/crawler-1.0.jar
+COPY --from=builder /home/app/crawler/target/nvip_lib /usr/local/lib/nvip_lib
+COPY --from=builder /home/app/crawler/target/crawler-2.0.jar /usr/local/lib/crawler-2.0.jar
 
 WORKDIR /usr/local/lib/
-ENTRYPOINT ["java", "-cp", "crawler-1.0.jar:nvip_lib/*", "edu.rit.se.nvip.CrawlerMain"]
+ENTRYPOINT ["java", "-cp", "crawler-2.0.jar:nvip_lib/*", "edu.rit.se.nvip.CrawlerMain"]
diff --git a/crawler/env.list b/crawler/env.list
@@ -2,9 +2,11 @@ HIKARI_URL=jdbc:mysql://host.docker.internal:3306/nvip_test?useSSL=false&allowPu
 HIKARI_USER=root
 HIKARI_PASSWORD=root
 
-MQ_HOST=localhost
-MQ_PORT=5672
-MQ_QUEUE_NAME=CRAWLER_OUT
+RABBIT_HOST=localhost
+RABBIT_PORT=5672
+RABBIT_USERNAME=guest
+RABBIT_PASSWORD=guest
+CRAWLER_OUTPUT_QUEUE=CRAWLER_OUT
 
 LOG_LEVEL=INFO