diff --git a/patchfinder/pom.xml b/patchfinder/pom.xml index b389a77a4..fbd42aff0 100644 --- a/patchfinder/pom.xml +++ b/patchfinder/pom.xml @@ -9,8 +9,8 @@ 1.0 - 1.8 - 1.8 + 17 + 17 UTF-8 diff --git a/patchfinder/src/main/java/FixFinderMain.java b/patchfinder/src/main/java/FixFinderMain.java index 78938fa2e..2daed5973 100644 --- a/patchfinder/src/main/java/FixFinderMain.java +++ b/patchfinder/src/main/java/FixFinderMain.java @@ -22,6 +22,7 @@ * SOFTWARE. */ +import db.DatabaseHelper; import env.FixFinderEnvVars; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -37,6 +38,11 @@ */ public class FixFinderMain extends Thread { private final static Logger logger = LogManager.getLogger(FixFinderMain.class); + private final DatabaseHelper databaseHelper; + + public FixFinderMain(DatabaseHelper dbh) { + this.databaseHelper = dbh; + } /** * Entry point for the FixFinder, initializes necessary classes and start listening for jobs with RabbitMQ @@ -45,18 +51,24 @@ public class FixFinderMain extends Thread { public void run() { logger.info("Starting FixFinder..."); - // Init FixFinder - FixFinder.init(); + // Get input mode + final String inputMode = FixFinderEnvVars.getInputMode(); // Determine run mode and start PatchFinder - switch (FixFinderEnvVars.getInputMode()) { + switch (inputMode) { case "db": + // Init FixFinder + FixFinder.init(this.databaseHelper); runDb(); break; case "rabbit": + // Init FixFinder + FixFinder.init(this.databaseHelper); runRabbit(); break; case "dev": + // Init FixFinder + FixFinder.init(this.databaseHelper); runDev(); break; default: @@ -95,7 +107,7 @@ private void runDev() { } public static void main(String[] args) { - FixFinderMain finder = new FixFinderMain(); - finder.start(); +// FixFinderMain finder = new FixFinderMain(); +// finder.start(); } } diff --git a/patchfinder/src/main/java/PatchFinderMain.java b/patchfinder/src/main/java/PatchFinderMain.java index 91e03f6c0..c57a178ba 100644 --- a/patchfinder/src/main/java/PatchFinderMain.java +++ b/patchfinder/src/main/java/PatchFinderMain.java @@ -22,6 +22,7 @@ * SOFTWARE. */ +import db.DatabaseHelper; import env.PatchFinderEnvVars; import messenger.Messenger; import model.CpeGroup; @@ -41,6 +42,11 @@ */ public class PatchFinderMain extends Thread { private final static Logger logger = LogManager.getLogger(PatchFinderMain.class); + private final DatabaseHelper databaseHelper; + + public PatchFinderMain(DatabaseHelper dbh) { + this.databaseHelper = dbh; + } /** * Entry point for the PatchFinder, initializes necessary classes and start listening for jobs with RabbitMQ @@ -49,7 +55,7 @@ public class PatchFinderMain extends Thread { public void run() { logger.info("Starting PatchFinder..."); // Init PatchFinder - PatchFinder.init(); + PatchFinder.init(this.databaseHelper); // Determine run mode and start PatchFinder switch (PatchFinderEnvVars.getInputMode()) { diff --git a/patchfinder/src/main/java/PatchFixMain.java b/patchfinder/src/main/java/PatchFixMain.java index 50a6bf4bc..aa7b03221 100644 --- a/patchfinder/src/main/java/PatchFixMain.java +++ b/patchfinder/src/main/java/PatchFixMain.java @@ -1,6 +1,16 @@ +import db.DatabaseHelper; +import env.SharedEnvVars; + public class PatchFixMain { public static void main(String[] args) { - new PatchFinderMain().start(); - new FixFinderMain().start(); + SharedEnvVars.initializeEnvVars(false); + final DatabaseHelper dbh = new DatabaseHelper( + SharedEnvVars.getDatabaseType(), + SharedEnvVars.getHikariUrl(), + SharedEnvVars.getHikariUser(), + SharedEnvVars.getHikariPassword() + ); + new PatchFinderMain(dbh).start(); + new FixFinderMain(dbh).start(); } } diff --git a/patchfinder/src/main/java/env/SharedEnvVars.java b/patchfinder/src/main/java/env/SharedEnvVars.java new file mode 100644 index 000000000..054a3c813 --- /dev/null +++ b/patchfinder/src/main/java/env/SharedEnvVars.java @@ -0,0 +1,112 @@ +package env; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.FileNotFoundException; +import java.util.HashMap; +import java.util.Map; + +import static env.EnvVarLoader.loadEnvVarsFromFile; + +public class SharedEnvVars { + private static final Logger logger = LogManager.getLogger(PatchFinderEnvVars.class); + private static final String envVarPath = "env.list"; + + // Default values for database environment variables + private static String databaseType = "mysql"; + private static String hikariUrl = "jdbc:mysql://localhost:3306/nvip?useSSL=false&allowPublicKeyRetrieval=true"; + private static String hikariUser = "root"; + private static String hikariPassword = "root"; + + public static String getDatabaseType() { return databaseType; } + public static String getHikariUrl() { return hikariUrl; } + public static String getHikariUser() { return hikariUser; } + public static String getHikariPassword() { return hikariPassword; } + + /** + * Loads environment variables from both env.list file and System.getenv(). If both of these fail, resorts to + * default values defined above. Prioritizes System.getenv() first and then from file second. + */ + public static void initializeEnvVars(boolean testMode) { + logger.info("CURRENT PATH --> " + System.getProperty("user.dir")); + if(testMode) logger.info("Initializing Test Environment Variables..."); + else logger.info("Initializing Environment Variables..."); + + Map fileProps = null; + Map systemProps = System.getenv(); + String filePath = envVarPath; + if(testMode) filePath = "src/test/" + filePath; + + try { + // Assumes in `nvip-crawler/patchfinder` working directory + fileProps = loadEnvVarsFromFile(filePath); + } catch (FileNotFoundException e){ + // If that path doesn't work, assumes we are in `nvip-crawler` directory and tries new path with `patchfinder` appended to it + try{ + String possiblePath = "patchfinder\\" + filePath; + fileProps = loadEnvVarsFromFile(possiblePath); + } catch (Exception ignored) {} + } + + // If env vars couldn't be loaded from file, pass in empty map + if(fileProps == null) fileProps = new HashMap<>(); + fetchEnvVars(systemProps, fileProps); + } + + /** + * Attempts to fetch all required environment variables from props map safely, logging any + * missing or incorrect variables. + * + * If environment variable is not found from System.getenv(), it will attempt to fetch it from the loaded file. If it + * is still not found, it will resort to default value. Priority: System.getenv() <- env.list file <- default values + * + * @param systemProps map of environment variables from System.getenv() + * @param fileProps map of environment variables read from file + */ + private static void fetchEnvVars(Map systemProps, Map fileProps) { + fetchHikariEnvVars(systemProps, fileProps); + } + + /** + * Initialize database env vars + * + * @param systemProps map of environment variables from System.getenv() + * @param fileProps map of environment variables read from file + */ + private static void fetchHikariEnvVars(Map systemProps, Map fileProps) { + + if(systemProps.containsKey("DB_TYPE")) { + databaseType = systemProps.get("DB_TYPE"); + logger.info("Setting DB_TYPE to {}", databaseType); + } else if (fileProps.containsKey("DB_TYPE")) { + databaseType = fileProps.get("DB_TYPE"); + logger.info("Setting DB_TYPE to {}", databaseType); + } else logger.warn("Could not fetch DB_TYPE from env vars, defaulting to {}", databaseType); + + if(systemProps.containsKey("HIKARI_URL")) { + hikariUrl = systemProps.get("HIKARI_URL"); + logger.info("Setting HIKARI_URL to {}", hikariUrl); + } else if (fileProps.containsKey("HIKARI_URL")) { + hikariUrl = fileProps.get("HIKARI_URL"); + logger.info("Setting HIKARI_URL to {}", hikariUrl); + } else logger.warn("Could not fetch HIKARI_URL from env vars, defaulting to {}", hikariUrl); + + if(systemProps.containsKey("HIKARI_USER")) { + hikariUser = systemProps.get("HIKARI_USER"); + logger.info("Setting HIKARI_USER to {}", hikariUser); + } else if (fileProps.containsKey("HIKARI_USER")) { + hikariUser = fileProps.get("HIKARI_USER"); + logger.info("Setting HIKARI_USER to {}", hikariUser); + } else logger.warn("Could not fetch HIKARI_USER from env vars, defaulting to {}", hikariUser); + + if(systemProps.containsKey("HIKARI_PASSWORD")) { + hikariPassword = systemProps.get("HIKARI_PASSWORD"); + logger.info("Setting HIKARI_PASSWORD to {}", hikariPassword); + } else if (fileProps.containsKey("HIKARI_PASSWORD")) { + hikariPassword = fileProps.get("HIKARI_PASSWORD"); + logger.info("Setting HIKARI_PASSWORD to {}", hikariPassword); + } else logger.warn("Could not fetch HIKARI_PASSWORD from env vars, defaulting to {}", hikariPassword); + + } +} diff --git a/patchfinder/src/main/java/fixes/FixFinder.java b/patchfinder/src/main/java/fixes/FixFinder.java index a846296dd..e4208f8eb 100644 --- a/patchfinder/src/main/java/fixes/FixFinder.java +++ b/patchfinder/src/main/java/fixes/FixFinder.java @@ -60,17 +60,12 @@ public class FixFinder { /** * Initialize the FixFinder and its subcomponents */ - public static void init() { + public static void init(DatabaseHelper dbh) { logger.info("Initializing FixFinder..."); // Init db helper logger.info("Initializing DatabaseHelper..."); - databaseHelper = new DatabaseHelper( - FixFinderEnvVars.getDatabaseType(), - FixFinderEnvVars.getHikariUrl(), - FixFinderEnvVars.getHikariUser(), - FixFinderEnvVars.getHikariPassword() - ); + databaseHelper = dbh; // Init FixUrlFinders logger.info("Initializing FixUrlFinders..."); @@ -170,4 +165,9 @@ public static void run(List cveIds) { // existingInserts // ); } + + @Override + public String toString() { + return this.getClass().getSimpleName(); + } } diff --git a/patchfinder/src/main/java/messenger/Messenger.java b/patchfinder/src/main/java/messenger/Messenger.java index fc7aaec16..3064c44fe 100644 --- a/patchfinder/src/main/java/messenger/Messenger.java +++ b/patchfinder/src/main/java/messenger/Messenger.java @@ -54,6 +54,15 @@ public class Messenger { private static final ObjectMapper OM = new ObjectMapper(); private ConnectionFactory factory; + private final BlockingQueue> jobListQueue = new LinkedBlockingQueue<>(); + + // Define callback handler + private final DeliverCallback deliverCallback = (consumerTag, delivery) -> { + String message = new String(delivery.getBody(), StandardCharsets.UTF_8); + List parsedIds = parseIds(message); + if(parsedIds.size() > 0 && !jobListQueue.offer(parsedIds)) logger.error("Job response could not be added to message queue"); + }; + /** * Initialize the Messenger class with RabbitMQ host, username, and password * @param host RabbitMQ host @@ -68,13 +77,13 @@ public Messenger(String host, String vhost, int port, String username, String pa factory.setUsername(username); factory.setPassword(password); - try { - factory.useSslProtocol(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } catch (KeyManagementException e) { - throw new RuntimeException(e); - } +// try { +// factory.useSslProtocol(); +// } catch (NoSuchAlgorithmException e) { +// throw new RuntimeException(e); +// } catch (KeyManagementException e) { +// throw new RuntimeException(e); +// } this.inputQueue = inputQueue; } @@ -86,40 +95,40 @@ public void setFactory(ConnectionFactory factory) { /** * Waits for a message from the PNE for pollInterval seconds, returning null unless a valid job was received * - * @param pollInterval time to wait before timing out and returning null + * @param pollInterval interval time in seconds to poll the blocking queue * @return null or a list of received CVE ids to find patches for */ public List waitForProductNameExtractorMessage(int pollInterval) { // Initialize job list - List cveIds = null; - - // Busy-wait loop for jobs - while(cveIds == null) { - try(Connection connection = factory.newConnection(); - Channel channel = connection.createChannel()){ - - channel.queueDeclare(inputQueue, false, false, false, null); - - BlockingQueue> messageQueue = new ArrayBlockingQueue<>(1); - - DeliverCallback deliverCallback = (consumerTag, delivery) -> { - String message = new String(delivery.getBody(), StandardCharsets.UTF_8); - List parsedIds = parseIds(message); - if(parsedIds.size() > 0 && !messageQueue.offer(parsedIds)) logger.error("Job response could not be added to message queue"); - }; - channel.basicConsume(inputQueue, true, deliverCallback, consumerTag -> { }); + List cveIds = new ArrayList<>(); + try(Connection connection = factory.newConnection(); + Channel channel = connection.createChannel()) { + // Declare the input queue + channel.queueDeclare(inputQueue, true, false, false, null); + channel.basicConsume(inputQueue, true, deliverCallback, consumerTag -> { }); + + // Busy-wait loop for jobs + while(cveIds.size() == 0) { + // Poll queue for jobs every poll interval logger.info("Polling message queue..."); - cveIds = messageQueue.poll(pollInterval, TimeUnit.SECONDS); - if(cveIds != null) logger.info("Received job with CVE(s) {}", cveIds); - } catch (TimeoutException | InterruptedException | IOException e) { - logger.error("Error occurred while getting jobs from the ProductNameExtractor: {}", e.toString()); - break; + // Create jobs list of lists for draining queue + final List> jobs = new ArrayList<>(); + // Drain queue to jobs list + final int numReceivedJobs = jobListQueue.drainTo(jobs); + // Flatten jobs into id list + jobs.forEach(cveIds::addAll); + + // Sleep if no jobs received + if(numReceivedJobs == 0) + synchronized (this) { wait(pollInterval * 1000L); } } + logger.info("Received job with CVE(s) {}", cveIds); + } catch (TimeoutException | InterruptedException | IOException e) { + logger.error("Error occurred while getting jobs from the ProductNameExtractor: {}", e.toString()); } - return cveIds; } @@ -147,9 +156,7 @@ private void sendDummyMessage(String queue, String message) { try(Connection connection = factory.newConnection(); Channel channel = connection.createChannel()){ - channel.queueDeclare(queue, false, false, false, null); - - channel.basicPublish("", queue, null, message.getBytes()); + channel.basicPublish("", queue, null, message.getBytes(StandardCharsets.UTF_8)); } catch (IOException | TimeoutException e) { logger.error("Failed to send dummy message: {}", e.toString()); @@ -157,17 +164,31 @@ private void sendDummyMessage(String queue, String message) { } public static void main(String[] args) { -// final Messenger m = new Messenger("localhost", "guest", "guest"); -// m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-2933\", \"CVE-2023-2934\"]"); - ObjectMapper OM = new ObjectMapper(); - try { - OM.writerWithDefaultPrettyPrinter().writeValue(new File("patchfinder/target/test.json"), "test1"); - OM.writerWithDefaultPrettyPrinter().writeValue(new File("patchfinder/target/test.json"), "test2"); -// OM.writeValue(new File("patchfinder/target/test.json"), "test1"); -// OM.writeValue(new File("patchfinder/target/test.json"), "test2"); - Thread.sleep(10000); - } catch (Exception e) { - e.printStackTrace(); - } + final String INPUT_QUEUE = "PNE_OUT"; + final Messenger m = new Messenger("localhost", "/", 5672 , "guest", "guest", INPUT_QUEUE); + m.sendDummyMessage(INPUT_QUEUE,"[\"CVE-2023-0001\", \"CVE-2023-0002\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0003\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0004\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0005\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0006\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0007\", \"CVE-2023-0008\", \"CVE-2023-0009\"]"); + + try { Thread.sleep(5000); } catch (Exception ignored) { } + + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0010\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0011\"]"); + m.sendDummyMessage(INPUT_QUEUE, "[\"CVE-2023-0012\"]"); + +// m.waitForProductNameExtractorMessage(5); +// ObjectMapper OM = new ObjectMapper(); +// try { +// OM.writerWithDefaultPrettyPrinter().writeValue(new File("patchfinder/target/test.json"), "test1"); +// OM.writerWithDefaultPrettyPrinter().writeValue(new File("patchfinder/target/test.json"), "test2"); +//// OM.writeValue(new File("patchfinder/target/test.json"), "test1"); +//// OM.writeValue(new File("patchfinder/target/test.json"), "test2"); +// Thread.sleep(10000); +// } catch (Exception e) { +// e.printStackTrace(); +// } } } diff --git a/patchfinder/src/main/java/patches/PatchFinder.java b/patchfinder/src/main/java/patches/PatchFinder.java index 35b928f45..a5e62a127 100644 --- a/patchfinder/src/main/java/patches/PatchFinder.java +++ b/patchfinder/src/main/java/patches/PatchFinder.java @@ -65,23 +65,19 @@ public class PatchFinder { protected static String patchSrcUrlPath = PatchFinderEnvVars.getPatchSrcUrlPath(); protected static int cveLimit = PatchFinderEnvVars.getCveLimit(); protected static int maxThreads = PatchFinderEnvVars.getMaxThreads(); +// private static final BlockingQueue workQueue = new LinkedBlockingQueue<>(); public static DatabaseHelper getDatabaseHelper() { return databaseHelper; } /** * Initialize the Patchfinder and its subcomponents */ - public static void init() { + public static void init(DatabaseHelper dbh) { logger.info("Initializing PatchFinder..."); // Init db helper logger.info("Initializing DatabaseHelper..."); - databaseHelper = new DatabaseHelper( - PatchFinderEnvVars.getDatabaseType(), - PatchFinderEnvVars.getHikariUrl(), - PatchFinderEnvVars.getHikariUser(), - PatchFinderEnvVars.getHikariPassword() - ); + databaseHelper = dbh; // Init PatchUrlFinder logger.info("Initializing PatchUrlFinder..."); @@ -324,15 +320,17 @@ public static void findPatchesMultiThreaded(Map> possi // Init clone path and clear previously stored repos File dir = new File(clonePath); if(!dir.exists()) { - logger.warn("Unable to locate clone path '{}' for previous run repo deletion", clonePath); + logger.warn("Could not locate clone directory at path '{}'", clonePath); try { dir.createNewFile(); } catch (IOException e) { logger.error("Failed to create missing directory '{}'", clonePath); } - } - else { - logger.info("Clearing any existing repos @ '{}'", clonePath); - try { FileUtils.delete(dir, FileUtils.RECURSIVE); } - catch (IOException e) { logger.error("Failed to clear clone dir @ '{}': {}", dir, e); } - } + } else logger.info("Clone directory already exists at {}", clonePath); + //TODO: Figure out a solid solution to handling overwriting existing cloned repos, have had 0 success with + // deleting programmatically so far, might be a job for the docker env to handle the destruction of the clone dir +// else { +// logger.info("Clearing any existing repos @ '{}'", clonePath); +// try { FileUtils.delete(dir, FileUtils.RECURSIVE); } +// catch (IOException e) { logger.error("Failed to clear clone dir @ '{}': {}", dir, e); } +// } // Determine the actual number of CVEs to be processed final int totalCVEsToProcess = Math.min(possiblePatchSources.size(), cveLimit); diff --git a/patchfinder/src/test/java/PatchFinderMainTest.java b/patchfinder/src/test/java/PatchFinderMainTest.java index adf0d6faf..b845a9a5d 100644 --- a/patchfinder/src/test/java/PatchFinderMainTest.java +++ b/patchfinder/src/test/java/PatchFinderMainTest.java @@ -52,7 +52,7 @@ public void testMain() { // Create a mock DatabaseHelper DatabaseHelper databaseHelperMock = mock(DatabaseHelper.class); - PatchFinder.init(); + PatchFinder.init(databaseHelperMock); // Create a mock Map of affected products Map affectedProductsMock = new HashMap<>(); @@ -70,13 +70,13 @@ public void testMain() { }); // Initialize PatchFinder with the mock Messenger - PatchFinder.init(); + PatchFinder.init(databaseHelperMock); // Call the main method then timeout after 10 seconds CountDownLatch latch = new CountDownLatch(1); new Thread(() -> { try { - new PatchFinderMain().start(); + new PatchFinderMain(databaseHelperMock).start(); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } diff --git a/patchfinder/src/test/java/patches/PatchFinderTest.java b/patchfinder/src/test/java/patches/PatchFinderTest.java index 2c16f96f5..52737d469 100644 --- a/patchfinder/src/test/java/patches/PatchFinderTest.java +++ b/patchfinder/src/test/java/patches/PatchFinderTest.java @@ -22,6 +22,7 @@ * SOFTWARE. */ +import db.DatabaseHelper; import env.PatchFinderEnvVars; import model.CpeEntry; import model.CpeGroup; @@ -44,11 +45,12 @@ * @author Richard Sawh */ public class PatchFinderTest { + private final DatabaseHelper databaseHelperMock = mock(DatabaseHelper.class); @Before public void setUp() { PatchFinderEnvVars.initializeEnvVars(true); - PatchFinder.init(); + PatchFinder.init(databaseHelperMock); } @Test @@ -100,7 +102,7 @@ public void testRun() { CpeGroup cpeGroup = new CpeGroup("apache", "airflow", "product_name_value", new HashMap<>()); possiblePatchSources.put("CVE-2023-1001", cpeGroup); - PatchFinder.init(); + PatchFinder.init(databaseHelperMock); try { // Call the run method and assert the expected behavior or outcome if(PatchFinder.run(possiblePatchSources, PatchFinder.cveLimit) == 0){