From 192e287034c693268422cb6963c978ecefa4c191 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Tue, 21 Nov 2023 17:42:18 -0500 Subject: [PATCH] Update output messages for job streaming to pf/ff --- productnameextractor/pom.xml | 4 +- productnameextractor/productnameextractor.iml | 570 ------------------ .../main/java/ProductNameExtractorMain.java | 18 +- .../src/main/java/db/DatabaseHelper.java | 53 +- .../java/env/ProductNameExtractorEnvVars.java | 29 +- .../src/main/java/messenger/Messenger.java | 186 +++--- .../AffectedProductIdentifier.java | 292 +++++---- .../src/test/java/db/DatabaseHelperTest.java | 27 +- .../env/ProductNameExtractorEnvVarsTest.java | 3 +- .../AffectedProductIdentifierTest.java | 4 +- 10 files changed, 325 insertions(+), 861 deletions(-) delete mode 100644 productnameextractor/productnameextractor.iml diff --git a/productnameextractor/pom.xml b/productnameextractor/pom.xml index 6af8a48a5..559a58bec 100644 --- a/productnameextractor/pom.xml +++ b/productnameextractor/pom.xml @@ -8,8 +8,8 @@ 1.0 - 1.8 - 1.8 + 17 + 17 UTF-8 diff --git a/productnameextractor/productnameextractor.iml b/productnameextractor/productnameextractor.iml deleted file mode 100644 index 54bfea321..000000000 --- a/productnameextractor/productnameextractor.iml +++ /dev/null @@ -1,570 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/productnameextractor/src/main/java/ProductNameExtractorMain.java b/productnameextractor/src/main/java/ProductNameExtractorMain.java index 6beb038c9..98be3333d 100644 --- a/productnameextractor/src/main/java/ProductNameExtractorMain.java +++ b/productnameextractor/src/main/java/ProductNameExtractorMain.java @@ -210,7 +210,12 @@ private static void dbMain(DatabaseHelper databaseHelper) { // Process vulnerabilities final long getProdStart = System.currentTimeMillis(); - final List affectedProducts = affectedProductIdentifier.identifyAffectedProducts(vulnList); + final List affectedProducts = new ArrayList<>(); + + for(CompositeVulnerability vuln : vulnList) { + affectedProducts.addAll(affectedProductIdentifier.identifyAffectedProducts(vuln)); + } + int numAffectedProducts = affectedProducts.size(); logger.info("Product Name Extractor found {} affected products in {} seconds", numAffectedProducts, Math.floor(((double) (System.currentTimeMillis() - getProdStart) / 1000) * 100) / 100); @@ -245,7 +250,8 @@ private static void rabbitMain(DatabaseHelper databaseHelper) { final Messenger rabbitMQ = new Messenger( factory, ProductNameExtractorEnvVars.getRabbitInputQueue(), - ProductNameExtractorEnvVars.getRabbitOutputQueue(), + ProductNameExtractorEnvVars.getRabbitPatchfinderOutputQueue(), + ProductNameExtractorEnvVars.getRabbitFixfinderOutputQueue(), affectedProductIdentifier, databaseHelper); @@ -258,11 +264,17 @@ private static void testMain() { logger.info("Test mode enabled, creating test vulnerability list..."); vulnList = createTestVulnList(); + // TODO: Deprecate and remove initializeProductIdentifier(vulnList); // Process vulnerabilities long getProdStart = System.currentTimeMillis(); - int numAffectedProducts = affectedProductIdentifier.identifyAffectedProducts(vulnList).size(); + final List affectedProducts = new ArrayList<>(); + for(CompositeVulnerability vuln : vulnList) { + affectedProducts.addAll(affectedProductIdentifier.identifyAffectedProducts(vuln)); + } + + int numAffectedProducts = affectedProducts.size(); logger.info("Product Name Extractor found {} affected products in the test run in {} seconds", numAffectedProducts, Math.floor(((double) (System.currentTimeMillis() - getProdStart) / 1000) * 100) / 100); diff --git a/productnameextractor/src/main/java/db/DatabaseHelper.java b/productnameextractor/src/main/java/db/DatabaseHelper.java index 8cd2af451..561116b2d 100644 --- a/productnameextractor/src/main/java/db/DatabaseHelper.java +++ b/productnameextractor/src/main/java/db/DatabaseHelper.java @@ -254,46 +254,43 @@ public List getAllCompositeVulnerabilities(int maxVulner } /** - * Gets list of specific vulnerabilities by their CVE IDs from the database, - * formats them into CompositeVulnerability objects, and returns the list. + * Gets specific vulnerability by CVE ID from the database, + * formats them into a CompositeVulnerability object and returns it. + * Returns null if no vulnerability found * - * @param cveIds list of CVEs to be pulled from database - * @return list of fetched vulnerabilities + * @param cveId CVE to be pulled from database + * @return fetched vulnerability */ - public List getSpecificCompositeVulnerabilities(List cveIds){ - ArrayList vulnList = new ArrayList<>(); + public CompositeVulnerability getSpecificCompositeVulnerability(String cveId){ synchronized (DatabaseHelper.class) { try (Connection connection = getConnection()) { // For each CVE ID in cveIds, query database for info specific to that cve - for(String cveId : cveIds){ - PreparedStatement pstmt = connection.prepareStatement(selectSpecificVulnerabilitySql); - pstmt.setString(1, cveId); - - ResultSet rs = pstmt.executeQuery(); - - while (rs.next()) { - int vulnId = rs.getInt("vuln_id"); - String description = rs.getString("description"); - - CompositeVulnerability vulnerability = new CompositeVulnerability( - vulnId, - cveId, - description, - CompositeVulnerability.CveReconcileStatus.UPDATE - ); - vulnList.add(vulnerability); - } - } - logger.info("Successfully loaded {} existing CVE items from DB! {} CVE items were not found in the DB", vulnList.size(), cveIds.size() - vulnList.size()); + PreparedStatement pstmt = connection.prepareStatement(selectSpecificVulnerabilitySql); + pstmt.setString(1, cveId); + + ResultSet rs = pstmt.executeQuery(); + + // If result found + if(rs.next()) { + int vulnId = rs.getInt("vuln_id"); + String description = rs.getString("description"); + + logger.info("Successfully found CVE '{}' from DB!", cveId); + return new CompositeVulnerability( + vulnId, + cveId, + description, + CompositeVulnerability.CveReconcileStatus.UPDATE + ); + } else logger.warn("CVE '{}' was not found in the DB!", cveId); } catch (Exception e) { logger.error("Error while getting existing vulnerabilities from DB\nException: {}", e.getMessage()); logger.error("This is a serious error! Product Name Extraction will not be able to proceed! Exiting..."); System.exit(1); } } - - return vulnList; + return null; } /** diff --git a/productnameextractor/src/main/java/env/ProductNameExtractorEnvVars.java b/productnameextractor/src/main/java/env/ProductNameExtractorEnvVars.java index 769d97bf9..603d4f868 100644 --- a/productnameextractor/src/main/java/env/ProductNameExtractorEnvVars.java +++ b/productnameextractor/src/main/java/env/ProductNameExtractorEnvVars.java @@ -89,7 +89,8 @@ public class ProductNameExtractorEnvVars { private static String rabbitUsername = "guest"; private static String rabbitPassword = "guest"; private static String rabbitInputQueue = "RECONCILER_OUT"; - private static String rabbitOutputQueue = "PNE_OUT"; + private static String rabbitPatchfinderOutputQueue = "PNE_OUT_PATCH"; + private static String rabbitFixfinderOutputQueue = "PNE_OUT_RABBIT"; // Automatically load env vars static{ @@ -155,7 +156,8 @@ public static String getRabbitVHost() { public static String getRabbitUsername() { return rabbitUsername; } public static String getRabbitPassword() { return rabbitPassword; } public static String getRabbitInputQueue() { return rabbitInputQueue; } - public static String getRabbitOutputQueue() { return rabbitOutputQueue; } + public static String getRabbitPatchfinderOutputQueue() { return rabbitPatchfinderOutputQueue; } + public static String getRabbitFixfinderOutputQueue() { return rabbitFixfinderOutputQueue; } public static String getInputMode() { return inputMode; } public static int getCveLimit() { return cveLimit; } @@ -414,14 +416,21 @@ private static void fetchRabbitEnvVars(Map systemProps, Map { - String message = new String(delivery.getBody(), StandardCharsets.UTF_8); - List cveIds = parseIds(message); - - if(!cveIds.isEmpty()){ - logger.info("Received job with CVE(s) {}", cveIds); - - // Pull specific cve information from database for each CVE ID passed from reconciler - List vulnList = databaseHelper.getSpecificCompositeVulnerabilities(cveIds); - - // Identify affected products from the CVEs - final long getProdStart = System.currentTimeMillis(); - List affectedProducts = affectedProductIdentifier.identifyAffectedProducts(vulnList); - - // Insert the affected products found into the database - databaseHelper.insertAffectedProductsToDB(affectedProducts); - logger.info("Product Name Extractor found and inserted {} affected products to the database in {} seconds", affectedProducts.size(), Math.floor(((double) (System.currentTimeMillis() - getProdStart) / 1000) * 100) / 100); - - // Clear cveIds, extract only the cveIds for which affected products were found to be sent to the Patchfinder - cveIds.clear(); - for (AffectedProduct affectedProduct : affectedProducts) { - if (!cveIds.contains(affectedProduct.getCveId())) cveIds.add(affectedProduct.getCveId()); + // TODO: Needed? +// channel.queueDeclare(inputQueue, true, false, false, null); +// channel.queueDeclare(outputQueue, true, false, false, null); + + channel.basicConsume(inputQueue, false, new DefaultConsumer(channel) { + @Override + public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { + // Get cveId and ensure it is not null + String cveId = parseMessage(new String(body, StandardCharsets.UTF_8)); + + if(cveId != null){ + // Pull specific cve information from database for each CVE ID passed from reconciler + CompositeVulnerability vuln = databaseHelper.getSpecificCompositeVulnerability(cveId); + + // Identify affected products from the CVEs + final long getProdStart = System.currentTimeMillis(); + List affectedProducts = affectedProductIdentifier.identifyAffectedProducts(vuln); + + // Insert the affected products found into the database + databaseHelper.insertAffectedProductsToDB(affectedProducts); + logger.info("Product Name Extractor found and inserted {} affected products to the database in {} seconds", affectedProducts.size(), Math.floor(((double) (System.currentTimeMillis() - getProdStart) / 1000) * 100) / 100); + +// // Clear cveIds, extract only the cveIds for which affected products were found to be sent to the Patchfinder +// cveIds.clear(); +// for (AffectedProduct affectedProduct : affectedProducts) { +// if (!cveIds.contains(affectedProduct.getCveId())) cveIds.add(affectedProduct.getCveId()); +// } + + logger.info("Sending jobs to patchfinder and fixfinder..."); + String response = genJson(cveId); + channel.basicPublish("", patchFinderOutputQueue, null, response.getBytes(StandardCharsets.UTF_8)); + channel.basicPublish("", fixFinderOutputQueue, null, response.getBytes(StandardCharsets.UTF_8)); + logger.info("Jobs have been sent!\n\n"); + channel.basicAck(envelope.getDeliveryTag(), false); } - - logger.info("Sending jobs to patchfinder..."); - String response = genJson(cveIds); - channel.basicPublish("", outputQueue, null, response.getBytes(StandardCharsets.UTF_8)); - logger.info("Jobs have been sent!\n\n"); } - }; - - channel.basicConsume(inputQueue, true, deliverCallback, consumerTag -> {}); + }); + +// DeliverCallback deliverCallback = (consumerTag, delivery) -> { +// String message = new String(delivery.getBody(), StandardCharsets.UTF_8); +// List cveIds = parseIds(message); +// +// if(!cveIds.isEmpty()){ +// logger.info("Received job with CVE(s) {}", cveIds); +// +// // Pull specific cve information from database for each CVE ID passed from reconciler +// List vulnList = databaseHelper.getSpecificCompositeVulnerabilities(cveIds); +// +// // Identify affected products from the CVEs +// final long getProdStart = System.currentTimeMillis(); +// List affectedProducts = affectedProductIdentifier.identifyAffectedProducts(vulnList); +// +// // Insert the affected products found into the database +// databaseHelper.insertAffectedProductsToDB(affectedProducts); +// logger.info("Product Name Extractor found and inserted {} affected products to the database in {} seconds", affectedProducts.size(), Math.floor(((double) (System.currentTimeMillis() - getProdStart) / 1000) * 100) / 100); +// +// // Clear cveIds, extract only the cveIds for which affected products were found to be sent to the Patchfinder +// cveIds.clear(); +// for (AffectedProduct affectedProduct : affectedProducts) { +// if (!cveIds.contains(affectedProduct.getCveId())) cveIds.add(affectedProduct.getCveId()); +// } +// +// logger.info("Sending jobs to patchfinder..."); +// String response = genJson(cveIds); +// channel.basicPublish("", outputQueue, null, response.getBytes(StandardCharsets.UTF_8)); +// logger.info("Jobs have been sent!\n\n"); +// } +// }; + +// channel.basicConsume(inputQueue, true, deliverCallback, consumerTag -> {}); } catch (IOException | TimeoutException e) { throw new RuntimeException(e); @@ -123,46 +158,41 @@ public void run() { } /** - * Takes a JSON string containing all CVE jobs to be processed and splits them into a list - * - * @param jsonString string containing the CVE IDs - * @return list of CVE IDs + * Parse an id from a given json string. (String should be {'cveId': 'CVE-2023-1001'}) + * @param jsonString a JSON representation of an array of String CVE ids + * @return parsed list of ids */ - @SuppressWarnings("unchecked") - public List parseIds(String jsonString) { + public static String parseMessage(String jsonString) { try { - List ids = new ArrayList<>(); - JsonNode node = OM.readTree(jsonString); - if (node.has("cveId")){ - ids.add(node.get("cveId").asText()); - } - return ids; + logger.info("Incoming CVE: '{}'", jsonString); + final JsonNode messageNode = OM.readTree(jsonString); + return messageNode.get("cveId").asText(); } catch (JsonProcessingException e) { - logger.error("Failed to parse list of ids from json string: {}", e.toString()); - return new ArrayList<>(); + logger.error("Failed to parse id from json string: {}", e.toString()); + return null; } } /** - * Takes in a list of CVE IDs and transforms it into a JSON string to be sent via RabbitMQ. - * - * @param cveIds list of CVE IDs - * @return single JSON string of all CVE IDs + * Generates the json string from the cveId string + * @param cveId + * @return */ - private String genJson(List cveIds) { + private String genJson(String cveId) { try { - return OM.writeValueAsString(cveIds); + Map cveJson = Map.of("cveId", cveId); + return OM.writeValueAsString(cveJson); } catch (JsonProcessingException e) { logger.error("Failed to convert list of ids to json string: {}", e.toString()); return ""; } } - private void sendDummyMessage(String queue, List cveIds) { + private void sendDummyMessage(String queue, String cveId) { try (Connection connection = factory.newConnection(); Channel channel = connection.createChannel()) { channel.queueDeclare(queue, false, false, false, null); - String message = genJson(cveIds); + String message = genJson(cveId); channel.basicPublish("", queue, null, message.getBytes(StandardCharsets.UTF_8)); logger.info("Successfully sent message:\n\"{}\"", message); } catch (IOException | TimeoutException e) { logger.error("Error sending message: {}", e.toString()); } @@ -177,22 +207,22 @@ private static List getIdsFromFile(String filename) { return new ArrayList<>(); } - private void sendDummyBatchedList(String queue, List messages, int batchSize) { - // 0 results in no batching - if(batchSize == 0) batchSize = messages.size(); - - // Get number of batches (including any partial batches) - final int numBatches = (int) Math.ceil((double) messages.size() / batchSize); - - // Determine if there is a partial batch - final boolean hasPartial = messages.size() % batchSize != 0; - - // Send batches - for (int i = 0; i < numBatches; i++) { - if(!hasPartial && i + 1 == numBatches) this.sendDummyMessage(queue, messages.subList(i * batchSize, messages.size() - 1)); - else this.sendDummyMessage(queue, messages.subList(i * batchSize, (i + 1) * batchSize)); - } - } +// private void sendDummyBatchedList(String queue, List messages, int batchSize) { +// // 0 results in no batching +// if(batchSize == 0) batchSize = messages.size(); +// +// // Get number of batches (including any partial batches) +// final int numBatches = (int) Math.ceil((double) messages.size() / batchSize); +// +// // Determine if there is a partial batch +// final boolean hasPartial = messages.size() % batchSize != 0; +// +// // Send batches +// for (int i = 0; i < numBatches; i++) { +// if(!hasPartial && i + 1 == numBatches) this.sendDummyMessage(queue, messages.subList(i * batchSize, messages.size() - 1)); +// else this.sendDummyMessage(queue, messages.subList(i * batchSize, (i + 1) * batchSize)); +// } +// } private static List getIdsFromJson(String path) { try { diff --git a/productnameextractor/src/main/java/productdetection/AffectedProductIdentifier.java b/productnameextractor/src/main/java/productdetection/AffectedProductIdentifier.java index 9b7b2f020..27e2830f6 100644 --- a/productnameextractor/src/main/java/productdetection/AffectedProductIdentifier.java +++ b/productnameextractor/src/main/java/productdetection/AffectedProductIdentifier.java @@ -117,48 +117,27 @@ public void initializeProductDetector(String resourceDir, String nlpDir, String * @param productNameDetector ProductDetector instance * @param cpeLookUp CpeLookup instance * @param vulnerability vulnerability being processed - * @param counterOfBadDescriptionCVEs counter of CVEs with bad descriptions - * @param counterOfSkippedCVEs counter of CVEs that have been skipped - * @param counterOfProcessedCVEs counter of CVEs that have been processed - * @param counterOfProcessedNERs counter of NERs that have been processed - * @param counterOfProcessedCPEs counter of CPEs that have been processed - * @param numOfProductsNotMappedToCPE counter of products that have not been mapped to a CPE - * @param numOfProductsMappedToCpe counter of products that have been mapped to a CPE - * @param totalNERTime timer for total time spent processing NERs - * @param totalCPETime timer for total time spent processing CPEs - * @param totalCVETime timer for total time spent processing CVEs */ - private void processVulnerability( + private int processVulnerability( ProductDetector productNameDetector, CpeLookUp cpeLookUp, - CompositeVulnerability vulnerability, - AtomicInteger counterOfBadDescriptionCVEs, - AtomicInteger counterOfSkippedCVEs, - AtomicInteger counterOfProcessedCVEs, - AtomicInteger counterOfProcessedNERs, - AtomicInteger counterOfProcessedCPEs, - AtomicInteger counterOfFailedCVEs, - AtomicInteger numOfProductsNotMappedToCPE, - AtomicInteger numOfProductsMappedToCpe, - AtomicLong totalNERTime, - AtomicLong totalCPETime, - AtomicLong totalCVETime + CompositeVulnerability vulnerability ) { String description = vulnerability.getDescription(); if (description == null || description.length() == 0) { - counterOfBadDescriptionCVEs.getAndIncrement(); - return; // skip the ones without a description +// counterOfBadDescriptionCVEs.getAndIncrement(); + return -2; // skip the ones without a description } // if a CVE did change, no need to extract products, assuming they are // already in DB!! if (vulnerability.getCveReconcileStatus() == CompositeVulnerability.CveReconcileStatus.DO_NOT_CHANGE) { - counterOfSkippedCVEs.getAndIncrement(); - return; +// counterOfSkippedCVEs.getAndIncrement(); + return -1; } - counterOfProcessedCVEs.getAndIncrement(); +// counterOfProcessedCVEs.getAndIncrement(); long startCVETime = System.currentTimeMillis(); try { @@ -187,8 +166,8 @@ private void processVulnerability( List productList = productNameDetector.getProductItems(descriptionWords); long nerTime = System.currentTimeMillis() - startNERTime; - counterOfProcessedNERs.getAndIncrement(); - totalNERTime.addAndGet(nerTime); +// counterOfProcessedNERs.getAndIncrement(); +// totalNERTime.addAndGet(nerTime); int numProductsFound = 0; @@ -197,11 +176,11 @@ private void processVulnerability( long startCPETime = System.currentTimeMillis(); List cpeIds = cpeLookUp.getCPEIds(productItem); long cpeTime = System.currentTimeMillis() - startCPETime; - totalCPETime.addAndGet(cpeTime); - counterOfProcessedCPEs.getAndIncrement(); +// totalCPETime.addAndGet(cpeTime); +// counterOfProcessedCPEs.getAndIncrement(); if (cpeIds == null || cpeIds.isEmpty()) { - numOfProductsNotMappedToCPE.getAndIncrement(); +// numOfProductsNotMappedToCPE.getAndIncrement(); logger.warn("Could not match CPEs for the predicted product name '{}'! | CVE-ID: {}", productItem.toString(), vulnerability.getCveId()); continue; } @@ -209,14 +188,16 @@ private void processVulnerability( // if CPE identified, add it as affected product for (String cpeID : cpeIds) { existingProducts.add(new AffectedProduct(vulnerability.getCveId(), cpeID, CpeLookUp.getNameFromCPEid(cpeID), CpeLookUp.getVersionFromCPEid(cpeID), CpeLookUp.getVendorFromCPEid(cpeID))); - numOfProductsMappedToCpe.getAndIncrement(); +// numOfProductsMappedToCpe.getAndIncrement(); numProductsFound++; } } - if(numProductsFound == 0) counterOfFailedCVEs.getAndIncrement(); - logger.info("Found {} Affected Product(s) for {} in {} seconds", numProductsFound, vulnerability.getCveId(), (double) (System.currentTimeMillis() - startCVETime) / 1000); + + if(numProductsFound == 0) return -3; + else return 0; +// counterOfFailedCVEs.getAndIncrement(); } } catch (Exception e) { @@ -225,7 +206,8 @@ private void processVulnerability( e.printStackTrace(); } - totalCVETime.addAndGet(System.currentTimeMillis() - startCVETime); +// totalCVETime.addAndGet(System.currentTimeMillis() - startCVETime); + return -3; } /** @@ -235,127 +217,125 @@ private void processVulnerability( * * @return list of affected products */ - public List identifyAffectedProducts(List vulnList) { - - int totalCVEtoProcess = vulnList.size(); - - logger.info("Starting to identify affected products for " + totalCVEtoProcess + " CVEs."); - long start = System.currentTimeMillis(); - - AtomicInteger numOfProductsMappedToCpe = new AtomicInteger(); - AtomicInteger numOfProductsNotMappedToCPE = new AtomicInteger(); - AtomicInteger counterOfProcessedNERs = new AtomicInteger(); - AtomicInteger counterOfProcessedCPEs = new AtomicInteger(); - AtomicInteger counterOfFailedCVEs = new AtomicInteger(); - AtomicInteger counterOfProcessedCVEs = new AtomicInteger(); - AtomicInteger counterOfSkippedCVEs = new AtomicInteger(); - AtomicInteger counterOfBadDescriptionCVEs = new AtomicInteger(); - AtomicLong totalNERTime = new AtomicLong(); - AtomicLong totalCPETime = new AtomicLong(); - AtomicLong totalCVETime = new AtomicLong(); - - final BlockingQueue workQueue = new ArrayBlockingQueue<>(vulnList.size()); - - final ThreadPoolExecutor executor = new ThreadPoolExecutor( - numThreads, - numThreads, - 15, - TimeUnit.SECONDS, - workQueue - ); - - executor.prestartAllCoreThreads(); - - for (int i = 0; i < vulnList.size(); i++) { - if(i >= totalCVEtoProcess){ - break; - } - CompositeVulnerability vulnerability = vulnList.get(i); - try { - if(!workQueue.offer(() -> processVulnerability( - productDetector, - cpeLookUp, - vulnerability, - counterOfBadDescriptionCVEs, - counterOfSkippedCVEs, - counterOfProcessedCVEs, - counterOfProcessedNERs, - counterOfProcessedCPEs, - counterOfFailedCVEs, - numOfProductsNotMappedToCPE, - numOfProductsMappedToCpe, - totalNERTime, - totalCPETime, - totalCVETime - ))) throw new Exception(); - } catch (Exception e) { - logger.error("Failed to add {} to the work queue: {}", vulnerability.getCveId(), e.toString()); - totalCVEtoProcess--; - counterOfSkippedCVEs.incrementAndGet(); - } - } - - executor.shutdown(); - - final int timeout = 15; - final int logFrequency = 60; - long secondsWaiting = 0; - int numCVEsProcessed = 0; - int lastNumCVEs = totalCVEtoProcess; - try { - while(!executor.awaitTermination(timeout, TimeUnit.SECONDS)) { - secondsWaiting += timeout; - - // Every minute, log a progress update - if(secondsWaiting % logFrequency == 0) { - - // Determine number of CVEs processed - final int currNumCVEs = workQueue.size(); // Current number of remaining CVEs - final int deltaNumCVEs = lastNumCVEs - currNumCVEs; // Change in CVEs since last progress update - - // Sum number processed - numCVEsProcessed += deltaNumCVEs; - - // Calculate rate, avg rate, and remaining time - final double rate = (double) deltaNumCVEs / logFrequency; // CVEs/sec - final double avgRate = (double) numCVEsProcessed / secondsWaiting; // CVEs/sec - final double remainingAvgTime = currNumCVEs / rate; // CVEs / CVEs/sec = remaining seconds - - // Log stats - logger.info( - "{} out of {} CVEs processed (SP: {} CVEs/sec | AVG SP: {} CVEs/sec | Est time remaining: {} minutes ({} seconds))...", - totalCVEtoProcess - currNumCVEs, - totalCVEtoProcess, - Math.floor(rate * 100) / 100, - Math.floor(avgRate * 100) / 100, - Math.floor(remainingAvgTime / 60 * 100) / 100, - Math.floor(remainingAvgTime * 100) / 100 - ); - - // Update lastNumCVEs - lastNumCVEs = currNumCVEs; - } - } - } catch (Exception ex) { - logger.error("Product extraction failed: {}", ex.toString()); - List remainingTasks = executor.shutdownNow(); - logger.error("{} tasks not executed", remainingTasks.size()); - } - - logger.info("Successfully processed {} out of {} CVEs!", counterOfProcessedCVEs, totalCVEtoProcess); - logger.info("Found products for {} CVE(s) | Failed to find products for {} CVE(s) | Skipped {} CVE(s)", - counterOfProcessedCVEs.get() - counterOfFailedCVEs.get(), counterOfFailedCVEs, counterOfSkippedCVEs); - - AtomicInteger count = new AtomicInteger(); - vulnList.stream().map(v -> v.getAffectedProducts().size()).forEach(count::addAndGet); - logger.info("Found {} affected products ({} unique excluding versions) from {} CVEs in {} seconds", count, cpeLookUp.getUniqueCPECount(), totalCVEtoProcess, Math.floor(((double) (System.currentTimeMillis() - start) / 1000) * 100) / 100); + public List identifyAffectedProducts(CompositeVulnerability vuln) { + logger.info("Starting to identify affected products for CVE '{}'", vuln.getCveId()); + +// int totalCVEtoProcess = vulnList.size(); +// long start = System.currentTimeMillis(); +// +// AtomicInteger numOfProductsMappedToCpe = new AtomicInteger(); +// AtomicInteger numOfProductsNotMappedToCPE = new AtomicInteger(); +// AtomicInteger counterOfProcessedNERs = new AtomicInteger(); +// AtomicInteger counterOfProcessedCPEs = new AtomicInteger(); +// AtomicInteger counterOfFailedCVEs = new AtomicInteger(); +// AtomicInteger counterOfProcessedCVEs = new AtomicInteger(); +// AtomicInteger counterOfSkippedCVEs = new AtomicInteger(); +// AtomicInteger counterOfBadDescriptionCVEs = new AtomicInteger(); +// AtomicLong totalNERTime = new AtomicLong(); +// AtomicLong totalCPETime = new AtomicLong(); +// AtomicLong totalCVETime = new AtomicLong(); +// +// final BlockingQueue workQueue = new ArrayBlockingQueue<>(vulnList.size()); +// +// final ThreadPoolExecutor executor = new ThreadPoolExecutor( +// numThreads, +// numThreads, +// 15, +// TimeUnit.SECONDS, +// workQueue +// ); +// +// executor.prestartAllCoreThreads(); +// +// for (int i = 0; i < vulnList.size(); i++) { +// if(i >= totalCVEtoProcess){ +// break; +// } +// CompositeVulnerability vulnerability = vulnList.get(i); +// try { +// if(!workQueue.offer(() -> processVulnerability( +// productDetector, +// cpeLookUp, +// vulnerability, +// counterOfBadDescriptionCVEs, +// counterOfSkippedCVEs, +// counterOfProcessedCVEs, +// counterOfProcessedNERs, +// counterOfProcessedCPEs, +// counterOfFailedCVEs, +// numOfProductsNotMappedToCPE, +// numOfProductsMappedToCpe, +// totalNERTime, +// totalCPETime, +// totalCVETime +// ))) throw new Exception(); +// } catch (Exception e) { +// logger.error("Failed to add {} to the work queue: {}", vulnerability.getCveId(), e.toString()); +// totalCVEtoProcess--; +// counterOfSkippedCVEs.incrementAndGet(); +// } +// } +// +// executor.shutdown(); + +// final int timeout = 15; +// final int logFrequency = 60; +// long secondsWaiting = 0; +// int numCVEsProcessed = 0; +// int lastNumCVEs = totalCVEtoProcess; +// try { +// while(!executor.awaitTermination(timeout, TimeUnit.SECONDS)) { +// secondsWaiting += timeout; +// +// // Every minute, log a progress update +// if(secondsWaiting % logFrequency == 0) { +// +// // Determine number of CVEs processed +// final int currNumCVEs = workQueue.size(); // Current number of remaining CVEs +// final int deltaNumCVEs = lastNumCVEs - currNumCVEs; // Change in CVEs since last progress update +// +// // Sum number processed +// numCVEsProcessed += deltaNumCVEs; +// +// // Calculate rate, avg rate, and remaining time +// final double rate = (double) deltaNumCVEs / logFrequency; // CVEs/sec +// final double avgRate = (double) numCVEsProcessed / secondsWaiting; // CVEs/sec +// final double remainingAvgTime = currNumCVEs / rate; // CVEs / CVEs/sec = remaining seconds +// +// // Log stats +// logger.info( +// "{} out of {} CVEs processed (SP: {} CVEs/sec | AVG SP: {} CVEs/sec | Est time remaining: {} minutes ({} seconds))...", +// totalCVEtoProcess - currNumCVEs, +// totalCVEtoProcess, +// Math.floor(rate * 100) / 100, +// Math.floor(avgRate * 100) / 100, +// Math.floor(remainingAvgTime / 60 * 100) / 100, +// Math.floor(remainingAvgTime * 100) / 100 +// ); +// +// // Update lastNumCVEs +// lastNumCVEs = currNumCVEs; +// } +// } +// } catch (Exception ex) { +// logger.error("Product extraction failed: {}", ex.toString()); +// List remainingTasks = executor.shutdownNow(); +// logger.error("{} tasks not executed", remainingTasks.size()); +// } + +// logger.info("Successfully processed {} out of {} CVEs!", counterOfProcessedCVEs, totalCVEtoProcess); +// logger.info("Found products for {} CVE(s) | Failed to find products for {} CVE(s) | Skipped {} CVE(s)", +// counterOfProcessedCVEs.get() - counterOfFailedCVEs.get(), counterOfFailedCVEs, counterOfSkippedCVEs); +// +// AtomicInteger count = new AtomicInteger(); +// vulnList.stream().map(v -> v.getAffectedProducts().size()).forEach(count::addAndGet); +// logger.info("Found {} affected products ({} unique excluding versions) from {} CVEs in {} seconds", count, cpeLookUp.getUniqueCPECount(), totalCVEtoProcess, Math.floor(((double) (System.currentTimeMillis() - start) / 1000) * 100) / 100); + + final int result = processVulnerability(productDetector, cpeLookUp, vuln); List affectedProducts = new ArrayList<>(); - for (CompositeVulnerability vulnerability : vulnList) { - if (vulnerability.getCveReconcileStatus() == CompositeVulnerability.CveReconcileStatus.DO_NOT_CHANGE) - continue; // skip the ones that are not changed! - affectedProducts.addAll(vulnerability.getAffectedProducts()); - } + if (vuln.getCveReconcileStatus() == CompositeVulnerability.CveReconcileStatus.DO_NOT_CHANGE) + affectedProducts.addAll(vuln.getAffectedProducts()); return affectedProducts; } diff --git a/productnameextractor/src/test/java/db/DatabaseHelperTest.java b/productnameextractor/src/test/java/db/DatabaseHelperTest.java index 04046212d..b2632c041 100644 --- a/productnameextractor/src/test/java/db/DatabaseHelperTest.java +++ b/productnameextractor/src/test/java/db/DatabaseHelperTest.java @@ -163,9 +163,11 @@ public void getAllCompositeVulnerabilitiesTest() throws SQLException { } // @Test - public void getSpecificCompositeVulnerabilitiesTest() throws SQLException{ + public void getSpecificCompositeVulnerabilityTest() throws SQLException{ List cveIds = new ArrayList<>(); + List descriptions = new ArrayList<>(); + // Create test data String cveId1 = "CVE-2021-20105"; String description1 = "Machform prior to version 16 is vulnerable to an open redirect in Safari_init.php due to an improperly sanitized 'ref' parameter."; @@ -175,10 +177,15 @@ public void getSpecificCompositeVulnerabilitiesTest() throws SQLException{ String cveId3 = "CVE-2019-3915"; String description3 = "Authentication Bypass by Capture-replay vulnerability in Verizon Fios Quantum Gateway (G1100) firmware version 02.01.00.05 allows an unauthenticated attacker with adjacent network access to intercept and replay login requests to gain access to the administrative web interface."; + // Store in list cveIds.add(cveId1); cveIds.add(cveId2); cveIds.add(cveId3); + descriptions.add(description1); + descriptions.add(description2); + descriptions.add(description3); + // Mock the database interactions when(conn.prepareStatement(anyString())).thenReturn(pstmt); when(pstmt.executeQuery()).thenReturn(res); @@ -186,16 +193,14 @@ public void getSpecificCompositeVulnerabilitiesTest() throws SQLException{ when(res.getInt("vuln_id")).thenReturn(1, 2, 3); when(res.getString("description")).thenReturn(description1, description2, description3); - List vulnList = dbh.getSpecificCompositeVulnerabilities(cveIds); - assertEquals(vulnList.size(), cveIds.size()); - - CompositeVulnerability vuln1 = vulnList.get(0); - CompositeVulnerability vuln2 = vulnList.get(1); - CompositeVulnerability vuln3 = vulnList.get(2); - - assertEquals(vuln1.getDescription(), description1); - assertEquals(vuln2.getDescription(), description2); - assertEquals(vuln3.getDescription(), description3); + // Test vulns + for (int i = 0; i < cveIds.size(); i++) { + String cveId = cveIds.get(i); + String description = descriptions.get(i); + CompositeVulnerability vuln = dbh.getSpecificCompositeVulnerability(cveId); + assertNotNull(vuln); + assertEquals(vuln.getDescription(), description); + } } // @Test diff --git a/productnameextractor/src/test/java/env/ProductNameExtractorEnvVarsTest.java b/productnameextractor/src/test/java/env/ProductNameExtractorEnvVarsTest.java index c7bc69e42..269c3b879 100644 --- a/productnameextractor/src/test/java/env/ProductNameExtractorEnvVarsTest.java +++ b/productnameextractor/src/test/java/env/ProductNameExtractorEnvVarsTest.java @@ -79,7 +79,8 @@ public void initializeAndGetEnvVarsTest(){ assertEquals("guest", ProductNameExtractorEnvVars.getRabbitUsername()); assertEquals("guest", ProductNameExtractorEnvVars.getRabbitPassword()); assertEquals("RECONCILER_OUT", ProductNameExtractorEnvVars.getRabbitInputQueue()); - assertEquals("PNE_OUT", ProductNameExtractorEnvVars.getRabbitOutputQueue()); + assertEquals("PNE_OUT_PATCH", ProductNameExtractorEnvVars.getRabbitPatchfinderOutputQueue()); + assertEquals("PNE_OUT_FIX", ProductNameExtractorEnvVars.getRabbitFixfinderOutputQueue()); } } diff --git a/productnameextractor/src/test/java/productdetection/AffectedProductIdentifierTest.java b/productnameextractor/src/test/java/productdetection/AffectedProductIdentifierTest.java index 286ed7891..0b1302951 100644 --- a/productnameextractor/src/test/java/productdetection/AffectedProductIdentifierTest.java +++ b/productnameextractor/src/test/java/productdetection/AffectedProductIdentifierTest.java @@ -74,7 +74,7 @@ public void affectedProductIdentifierTest() { } // Identify releases - affectedProductIdentifier.identifyAffectedProducts(vulnList); + affectedProductIdentifier.identifyAffectedProducts(v); System.out.println(v.getAffectedProducts()); @@ -119,7 +119,7 @@ public void testIdentifyAffectedProducts() throws Exception { AffectedProductIdentifier identifier = new AffectedProductIdentifier(2, vulnList); // Simulate the method call - List affectedProducts = identifier.identifyAffectedProducts(vulnList); + List affectedProducts = identifier.identifyAffectedProducts(vulnerability); // Add assertions based on the expected behavior of the method assertEquals(affectedProducts.size(), 0);