Versioning enhancements job streaming merge

crawler/src/main/java/edu/rit/se/nvip/CrawlerMain.java

@@ -153,7 +153,7 @@ public void run(){
                 log.info("CVE: {}:\n", cveId);
                 for (RawVulnerability vuln: crawledCVEs.get(cveId)) {
                     String description = vuln.getDescription().length() > 100 ? vuln.getDescription().substring(0, 100) + "...": vuln.getDescription();
-                    log.info("[{} | {}]\n", vuln.getSourceURL(), description);
+                    log.info("[{} | {}]\n", vuln.getSourceUrl(), description);
                 }
             }
         } else {
@@ -474,8 +474,8 @@ private int cvesToCsv(HashMap<String, ArrayList<RawVulnerability>> crawledCVEs){
             for (ArrayList<RawVulnerability> vulnList : crawledCVEs.values()) {
                 for (RawVulnerability vuln : vulnList) {
                     String desc = vuln.getDescription().replace("\r\n", ". ").replace("\n", ". ").replace("\r", ". ").replace("\t", " ");
-                    String[] data = {vuln.getCveId(), desc, vuln.getCreateDate(), vuln.getPublishDate(), 
-                        vuln.getLastModifiedDate(), vuln.getSourceURL(), vuln.getSourceType()};
+                    String[] data = {vuln.getCveId(), desc, vuln.getCreateDateString(), vuln.getPublishDateString(),
+                        vuln.getLastModifiedDateString(), vuln.getSourceUrl(), vuln.getSourceType().type};
                     writer.writeNext(data, false);
                     lineCount++;
                 }
@@ -530,19 +530,19 @@ private void updateSourceTypes(HashMap<String, ArrayList<RawVulnerability>> craw
         // For each raw CVE,
         for (String cveId: crawledCves.keySet()) {
             for (RawVulnerability vuln: crawledCves.get(cveId)) {
-                if(vuln.getSourceURL() == null || vuln.getSourceURL().equals("")){
+                if(vuln.getSourceUrl() == null || vuln.getSourceUrl().equals("")){
                     vuln.setSourceType("other");
                     continue;
                 }
 
                 // Set source type if the URL is listed in the types file
                 // Otherwise, just set the source type to 'other'
                 try{
-                    URL sourceURL = new URL(vuln.getSourceURL());
+                    URL sourceURL = new URL(vuln.getSourceUrl());
                     vuln.setSourceType(sourceTypes.get(sourceURL.getHost()));
                 }
                 catch(MalformedURLException e){
-                    log.warn("Bad sourceURL {}: {}", vuln.getSourceURL(), e.toString());
+                    log.warn("Bad sourceURL {}: {}", vuln.getSourceUrl(), e.toString());
                 }
 
                 if(vuln.getSourceType() == null){

crawler/src/main/java/edu/rit/se/nvip/crawler/CveCrawler.java

@@ -33,8 +33,6 @@
 import edu.rit.se.nvip.db.model.RawVulnerability;
 
 import lombok.extern.slf4j.Slf4j;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -147,7 +145,7 @@ public void visit(Page page) {
 			} else {
 				for (RawVulnerability vulnerability : vulnerabilityList) {
 					if (vulnerability.getCveId().isEmpty()) {
-						log.info("A cve found by the {} parser at the URL {} has an empty cve_id and will not be inserted", vulnerability.getParserType(), vulnerability.getSourceURL());
+						log.info("A cve found by the {} parser at the URL {} has an empty cve_id and will not be inserted", vulnerability.getParserType(), vulnerability.getSourceUrl());
 						continue;
 					}
 					if (foundCVEs.get(vulnerability.getCveId()) != null) {

crawler/src/test/java/edu/rit/se/nvip/crawler/github/PyPAGithubScraperTest.java

@@ -22,7 +22,7 @@ public void testPyPA() {
 
         assertEquals(vuln.getCveId(), "CVE-2017-16763");
         assertTrue(vuln.getDescription().contains("An exploitable vulnerability exists in the YAML parsing functionality in config.py in Confire 0.2.0"));
-        assertEquals(vuln.getPublishDate(), "2017-11-10 09:29:00");
-        assertEquals(vuln.getLastModifiedDate(), "2021-08-25 04:29:57");
+        assertEquals("2017-11-10 09:29:00", vuln.getPublishDateString());
+        assertEquals("2021-08-25 04:29:57", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/ABBParserTest.java

@@ -23,7 +23,7 @@ public void testABBDownloadAndParse() {
         RawVulnerability vuln = list.get(0);
         assertEquals("CVE-2023-0580", vuln.getCveId());
         assertTrue(vuln.getDescription().contains("An attacker who successfully exploited this vulnerability could gain access to the protected application"));
-        assertEquals("2023-03-27 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-27 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-03-27 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-27 00:00:00", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AcronisParserTest.java

@@ -47,8 +47,8 @@ public void testAcronisSingle() {
         RawVulnerability vuln = list.get(0);
         assertEquals("CVE-2022-3405", vuln.getCveId());
         assertTrue(vuln.getDescription().contains("Code execution and sensitive information disclosure due"));
-        assertEquals("2022-11-17 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-09 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-11-17 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-09 00:00:00", vuln.getLastModifiedDateString());
     }
 
     // Multiple CVE, nothing above title, located in description
@@ -63,8 +63,8 @@ public void testAcronisMultiple() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2022-3602");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("OpenSSL project team released a security advisory"));
-        assertEquals("2022-11-01 00:00:00", vuln.getPublishDate());
-        assertEquals("2022-11-01 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-11-01 00:00:00", vuln.getPublishDateString());
+        assertEquals("2022-11-01 00:00:00", vuln.getLastModifiedDateString());
     }
 
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AdobeParserTest.java

@@ -47,8 +47,8 @@ public void testAdobe() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2023-22247");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("Adobe has released a security update for Adobe Commerce and Magento Open Source."));
-        assertEquals("2023-03-14 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-14 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-03-14 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-14 00:00:00", vuln.getLastModifiedDateString());
 
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AliasRoboParserTest.java

@@ -48,8 +48,8 @@ public void testAliasObj() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2023-24012");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("Attacker can arbitrarily craft malicious DDS Participants"));
-        assertEquals("2023-02-25 04:55:00", vuln.getPublishDate());
-        assertEquals("2023-02-25 04:55:00", vuln.getLastModifiedDate());
+        assertEquals("2023-02-25 04:55:00", vuln.getPublishDateString());
+        assertEquals("2023-02-25 04:55:00", vuln.getLastModifiedDateString());
 
     }
 
@@ -65,8 +65,8 @@ public void testAlias() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2020-10292");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("Visual Components (owned by KUKA) is a robotic simulator that allows"));
-        assertEquals("2020-11-06 04:26:00", vuln.getPublishDate());
-        assertEquals("2020-11-06 04:26:00", vuln.getLastModifiedDate());
+        assertEquals("2020-11-06 04:26:00", vuln.getPublishDateString());
+        assertEquals("2020-11-06 04:26:00", vuln.getLastModifiedDateString());
 
     }
 

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AmpereParserTest.java

@@ -45,7 +45,7 @@ public void testAmpere() {
         assertEquals(6, list.size());
         RawVulnerability vuln = getVulnerability(list, "CVE-2022-46892");
         assertTrue(vuln.getDescription().contains("A Root complex is typically disabled during boot via the BIOS"));
-        assertEquals("2023-02-14 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-02-14 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-02-14 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-02-14 00:00:00", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AndroidParserTest.java

@@ -49,8 +49,8 @@ public void testAndroidBulletin() {
         assertEquals("CVE-2023-20933", vuln.getCveId());
         assertTrue(vuln.getDescription().contains("local escalation of privilege with no additional execution privileges needed"));
         assertFalse(vuln.getDescription().contains("lead to remote code execution with no additional"));
-        assertEquals("2023-02-06 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-02-08 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-02-06 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-02-08 00:00:00", vuln.getLastModifiedDateString());
     }
 
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AnquankeParserTest.java

@@ -24,7 +24,7 @@ public void testAnquankeParser() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2020-5764");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("安卓MX Player播放器路径穿越和代码执行漏洞"));
-        assertEquals("2020-07-10 16:30:16", vuln.getPublishDate());
-        assertEquals("2020-07-10 16:30:16", vuln.getLastModifiedDate());
+        assertEquals("2020-07-10 16:30:16", vuln.getPublishDateString());
+        assertEquals("2020-07-10 16:30:16", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AristaParserTest.java

@@ -24,8 +24,8 @@ public void testAristaSingle() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2023-24546");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("This advisory impacts the Arista CloudVision Portal products when run on-premise"));
-        assertEquals("2023-03-07 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-07 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-03-07 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-07 00:00:00", vuln.getLastModifiedDateString());
     }
 
 
@@ -40,7 +40,7 @@ public void testAristaMultiple() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2021-28509");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("This advisory documents the impact of an internally found vulnerability in Arista EOS state streaming telemetry agent TerminAttr and OpenConfig transport protocols."));
-        assertEquals("2022-05-25 00:00:00", vuln.getPublishDate());
-        assertEquals("2022-05-27 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-05-25 00:00:00", vuln.getPublishDateString());
+        assertEquals("2022-05-27 00:00:00", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/ArubaParserTest.java

@@ -46,8 +46,8 @@ public void testArubaSingle() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2022-23678");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("A vulnerability exists in the Aruba VIA client for Microsoft"));
-        assertEquals("2022-07-26 00:00:00", vuln.getPublishDate());
-        assertEquals("2022-08-19 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-07-26 00:00:00", vuln.getPublishDateString());
+        assertEquals("2022-08-19 00:00:00", vuln.getLastModifiedDateString());
     }
 
 
@@ -63,8 +63,8 @@ public void testArubaMultiple() {
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("further privileges on the ClearPass instance"));
         assertFalse(vuln.getDescription().contains("execute arbitrary script code in a victim's"));
-        assertEquals("2023-03-14 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-14 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-03-14 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-14 00:00:00", vuln.getLastModifiedDateString());
     }
 
     @Test

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AsustorParserTest.java

@@ -33,8 +33,8 @@ public void testAsustorParserSingle() {
         RawVulnerability vuln = getVulnerability(list, "CVE-2022-0847");
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("A flaw was found in the way the \"flags\" member of the new pipe buffer structure was lacking prope"));
-        assertEquals("2022-03-11 00:00:00", vuln.getPublishDate());
-        assertEquals("2022-07-07 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-03-11 00:00:00", vuln.getPublishDateString());
+        assertEquals("2022-07-07 00:00:00", vuln.getLastModifiedDateString());
     }
 
     @Test
@@ -49,8 +49,8 @@ public void testAsustorParserMultiple() {
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("The vulnerability affects all RSA padding modes: PKCS#1 v1.5, RSA-OEAP and RSASVE."));
         assertFalse(vuln.getDescription().contains("This could be exploited by an attacker who has the ability to supply malicious PEM files for parsing to achieve a denial of service attack."));
-        assertEquals("2023-03-31 00:00:00", vuln.getPublishDate());
-        assertEquals("2023-03-31 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2023-03-31 00:00:00", vuln.getPublishDateString());
+        assertEquals("2023-03-31 00:00:00", vuln.getLastModifiedDateString());
     }
 
 

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AtlassianParserTest.java

@@ -46,8 +46,8 @@ public void testAtlassianSingleNoDesc() {
         assertEquals("CVE-2022-36804", vuln.getCveId());
         assertTrue(vuln.getDescription().contains("command injection vulnerability in multiple API endpoints"));
         assertFalse(vuln.getDescription().contains("evaluate its applicability to your own IT environment"));
-        assertEquals("2022-08-24 10:00:00", vuln.getPublishDate());
-        assertEquals("2022-08-24 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-08-24 10:00:00", vuln.getPublishDateString());
+        assertEquals("2022-08-24 00:00:00", vuln.getLastModifiedDateString());
 
     }
 
@@ -69,8 +69,8 @@ public void testAtlassianMultipleNoDesc() {
         assertFalse(vuln1.getDescription().contains(desc2));
         assertTrue(vuln2.getDescription().contains(desc2));
         assertFalse(vuln2.getDescription().contains(desc1));
-        assertEquals("2023-02-15 10:00:00", vuln1.getPublishDate());
-        assertEquals("2023-02-17 00:00:00", vuln2.getLastModifiedDate());
+        assertEquals("2023-02-15 10:00:00", vuln1.getPublishDateString());
+        assertEquals("2023-02-17 00:00:00", vuln2.getLastModifiedDateString());
     }
 
     @Test
@@ -84,8 +84,8 @@ public void testAtlassianSingleWithDesc() {
         RawVulnerability vuln = list.get(0);
         assertEquals("CVE-2019-15006", vuln.getCveId());
         assertTrue(vuln.getDescription().contains("An attacker could perform the described attack by denying their victim access"));
-        assertEquals("2019-12-18 10:00:00", vuln.getPublishDate());
-        assertEquals("2020-01-08 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2019-12-18 10:00:00", vuln.getPublishDateString());
+        assertEquals("2020-01-08 00:00:00", vuln.getLastModifiedDateString());
     }
 
     @Test
@@ -115,7 +115,7 @@ public void testAtlassianMultipleWithDesc() {
         assertTrue(vuln3.getDescription().contains(desc3));
         assertFalse(vuln3.getDescription().contains(desc1));
         assertFalse(vuln3.getDescription().contains(desc2));
-        assertEquals("2020-01-15 10:00:00", vuln1.getPublishDate());
-        assertEquals("2020-01-28 00:00:00", vuln2.getLastModifiedDate());
+        assertEquals("2020-01-15 10:00:00", vuln1.getPublishDateString());
+        assertEquals("2020-01-28 00:00:00", vuln2.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/AutodeskParserTest.java

@@ -52,8 +52,8 @@ public void testAutodesk() {
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("Expat"));
         assertTrue(vuln.getDescription().contains("Autodesk products leveraging internal components"));
-        assertEquals("2022-07-28 00:00:00", vuln.getLastModifiedDate());
-        assertEquals("2022-10-12 00:00:00", vuln.getPublishDate());
+        assertEquals("2022-07-28 00:00:00", vuln.getLastModifiedDateString());
+        assertEquals("2022-10-12 00:00:00", vuln.getPublishDateString());
 
         vuln = getVulnerability(list, "CVE-2021-22947");
         assertNotNull(vuln);
@@ -69,7 +69,7 @@ public void testAutodeskMulti() {
         assertNotNull(vuln);
         assertTrue(vuln.getDescription().contains("A maliciously crafted PCT"));
         assertFalse(vuln.getDescription().contains("Applications and services that utilize"));
-        assertEquals("2022-12-14 00:00:00", vuln.getPublishDate());
-        assertEquals("2022-12-14 00:00:00", vuln.getLastModifiedDate());
+        assertEquals("2022-12-14 00:00:00", vuln.getPublishDateString());
+        assertEquals("2022-12-14 00:00:00", vuln.getLastModifiedDateString());
     }
 }

crawler/src/test/java/edu/rit/se/nvip/crawler/htmlparser/BoschSecurityParserTest.java

@@ -59,8 +59,8 @@ public void testBoschSecurityParser() throws IOException {
                 vuln1.getDescription());
         assertEquals("Race condition in the __find_get_block_slow function in the ISO9660 filesystem in Linux 2.6.18 and possibly other versions allows local users to cause a denial of service (infinite loop) by mounting a crafted ISO9660 filesystem containing malformed data structures.",
                 vuln2.getDescription());
-        assertEquals("2022-11-23 00:00:00", vuln1.getPublishDate());
-        assertEquals("2022-11-23 00:00:00", vuln1.getLastModifiedDate());
+        assertEquals("2022-11-23 00:00:00", vuln1.getPublishDateString());
+        assertEquals("2022-11-23 00:00:00", vuln1.getLastModifiedDateString());
 
     }
 }