Skip to content

Commit

Permalink
Replaced logger with @slf4j annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
@ctevse
ctevse committed Oct 20, 2023
1 parent 317a5d0 commit 2cca00b
Showing 1 changed file with 50 additions and 51 deletions.
101 changes: 50 additions & 51 deletions crawler/src/main/java/edu/rit/se/nvip/CrawlerMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import lombok.extern.slf4j.Slf4j;

import java.io.*;
import java.net.MalformedURLException;
Expand All @@ -25,9 +24,9 @@
import java.lang.reflect.Modifier;


@Slf4j
public class CrawlerMain {

private static final Logger logger = LogManager.getLogger(CrawlerMain.class);
private final RawDescriptionRepository rawDescriptionRepository;

private final Map<String, Object> crawlerVars = new HashMap<>();
Expand All @@ -48,7 +47,7 @@ public static void main(String[] args) {
// get sources from the seeds file or the database
DatabaseHelper databaseHelper = DatabaseHelper.getInstance();
if (!databaseHelper.testDbConnection()) {
logger.error("Error in database connection! Please check if the database configured in DB Envvars is up and running!");
log.error("Error in database connection! Please check if the database configured in DB Envvars is up and running!");
System.exit(1);
}

Expand All @@ -68,7 +67,7 @@ public void run(){
ConnectionFactory connectionFactory = getConnectionFactory();

if (!this.testMQConnection(connectionFactory)) {
logger.error("ERROR: Failed to connect to RabbitMQ server on {}:{}/{}",
log.error("ERROR: Failed to connect to RabbitMQ server on {}:{}/{}",
dataVars.get("mqHost"),
dataVars.get("mqVirtualHost"),
dataVars.get("mqPort"));
Expand All @@ -77,14 +76,14 @@ public void run(){

boolean crawlerTestMode = (boolean) crawlerVars.get("testMode");
if (crawlerTestMode)
logger.info("Starting Crawler IN TEST MODE using {}",
log.info("Starting Crawler IN TEST MODE using {}",
((String)crawlerVars.get("seedFileDir")).split("/")[((String)crawlerVars.get("seedFileDir")).split("/").length - 1]);
else
logger.info("Starting Crawler...");
log.info("Starting Crawler...");

// TODO: Move this to reconciler/processor
if ((Boolean) dataVars.get("refreshNvdList")) {
logger.info("Refreshing NVD CVE List");
log.info("Refreshing NVD CVE List");
// new NvdCveController().updateNvdDataTable((String) dataVars.get("nvdUrl"));
}

Expand All @@ -103,12 +102,12 @@ public void run(){
{
String domain = reader.nextLine();
if (domain.length() > 5) {
//logger.info("Added {} to whitelist", domain);
//log.info("Added {} to whitelist", domain);
whiteList.add(domain);
}
}
} catch (FileNotFoundException e) {
logger.error("Unable to read whitelist file");
log.error("Unable to read whitelist file");
throw new RuntimeException(e);
}

Expand All @@ -120,11 +119,11 @@ public void run(){
}

long crawlEndTime = System.currentTimeMillis();
logger.info("Crawler Finished\nTime: {} seconds", (crawlEndTime - crawlStartTime) / 1000.0);
log.info("Crawler Finished\nTime: {} seconds", (crawlEndTime - crawlStartTime) / 1000.0);

// Merge CVEs found in python GitHub with CVEs that were crawled
if (!crawlerTestMode) {
logger.info("Merging Python CVEs with Crawled CVEs");
log.info("Merging Python CVEs with Crawled CVEs");
for (String pyCve: pyCves.keySet()) {
if (crawledCVEs.containsKey(pyCve)) {
crawledCVEs.get(pyCve).add(pyCves.get(pyCve));
Expand All @@ -139,31 +138,31 @@ public void run(){
// Update the source types for the found CVEs and insert new entries into the rawdescriptions table
updateSourceTypes(crawledCVEs);

logger.info("Outputting CVEs to a CSV and JSON");
log.info("Outputting CVEs to a CSV and JSON");
int linesWritten = cvesToCsv(crawledCVEs);
logger.info("Wrote {} lines to {}", linesWritten, (String)crawlerVars.get("testOutputDir")+"/test_output.csv");
log.info("Wrote {} lines to {}", linesWritten, (String)crawlerVars.get("testOutputDir")+"/test_output.csv");
cvesToJson(crawledCVEs);
logger.info("Done!");
log.info("Done!");

// Output results in testmode
// Store raw data in DB otherwise
if (crawlerTestMode) {
logger.info("CVEs Found: {}", crawledCVEs.size());
log.info("CVEs Found: {}", crawledCVEs.size());
for (String cveId: crawledCVEs.keySet()) {
logger.info("CVE: {}:\n", cveId);
log.info("CVE: {}:\n", cveId);
for (RawVulnerability vuln: crawledCVEs.get(cveId)) {
String description = vuln.getDescription().length() > 100 ? vuln.getDescription().substring(0, 100) + "...": vuln.getDescription();
logger.info("[{} | {}]\n", vuln.getSourceURL(), description);
log.info("[{} | {}]\n", vuln.getSourceURL(), description);
}
}
} else {
// Update the source types for the found CVEs and insert new entries into the rawdescriptions table
logger.info("Done! Preparing to insert all raw data found in this run!");
log.info("Done! Preparing to insert all raw data found in this run!");
insertRawCVEsAndPrepareMessage(crawledCVEs);
logger.info("Raw data inserted and Message sent successfully!");
log.info("Raw data inserted and Message sent successfully!");
}

logger.info("Done!");
log.info("Done!");
}

/**
Expand Down Expand Up @@ -281,7 +280,7 @@ private void addEnvvarString(Map<String, Object> envvarMap, String envvarName, S
if (envvarValue != null && !envvarValue.isEmpty()) {
envvarMap.put(envvarName, envvarValue);
} else {
logger.warn(warningMessage);
log.warn(warningMessage);
envvarMap.put(envvarName, defaultValue);
}
}
Expand All @@ -299,7 +298,7 @@ private void addEnvvarBool(Map<String, Object> envvarMap, String envvarName, Str
if (envvarValue != null && !envvarValue.isEmpty()) {
envvarMap.put(envvarName, Boolean.parseBoolean(envvarValue));
} else {
logger.warn(warningMessage);
log.warn(warningMessage);
envvarMap.put(envvarName, defaultValue);
}
}
Expand All @@ -319,12 +318,12 @@ private void addEnvvarInt(Map<String, Object> envvarMap, String envvarName, Stri
try {
envvarMap.put(envvarName, Integer.parseInt(envvarValue));
} catch (NumberFormatException e) {
logger.warn("WARNING: Variable: {} = {} is not an integer, using 1 as default value", ennvarName
log.warn("WARNING: Variable: {} = {} is not an integer, using 1 as default value", ennvarName
, defaultValue);
envvarMap.put(envvarName, defaultValue);
}
} else {
logger.warn(warningMessage);
log.warn(warningMessage);
envvarMap.put(envvarName, defaultValue);
}
}
Expand All @@ -341,27 +340,27 @@ private void checkDataDirs() {
String testOutputDir = (String) crawlerVars.get("testOutputDir");

if (!new File(dataDir).exists()) {
logger.error("The data dir provided does not exist, check the 'NVIP_DATA_DIR' key in the env.list file, currently configured data dir is {}", dataDir);
log.error("The data dir provided does not exist, check the 'NVIP_DATA_DIR' key in the env.list file, currently configured data dir is {}", dataDir);
System.exit(1);
}

if (!new File(crawlerSeeds).exists()) {
logger.error("The crawler seeds path provided: {} does not exits!", crawlerSeeds);
log.error("The crawler seeds path provided: {} does not exits!", crawlerSeeds);
System.exit(1);
}

if (!new File(whitelistFileDir).exists()) {
logger.error("The whitelist domain file path provided: {} does not exits!", whitelistFileDir);
log.error("The whitelist domain file path provided: {} does not exits!", whitelistFileDir);
System.exit(1);
}

if (!new File(crawlerOutputDir).exists()) {
logger.error("The crawler output dir provided: {} does not exits!", crawlerOutputDir);
log.error("The crawler output dir provided: {} does not exits!", crawlerOutputDir);
System.exit(1);
}

if (!new File(testOutputDir).exists()) {
logger.error("The crawler output dir provided: {} does not exits!", testOutputDir);
log.error("The crawler output dir provided: {} does not exits!", testOutputDir);
System.exit(1);
}
}
Expand All @@ -377,25 +376,25 @@ public List<String> grabSeedURLs() {
File seeds = new File((String) crawlerVars.get("seedFileDir"));
BufferedReader seedReader = new BufferedReader(new FileReader(seeds));
// List<String> seedURLs = new ArrayList<>();
logger.info("Loading the following urls: ");
log.info("Loading the following urls: ");

String url = "";
while (url != null) {
urls.add(url);
url = seedReader.readLine();
}

// logger.info("Loaded {} seed URLS from {}", seedURLs.size(), seeds.getAbsolutePath());
// log.info("Loaded {} seed URLS from {}", seedURLs.size(), seeds.getAbsolutePath());

// for (String seedURL : seedURLs) {
// if (!urls.contains(seedURL))
// urls.add(seedURL);
// }

logger.info("Loaded {} total seed URLs", urls.size());
log.info("Loaded {} total seed URLs", urls.size());

} catch (IOException e) {
logger.error("Error while starting NVIP: {}", e.toString());
log.error("Error while starting NVIP: {}", e.toString());
}
return urls;
}
Expand All @@ -416,7 +415,7 @@ protected HashMap<String, ArrayList<RawVulnerability>> crawlCVEs(List<String> wh
*/
List<String> urls = grabSeedURLs();

logger.info("Starting the NVIP crawl process now to look for CVEs at {} locations with {} threads...",
log.info("Starting the NVIP crawl process now to look for CVEs at {} locations with {} threads...",
urls.size(), crawlerVars.get("crawlerNum"));

CveCrawlController crawlerController = new CveCrawlController(urls, whiteList, crawlerVars);
Expand Down Expand Up @@ -444,7 +443,7 @@ private void cvesToJson(HashMap<String, ArrayList<RawVulnerability>> crawledCVEs

Gson gson = builder.excludeFieldsWithModifiers(Modifier.FINAL).create();
String json = gson.toJson(crawledCVEs);
// logger.info(json);
// log.info(json);

try{
String filepath = (String) crawlerVars.get("testOutputDir") + "/test_output.json";
Expand All @@ -453,7 +452,7 @@ private void cvesToJson(HashMap<String, ArrayList<RawVulnerability>> crawledCVEs
output.write(json);
output.close();
} catch (IOException e) {
logger.error("Exception while writing list to JSON file!" + e);
log.error("Exception while writing list to JSON file!" + e);
}

}
Expand All @@ -464,7 +463,7 @@ private int cvesToCsv(HashMap<String, ArrayList<RawVulnerability>> crawledCVEs){

try {
String filepath = (String) crawlerVars.get("testOutputDir") + "/test_output.csv";
logger.info("Writing to CSV: {}", filepath);
log.info("Writing to CSV: {}", filepath);
FileWriter fileWriter = new FileWriter(filepath, false);
writer = new CSVWriter(fileWriter, '\t', CSVWriter.NO_QUOTE_CHARACTER, CSVWriter.NO_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END);

Expand All @@ -483,7 +482,7 @@ private int cvesToCsv(HashMap<String, ArrayList<RawVulnerability>> crawledCVEs){

writer.close();
} catch (IOException | NullPointerException e) {
logger.error("Exception while writing list to CSV file!" + e);
log.error("Exception while writing list to CSV file!" + e);
return 0;
}

Expand All @@ -506,16 +505,16 @@ private void createSourceTypeMap(){
while (source != null) {
String[] tokens = source.split(" ");
if (tokens.length < 2)
logger.warn("Source {} is not formatted correctly", source);
log.warn("Source {} is not formatted correctly", source);
else
sourceTypes.put(tokens[0], tokens[1]);
source = sourceReader.readLine();
}

logger.info("Loaded {} total sources/types", sourceTypes.size());
log.info("Loaded {} total sources/types", sourceTypes.size());

} catch (IOException e) {
logger.error("Error while starting NVIP: {}", e.toString());
log.error("Error while starting NVIP: {}", e.toString());
}
}

Expand All @@ -542,7 +541,7 @@ private void updateSourceTypes(HashMap<String, ArrayList<RawVulnerability>> craw
vuln.setSourceType(sourceTypes.get(sourceURL.getHost()));
}
catch(MalformedURLException e){
logger.warn("Bad sourceURL {}: {}", vuln.getSourceURL(), e.toString());
log.warn("Bad sourceURL {}: {}", vuln.getSourceURL(), e.toString());
}

if(vuln.getSourceType() == null){
Expand All @@ -561,7 +560,7 @@ private void insertRawCVEsAndPrepareMessage(HashMap<String, ArrayList<RawVulnera
ConnectionFactory factory = getConnectionFactory();

try (Connection connection = factory.newConnection()){
logger.info("Inserting {} CVEs to DB", crawledCves.size());
log.info("Inserting {} CVEs to DB", crawledCves.size());

List<RawVulnerability> vulnsToInsert = crawledCves.values().stream()
.flatMap(Collection::stream)
Expand All @@ -570,8 +569,8 @@ private void insertRawCVEsAndPrepareMessage(HashMap<String, ArrayList<RawVulnera

List<RawVulnerability> insertedVulns = rawDescriptionRepository.batchInsertRawVulnerability(vulnsToInsert);

logger.info("Inserted {} raw CVE entries in rawdescriptions", insertedVulns.size());
logger.info("Notifying Reconciler to reconciler {} new raw data entries", insertedVulns.size());
log.info("Inserted {} raw CVE entries in rawdescriptions", insertedVulns.size());
log.info("Notifying Reconciler to reconciler {} new raw data entries", insertedVulns.size());

try(Channel channel = connection.createChannel()) {
// Prepare the message and send it to the MQ server for Reconciler to pick up
Expand All @@ -585,15 +584,15 @@ private void insertRawCVEsAndPrepareMessage(HashMap<String, ArrayList<RawVulnera
// Declare a queue and send the message
String queueName = dataVars.get("mqQueueName") + "";
channel.queueDeclare(queueName, false, false, false, null);
logger.info("Queue '{}' created successfully.", queueName);
log.info("Queue '{}' created successfully.", queueName);
channel.basicPublish("", queueName, null, cveArray.getBytes());
logger.info("outgoing cve message: {}", cveArray);
logger.info(cveArray.getBytes());
logger.info("Message to Reconciler sent successfully.");
log.info("outgoing cve message: {}", cveArray);
log.info("{}", cveArray.getBytes());
log.info("Message to Reconciler sent successfully.");
}

} catch (Exception ex) {
logger.error("ERROR: Failed to send message to MQ server on {} via port {}", dataVars.get("mqHost"),
log.error("ERROR: Failed to send message to MQ server on {} via port {}", dataVars.get("mqHost"),
dataVars.get("mqPort"));
}
}
Expand Down

0 comments on commit 2cca00b

Please sign in to comment.