diff --git a/nvip_data/mysql-database/newDB/db.init.xml b/nvip_data/mysql-database/newDB/db.init.xml index a6176422e..04cde8ff5 100644 --- a/nvip_data/mysql-database/newDB/db.init.xml +++ b/nvip_data/mysql-database/newDB/db.init.xml @@ -930,5 +930,25 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/reconciler/src/main/java/edu/rit/se/nvip/DatabaseHelper.java b/reconciler/src/main/java/edu/rit/se/nvip/DatabaseHelper.java index 27b9d6bec..cde3fa68e 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/DatabaseHelper.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/DatabaseHelper.java @@ -68,6 +68,8 @@ public class DatabaseHelper { private static final String SELECT_MITRE_BY_DATE = "SELECT cve_id FROM mitredata WHERE last_modified >= DATE_SUB(NOW(), INTERVAL 2 MINUTE)"; private static final String INSERT_RUN_STATS = "INSERT INTO runhistory (run_date_time, total_cve_count, new_cve_count, updated_cve_count, not_in_nvd_count, not_in_mitre_count, not_in_both_count, avg_time_gap_nvd, avg_time_gap_mitre)" + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static final String EXPLOIT_EXISTS = "SELECT id FROM exploit WHERE cve_id = ?"; + private static final String INSERT_SSVC = "INSERT INTO ssvc (cve_id, automatable, exploit_status, technical_impact) VALUES (?, ?, ?, ?)"; public static synchronized DatabaseHelper getInstance() { if (databaseHelper == null) { @@ -785,4 +787,41 @@ public Set attachMitreVulns(Set return out; } + public boolean exploitExists(String cveId) { + try (Connection conn = getConnection(); PreparedStatement pstmt = conn.prepareStatement(EXPLOIT_EXISTS)) { + pstmt.setString(1, cveId); + return pstmt.execute(); + } catch (SQLException ex) { + logger.error("Error while fetching exploit data"); + logger.error(ex); + return false; + } + } + + public void insertSSVCSet(Set vulns) { + try (Connection conn = getConnection(); PreparedStatement pstmt = conn.prepareStatement(INSERT_SSVC)) { + conn.setAutoCommit(false); + for (CompositeVulnerability vuln : vulns) { + // Get SSVC data + final SSVC ssvc = vuln.getSSVC(); + + // Skip vulns w/o data + if (!vuln.isRecharacterized() || ssvc == null) continue; + + // Insert data into statement + pstmt.setString(1, vuln.getCveId()); + pstmt.setBoolean(2, ssvc.isAutomatable()); + pstmt.setString(3, ssvc.getExploitStatus()); + pstmt.setBoolean(4, ssvc.getTechnicalImpact()); + + } + // Execute statement + pstmt.execute(); + conn.commit(); + } catch (SQLException ex) { + logger.error("Error while inserting vdo labels"); + logger.error(ex); + } + } + } diff --git a/reconciler/src/main/java/edu/rit/se/nvip/ReconcilerController.java b/reconciler/src/main/java/edu/rit/se/nvip/ReconcilerController.java index 680616695..e73064676 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/ReconcilerController.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/ReconcilerController.java @@ -111,6 +111,7 @@ public void main(Set jobs) { dbh.insertCvssBatch(recharacterized); dbh.insertVdoBatch(recharacterized); + dbh.insertSSVCSet(recharacterized); } // PNE team no longer wants a finish message //messenger.sendPNEFinishMessage(); @@ -144,7 +145,7 @@ public CveCharacterizer call() { try { String[] trainingDataInfo = {ReconcilerEnvVars.getTrainingDataDir(), ReconcilerEnvVars.getTrainingData()}; logger.info("Setting NVIP_CVE_CHARACTERIZATION_LIMIT to {}", ReconcilerEnvVars.getCharacterizationLimit()); - return new CveCharacterizer(trainingDataInfo[0], trainingDataInfo[1], ReconcilerEnvVars.getCharacterizationApproach(), ReconcilerEnvVars.getCharacterizationMethod()); + return new CveCharacterizer(trainingDataInfo[0], trainingDataInfo[1], ReconcilerEnvVars.getCharacterizationApproach(), ReconcilerEnvVars.getCharacterizationMethod(), dbh); } catch (NullPointerException | NumberFormatException e) { logger.warn("Could not fetch NVIP_CVE_CHARACTERIZATION_TRAINING_DATA or NVIP_CVE_CHARACTERIZATION_TRAINING_DATA_DIR from env vars"); return null; diff --git a/reconciler/src/main/java/edu/rit/se/nvip/characterizer/CveCharacterizer.java b/reconciler/src/main/java/edu/rit/se/nvip/characterizer/CveCharacterizer.java index 504f699f0..caff45434 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/characterizer/CveCharacterizer.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/characterizer/CveCharacterizer.java @@ -22,6 +22,8 @@ * SOFTWARE. */ +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.rit.se.nvip.DatabaseHelper; import edu.rit.se.nvip.automatedcvss.CvssScoreCalculator; import edu.rit.se.nvip.automatedcvss.PartialCvssVectorGenerator; import edu.rit.se.nvip.automatedcvss.preprocessor.CvePreProcessor; @@ -37,8 +39,13 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import sun.net.www.protocol.https.HttpsURLConnectionImpl; -import java.io.File; +import javax.net.ssl.HttpsURLConnection; +import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLEncoder; import java.nio.file.Paths; import java.util.*; @@ -51,6 +58,8 @@ public class CveCharacterizer { private Logger logger = LogManager.getLogger(CveCharacterizer.class.getSimpleName()); private final Map nounGroupToClassifier = new HashMap<>(); + private final static ObjectMapper OM = new ObjectMapper(); + private final DatabaseHelper dbh; /** * these two vars are used to derive the CVSS vector from VDO labels and then @@ -74,10 +83,12 @@ public CveCharacterizer(CvePreProcessor cvePreProcessor, CveClassifierFactory cveClassifierFactory, CvssScoreCalculator cvssScoreCalculator, PartialCvssVectorGenerator partialCvssVectorGenerator, - String trainingDataPath, String trainingDataFiles, String approach, String method) { + String trainingDataPath, String trainingDataFiles, String approach, String method, + DatabaseHelper dbh) { this.cvssScoreCalculator = cvssScoreCalculator; this.partialCvssVectorGenerator = partialCvssVectorGenerator; this.cvePreProcessor = cvePreProcessor; + this.dbh = dbh; try { /** @@ -133,8 +144,8 @@ public CveCharacterizer(CvePreProcessor cvePreProcessor, */ //removed boolean loadSerializedModels as well as exploitability package - public CveCharacterizer(String trainingDataPath, String trainingDataFiles, String approach, String method) { - this(new CvePreProcessor(true), new CveClassifierFactory(), new CvssScoreCalculator(), new PartialCvssVectorGenerator(), trainingDataPath, trainingDataFiles, approach, method); + public CveCharacterizer(String trainingDataPath, String trainingDataFiles, String approach, String method, DatabaseHelper dbh) { + this(new CvePreProcessor(true), new CveClassifierFactory(), new CvssScoreCalculator(), new PartialCvssVectorGenerator(), trainingDataPath, trainingDataFiles, approach, method, dbh); } /** @@ -208,10 +219,8 @@ public void characterizeCveList(Set cveSet, int limit) { vuln.addCvssScore(score); // logger.info("CVSS Score predicted for {}", vulnerability.getCveId()); - // TODO: Route query to this object - // Get SSVC score - final SSVC ssvc = new SSVC(false, null, false); - // TODO: Attach object to vuln + // Get SSVC + vuln.setSSVC(characterizeCveForSSVC(vuln)); // log if (totCharacterized % 100 == 0 && totCharacterized > 0) { @@ -231,6 +240,39 @@ public void characterizeCveList(Set cveSet, int limit) { logger.info("{} CVEs did not have a good description, and {} CVEs had the same description (after reconciliation) and skipped!", countBadDescription, countNotChanged); } + // Query SSVC AI models for scoring + private SSVC characterizeCveForSSVC(CompositeVulnerability vuln) { + try { + // Create parameter dict + final Map params = new HashMap<>(); + params.put("cveId", vuln.getCveId()); + params.put("description", vuln.getDescription()); + params.put("exploitStatus", dbh.exploitExists(vuln.getCveId()) ? "POC" : "NONE"); + + // Create url object + final URL url = new URL("ssvc" + getParamsString(params)); + +// // Setup connection and parameters +// final HttpsURLConnection conn = (HttpsURLConnection) url.openConnection(); +// conn.setRequestMethod("GET"); +// conn.setDoOutput(true); +// DataOutputStream out = new DataOutputStream(conn.getOutputStream()); +// out.writeBytes(getParamsString(params)); +// out.flush(); +// out.close(); + + // Build object from request response + return OM.readValue(url, SSVC.class); + +// // Extract values from response +// +// // Build SSVC object +// final SSVC ssvc = new SSVC(automatable, exploitExists, technicalImpact); + } catch (Exception e) { + return null; + } + } + /** * get VDO labels and return a double array that includes the * mean/minimum/maximum and standard deviation of the CVSS scores in NVD @@ -247,4 +289,21 @@ private double getCvssScoreFromVdoLabels(Set predictionsForVuln) { // return cvssScoreCalculator.getCvssScoreJython(cvssVec)[0]; // old way of doing it return cvssScoreCalculator.lookupCvssScore(cvssVec); // should return same number as the old way but doesn't rely on python } + + private static String getParamsString(Map params) + throws UnsupportedEncodingException { + StringBuilder result = new StringBuilder(); + + for (Map.Entry entry : params.entrySet()) { + result.append(URLEncoder.encode(entry.getKey(), "UTF-8")); + result.append("="); + result.append(URLEncoder.encode(entry.getValue(), "UTF-8")); + result.append("&"); + } + + String resultString = result.toString(); + return resultString.length() > 0 + ? resultString.substring(0, resultString.length() - 1) + : resultString; + } } diff --git a/reconciler/src/main/java/edu/rit/se/nvip/model/CompositeVulnerability.java b/reconciler/src/main/java/edu/rit/se/nvip/model/CompositeVulnerability.java index 361b721b4..0b7248e60 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/model/CompositeVulnerability.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/model/CompositeVulnerability.java @@ -25,6 +25,9 @@ public enum ReconciliationStatus { // cvss scoring private CvssScore cvssScore; + // ssvc scoring + private SSVC ssvc; + //list of related cwes private final List cweList = new ArrayList<>(); private ReconciliationStatus recStatus; @@ -193,11 +196,15 @@ public CvssScore getCvssScoreInfo() { return cvssScore; } + public SSVC getSSVC() { return ssvc; } + public void addCvssScore(CvssScore cvss) { this.cvssScore = cvss; this.recharacterized = true; } + public void setSSVC(SSVC ssvc) { this.ssvc = ssvc; } + public boolean isRecharacterized() { return this.recharacterized; } diff --git a/reconciler/src/main/java/edu/rit/se/nvip/model/SSVC.java b/reconciler/src/main/java/edu/rit/se/nvip/model/SSVC.java index 4895a05b3..9b34f09cb 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/model/SSVC.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/model/SSVC.java @@ -13,4 +13,8 @@ public SSVC(boolean automatable, EXPLOIT_STATUS exploitStatus, boolean technical this.exploitStatus = exploitStatus; this.technicalImpact = technicalImpact; } + + public boolean isAutomatable() { return automatable; } + public String getExploitStatus() { return exploitStatus.toString(); } + public boolean getTechnicalImpact() { return technicalImpact; } } diff --git a/reconciler/src/main/java/edu/rit/se/nvip/sandbox/characterizerRealTest.java b/reconciler/src/main/java/edu/rit/se/nvip/sandbox/characterizerRealTest.java index 3f734dff7..140255adb 100644 --- a/reconciler/src/main/java/edu/rit/se/nvip/sandbox/characterizerRealTest.java +++ b/reconciler/src/main/java/edu/rit/se/nvip/sandbox/characterizerRealTest.java @@ -38,7 +38,7 @@ public static void main(String[] args){ String[] trainingDataInfo = {ReconcilerEnvVars.getTrainingDataDir(), ReconcilerEnvVars.getTrainingData()}; CveCharacterizer characterizer = new CveCharacterizer(trainingDataInfo[0], trainingDataInfo[1], ReconcilerEnvVars.getCharacterizationApproach(), - ReconcilerEnvVars.getCharacterizationMethod()); + ReconcilerEnvVars.getCharacterizationMethod(), null); // TODO: Add/mock DBH Set cveSet = new HashSet<>(); diff --git a/reconciler/src/test/java/edu/rit/se/nvip/characterizer/CveCharacterizerTest.java b/reconciler/src/test/java/edu/rit/se/nvip/characterizer/CveCharacterizerTest.java index c7351eb4e..d01c568e8 100644 --- a/reconciler/src/test/java/edu/rit/se/nvip/characterizer/CveCharacterizerTest.java +++ b/reconciler/src/test/java/edu/rit/se/nvip/characterizer/CveCharacterizerTest.java @@ -90,7 +90,7 @@ public void testCveCharacterization() { when(mockPartialCvssVectorGenerator.getCVssVector(anySet())).thenReturn(new String[8]); //create characterizer with the mocks manually injected CveCharacterizer cveCharacterizer = new CveCharacterizer(mockPreProcessor, mockCveClassifierFactory, mockCvssScoreCalculator, mockPartialCvssVectorGenerator, - trainingDataInfo[0], trainingDataInfo[1], "ML", "NB"); + trainingDataInfo[0], trainingDataInfo[1], "ML", "NB", null); // TODO: Add/mock dbh