static void Main(string[] args)
{
//https://stackoverflow.com/questions/5940225/fastest-way-of-inserting-in-entity-framework
model.Configuration.AutoDetectChangesEnabled = false;
model.Configuration.ValidateOnSaveEnabled = false;
int iCptYear = DateTime.Now.Year;
//XORCISMEntities model = new XORCISMEntities();
//int iVocabularySCIPID = 0;// 1044; //SCIP
#region vocabularyscip
try
{
iVocabularySCIPID = model.VOCABULARY.Where(o => o.VocabularyName == "SCIP").Select(o => o.VocabularyID).FirstOrDefault();
}
catch (Exception ex)
{
}
if (iVocabularySCIPID <= 0)
{
VOCABULARY oVocabulary = new VOCABULARY();
oVocabulary.CreatedDate = DateTimeOffset.Now;
oVocabulary.VocabularyName = "SCIP";
model.VOCABULARY.Add(oVocabulary);
model.SaveChanges();
iVocabularySCIPID = oVocabulary.VocabularyID;
Console.WriteLine("DEBUG iVocabularySCIPID=" + iVocabularySCIPID);
}
#endregion vocabularyscip
while (iCptYear > 2003)
{
string sURI = "refmap" + iCptYear;
Console.WriteLine("DEBUG *************************************************************");
Console.WriteLine("DEBUG " + DateTimeOffset.Now.ToString());
Console.WriteLine("DEBUG Working on " + sURI);
string sDownloadFileURL = "http://www.scip.ch/en/?vuldb." + sURI;
iCptYear--;
HttpWebRequest webRequest = null;
HttpWebResponse webResponse = null;
webRequest = (HttpWebRequest)WebRequest.Create(new Uri(sDownloadFileURL));
webRequest.Method = "GET";
//webRequest.Credentials = CredentialCache.DefaultCredentials;
//webRequest.Timeout = 20 * 60 * 1000; //20 minutes
webResponse = (HttpWebResponse)webRequest.GetResponse();
StreamReader SR = new StreamReader(webResponse.GetResponseStream());
string sResponseText = SR.ReadToEnd();
//Console.WriteLine(sResponseText);
SR.Close();
webResponse.Close();
StreamWriter swStreamWriter = new StreamWriter(sURI + ".txt");
swStreamWriter.Write(sResponseText);
swStreamWriter.Close();
StreamReader srStreamReader = new StreamReader(sURI + ".txt");
string sLine = srStreamReader.ReadLine();
string sTemp = string.Empty;
string sCurrentVULDB = string.Empty;
string sCurrentCVE = string.Empty;
int iVulnerabilityID = 0;
Regex myRegexVULDB = new Regex(@"<a href=\""\?vuldb\.[0-9](.*?)\"""); //TODO Review
//Regex myRegexCVE = new Regex("CVE-[0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]");
Regex myRegexCVE = new Regex(@"CVE-(19|20)\d\d-(0\d{3}|[1-9]\d{3,})");
//https://cve.mitre.org/cve/identifiers/tech-guidance.html
Regex myRegexSECTRACK = new Regex(@"securitytracker.com/id/(.*?)\"" "); //TODO Review
Regex myRegexSECUNIA = new Regex(@"secunia.com/advisories/(.*?)\"" "); //TODO Review
Regex myRegexBID = new Regex(@"securityfocus.com/bid/(.*?)\"" "); //TODO Review
Regex myRegexXFORCE = new Regex(@"xforce.iss.net/xforce/xfdb/(.*?)\"" "); //TODO Review
Regex myRegexOSVDB = new Regex(@"osvdb.org/[0-9](.*?)\"" "); //TODO Review
while (sLine != null)
{
sLine = sLine.Replace("securitytracker.com/id?", "securitytracker.com/id/");
//sLine = sLine.Replace("https://www.", "http://");
//sLine = sLine.Replace("http://www.", "http://");
sLine = sLine.Replace("osvdb.org/displayvuln.php?osvdbid=", "osvdb.org/");
sLine = sLine.Replace("osvdb.org/show/osvdb/", "osvdb.org/");
//TODO? microsoft.com MS
sTemp = myRegexVULDB.Match(sLine).ToString();
if (sTemp != "")
{
sTemp = sTemp.Replace("<a href=", "");
sTemp = sTemp.Replace("\"", "");
sTemp = sTemp.Replace("?vuldb.", "");
//TODO check if ok
sCurrentVULDB = sTemp;
Console.WriteLine("*************************************************************");
Console.WriteLine("DEBUG " + DateTimeOffset.Now.ToString());
Console.WriteLine("DEBUG SCIP VULDB:" + sCurrentVULDB);
}
else
{
sTemp = myRegexCVE.Match(sLine).ToString();
if (sTemp != "")
{
#region cve
sCurrentCVE = sTemp;
Console.WriteLine("DEBUG " + DateTimeOffset.Now.ToString());
Console.WriteLine("DEBUG CVE:" + sCurrentCVE);
//TODO double-check if it is real CVE-ID
try
{
iVulnerabilityID = vuln_nodel.VULNERABILITY.Where(o => o.VULReferential == "cve" && o.VULReferentialID == sCurrentCVE).Select(o => o.VulnerabilityID).FirstOrDefault();
}
catch (Exception exCVE)
{
//Console.WriteLine("Exception exCVE " + exCVE.Message + " " + exCVE.InnerException);
}
if (iVulnerabilityID <= 0)
{
try
{
VULNERABILITY oVulnerability = new VULNERABILITY();
oVulnerability.CreatedDate = DateTimeOffset.Now;
oVulnerability.VocabularyID = iVocabularySCIPID;
oVulnerability.VULReferential = "cve";
oVulnerability.VULReferentialID = sCurrentCVE;
oVulnerability.timestamp = DateTimeOffset.Now;
vuln_nodel.VULNERABILITY.Add(oVulnerability);
vuln_nodel.SaveChanges();
iVulnerabilityID = oVulnerability.VulnerabilityID;
}
catch (System.Data.Entity.Validation.DbEntityValidationException e)
{
System.Text.StringBuilder sb = new System.Text.StringBuilder();
foreach (var eve in e.EntityValidationErrors)
{
sb.AppendLine(string.Format("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
eve.Entry.Entity.GetType().Name,
eve.Entry.State));
foreach (var ve in eve.ValidationErrors)
{
sb.AppendLine(string.Format("- Property: \"{0}\", Error: \"{1}\"",
ve.PropertyName,
ve.ErrorMessage));
}
}
//throw new DbEntityValidationException(sb.ToString(), e);
Console.WriteLine("Exception DbEntityValidationExceptionUPDATECAPEC " + sb.ToString());
}
catch (Exception exSCIPCVE)
{
Console.WriteLine("Exception exSCIPCVE " + exSCIPCVE.Message + " " + exSCIPCVE.InnerException);
}
}
else
{
//Update VULNERABILITY
}
Console.WriteLine("DEBUG " + DateTimeOffset.Now.ToString());
Console.WriteLine("DEBUG iVulnerabilityID=" + iVulnerabilityID);
sSource = "SCIP";
sSourceID = sCurrentVULDB;
sReferenceURL = "http://scip.ch/?vuldb." + sCurrentVULDB;
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
#endregion cve
}
else
{
//<td><a href="http://osvdb.org/3314" title="osvdb.org/3314">3314</a></td>
sTemp = myRegexOSVDB.Match(sLine).ToString();
if (sTemp != "")
{
#region osvdb
//Console.WriteLine(sTemp);
sSource = "OSVDB";
sSourceID = sTemp.Replace("osvdb.org/", "");
sSourceID = sSourceID.Replace("/", "");
sSourceID = sSourceID.Replace("\"", "").Trim();
//Console.WriteLine(sSourceID);
try
{
int iTest = int.Parse(sSourceID);
sReferenceURL = "http://osvdb.org/" + sSourceID;
Console.WriteLine("DEBUG " + sReferenceURL);
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
}
catch (Exception exSCIPOSVDBID)
{
Console.WriteLine("Exception exSCIPOSVDBID " + sSourceID + " " + exSCIPOSVDBID.Message + " " + exSCIPOSVDBID.InnerException);
}
//TODO see Import_all
//fRequestOSVDB();
#endregion osvdb
}
else
{
#region securitytracker
////http://securitytracker.com/id?1028074
//http://securitytracker.com/id/1029599
sTemp = myRegexSECTRACK.Match(sLine).ToString();
if (sTemp != "")
{
//Console.WriteLine(sTemp);
sSource = "SECTRACK";
sSourceID = sTemp.Replace("securitytracker.com/id/", "");
sSourceID = sSourceID.Replace("/", "");
sSourceID = sSourceID.Replace("\"", "").Trim();
//Console.WriteLine(sSourceID);
sReferenceURL = "http://securitytracker.com/id/" + sSourceID;
Console.WriteLine("DEBUG " + sReferenceURL);
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
}
#endregion securitytracker
else
{
#region secunia
//http://secunia.com/advisories/58347
sTemp = myRegexSECUNIA.Match(sLine).ToString();
if (sTemp != "")
{
//Console.WriteLine(sTemp);
sSource = "SECUNIA";
sSourceID = sTemp.Replace("secunia.com/advisories/", "");
sSourceID = sSourceID.Replace("/", "");
sSourceID = sSourceID.Replace("\"", "").Trim();
//Console.WriteLine(sSourceID);
sReferenceURL = "http://secunia.com/advisories/" + sSourceID;
Console.WriteLine("DEBUG " + sReferenceURL);
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
}
#endregion secunia
else
{
#region securityfocus
//http://securityfocus.com/bid/123
sTemp = myRegexBID.Match(sLine).ToString();
if (sTemp != "")
{
//Console.WriteLine(sTemp);
sSource = "BID";
sSourceID = sTemp.Replace("securityfocus.com/bid/", "");
sSourceID = sSourceID.Replace("/", "");
sSourceID = sSourceID.Replace("\"", "").Trim();
//Console.WriteLine(sSourceID);
sReferenceURL = "http://securityfocus.com/bid/" + sSourceID;
Console.WriteLine("DEBUG " + sReferenceURL);
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
}
#endregion securityfocus
else
{
#region xforce
//http://xforce.iss.net/xforce/xfdb/123
sTemp = myRegexXFORCE.Match(sLine).ToString();
if (sTemp != "")
{
//Console.WriteLine(sTemp);
sSource = "XF";
sSourceID = sTemp.Replace("xforce.iss.net/xforce/xfdb/", "");
sSourceID = sSourceID.Replace("/", "");
sSourceID = sSourceID.Replace("\"", "").Trim();
//Console.WriteLine(sSourceID);
sReferenceURL = "http://xforce.iss.net/xforce/xfdb/" + sSourceID;
Console.WriteLine("DEBUG " + sReferenceURL);
fAddReference(iVulnerabilityID); //, sSource, sSourceID, sReferenceURL);
}
#endregion xforce
else
{
//TODO
}
}
}
}
}
}
}
sLine = srStreamReader.ReadLine();
}
srStreamReader.Close();
}
//FREE
try
{
model.SaveChanges();
}
catch (System.Data.Entity.Validation.DbEntityValidationException e)
{
System.Text.StringBuilder sb = new System.Text.StringBuilder();
foreach (var eve in e.EntityValidationErrors)
{
sb.AppendLine(string.Format("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
eve.Entry.Entity.GetType().Name,
eve.Entry.State));
foreach (var ve in eve.ValidationErrors)
{
sb.AppendLine(string.Format("- Property: \"{0}\", Error: \"{1}\"",
ve.PropertyName,
ve.ErrorMessage));
}
}
//throw new DbEntityValidationException(sb.ToString(), e);
Console.WriteLine("Exception DbEntityValidationExceptionFINALSAVE " + sb.ToString());
}
catch (Exception exFINALSAVE)
{
Console.WriteLine("Exception exFINALSAVE " + exFINALSAVE.Message + " " + exFINALSAVE.InnerException);
}
model.Dispose();
}
private void button1_Click_1(object sender, EventArgs e)
{
//NOTE: OUTDATED Project. See Import_all
string filename;
//http://www.saintcorporation.com/xml/exploits.xml
try
{
WebClient wc = new WebClient();
wc.DownloadFile("http://www.saintcorporation.com/xml/exploits.xml", "C:/nvdcve/exploits.xml"); //HARDCODED
//
wc.Dispose();
//MessageBox.Show("Download is completed", "info", MessageBoxButtons.OK, MessageBoxIcon.Question, MessageBoxDefaultButton.Button1);
}
catch (Exception ex)
{
MessageBox.Show("Error while downloading exploits.xml\n" + ex.Message, "Erreur", MessageBoxButtons.OK, MessageBoxIcon.Hand, MessageBoxDefaultButton.Button1);
}
filename = @"C:\nvdcve\exploits.xml"; //HARDCODED
XmlDocument doc = new XmlDocument();
doc.Load(filename);
string query = "/xml/body/exploits";
XmlNode report;
report = doc.SelectSingleNode(query);
XORCISMEntities model = new XORCISMEntities();
XVULNERABILITYEntities vuln_model = new XVULNERABILITYEntities();
foreach (XmlNode n in report.ChildNodes)
{
//if (n.Name.ToUpper() == "exploit".ToUpper() && n.ChildNodes != null && n.ChildNodes.Count > 0)
//{
EXPLOIT sploit = new EXPLOIT();
string myRefID = n.Attributes["id"].InnerText;
sploit.ExploitRefID = myRefID;
sploit.ExploitName = n.Attributes["id"].InnerText;
sploit.ExploitReferential = "saint";
sploit.ExploitDescription = HelperGetChildInnerText(n, "description");
//TODO
//sploit.saint_id = HelperGetChildInnerText(n, "saint_id");
sploit.ExploitType = HelperGetChildInnerText(n, "type");
//Search the VulnerabilityID
string myCVE = HelperGetChildInnerText(n, "cve");
int vulnID = 0;
if (myCVE != "")
{
var syn = from S in vuln_model.VULNERABILITY
where S.VULReferential.Equals("cve") &&
S.VULReferentialID.Equals(myCVE)
select S;
if (syn.Count() != 0)
{
vulnID = syn.ToList().First().VulnerabilityID;
// MessageBox.Show("VulnerabilityID of " + myCVE + " is:" + vulnID);
}
else
{
//MessageBox.Show("Import_saint_exploits CVE not found! " + myCVE);
//CANDIDATE
VULNERABILITY canCVE = new VULNERABILITY();
canCVE.VULReferential = "cve";
canCVE.VULReferentialID = myCVE;
canCVE.VULDescription = "CANDIDATE";
vuln_model.VULNERABILITY.Add(canCVE);
vuln_model.SaveChanges();
vulnID = canCVE.VulnerabilityID;
// return;
}
}
//Check if the exploit already exists in the database
var syna = from S in model.EXPLOIT
where S.ExploitReferential.Equals("saint") &&
S.ExploitRefID.Equals(myRefID)
select S;
if (syna.Count() == 0)
{
model.EXPLOIT.Add(sploit);
}
else
{
sploit.ExploitID = syna.ToList().First().ExploitID;
}
try
{
model.SaveChanges();
}
catch (FormatException ex)
{
MessageBox.Show("FormatException AddToEXPLOIT : " + ex);
return;
}
if (vulnID != 0)
{
//Check if EXPLOITFORVULNERABILITY already exists in the database
var synj = from S in model.EXPLOITFORVULNERABILITY
where S.VulnerabilityID.Equals(vulnID) &&
S.ExploitID.Equals(sploit.ExploitID)
select S;
if (synj.Count() == 0)
{
EXPLOITFORVULNERABILITY sploitvuln = new EXPLOITFORVULNERABILITY();
sploitvuln.VulnerabilityID = vulnID;
sploitvuln.ExploitID = sploit.ExploitID;
try
{
model.EXPLOITFORVULNERABILITY.Add(sploitvuln);
model.SaveChanges();
}
catch (FormatException ex)
{
MessageBox.Show("AddToEXPLOITFORVULNERABILITY : " + ex);
}
}
}
//****************************************************************
// OSVDB
string myOSVDB = HelperGetChildInnerText(n, "osvdb");
if (myOSVDB != "")
{
//Check if the OSVDB reference already exists in the database
int osvdbID = 0;
var syn2 = from S in model.REFERENCE
where S.Source.Equals("OSVDB") &&
S.ReferenceTitle.Equals(myOSVDB)
select S;
REFERENCE RefJA = new REFERENCE();
if (syn2.Count() != 0)
{
//UPDATE
osvdbID = syn2.ToList().First().ReferenceID;
RefJA.ReferenceID = osvdbID;
RefJA.ReferenceURL = "http://osvdb.org/" + myOSVDB;
model.SaveChanges();
}
else
{
//Add the OSVDB Reference
RefJA.Source = "OSVDB";
RefJA.ReferenceTitle = myOSVDB;
RefJA.ReferenceURL = "http://osvdb.org/" + myOSVDB;
model.REFERENCE.Add(RefJA);
model.SaveChanges();
osvdbID = RefJA.ReferenceID;
}
//Check if the EXPLOITFORREFERENCE already exists in the database
var syn3 = from S in model.EXPLOITFORREFERENCE
where S.ExploitID.Equals(sploit.ExploitID) &&
S.ReferenceID.Equals(osvdbID)
select S;
if (syn3.Count() == 0)
{
EXPLOITFORREFERENCE sploitref = new EXPLOITFORREFERENCE();
sploitref.ExploitID = sploit.ExploitID;
sploitref.ReferenceID = osvdbID;
model.EXPLOITFORREFERENCE.Add(sploitref);
model.SaveChanges();
}
}
//****************************************************************
// BID
string myBID = HelperGetChildInnerText(n, "bid");
if (myBID != "")
{
//Check if the BID reference already exists in the database
int bidID = 0;
var syn2 = from S in model.REFERENCE
where S.Source.Equals("BID") &&
S.ReferenceTitle.Equals(myBID)
select S;
if (syn2.Count() != 0)
{
bidID = syn2.ToList().First().ReferenceID;
}
else
{
//Add the OSVDB Reference
REFERENCE RefJA = new REFERENCE();
RefJA.Source = "BID";
RefJA.ReferenceTitle = myBID;
RefJA.ReferenceURL = "http://securityfocus.com/bid/" + myBID;
model.REFERENCE.Add(RefJA);
model.SaveChanges();
bidID = RefJA.ReferenceID;
}
//Check if the EXPLOITFORREFERENCE already exists in the database
var syn3 = from S in model.EXPLOITFORREFERENCE
where S.ExploitID.Equals(sploit.ExploitID) &&
S.ReferenceID.Equals(bidID)
select S;
if (syn3.Count() == 0)
{
EXPLOITFORREFERENCE sploitref = new EXPLOITFORREFERENCE();
sploitref.ExploitID = sploit.ExploitID;
sploitref.ReferenceID = bidID;
model.EXPLOITFORREFERENCE.Add(sploitref);
model.SaveChanges();
}
}
//}
}
MessageBox.Show("FINISHED MISTER_X");
}