public static O2RulePack createRules_SourcesAndSinks(String sCirDataFile)
{
var rpRulePack = new O2RulePack();
if (false == File.Exists(sCirDataFile))
{
DI.log.error("in createRules_SourcesAndSinks, provide CirData file not found: {0}", sCirDataFile);
}
else
{
List <String> lsFunctions = MiscUtils.getFunctionsSignaturesFrom02CirData(sCirDataFile);
// in this type of scan, there are two rules
// if functions make no calls then they are maked as both Sources and Sinks
// other cases receive no marking
// sinks have preference (for the cases there there are no calls into and from
CirData fcdCirData = CirLoad.loadSerializedO2CirDataObject(sCirDataFile);
foreach (string sFunction in lsFunctions)
{
ICirFunction cfCirFunction = fcdCirData.dFunctions_bySignature[sFunction];
if (cfCirFunction.FunctionsCalledUniqueList.Count == 0)
{
addRule(createRule(O2RuleType.Sink, sFunction, fcdCirData.sDbId), rpRulePack);
addRule(createRule(O2RuleType.Source, sFunction, fcdCirData.sDbId), rpRulePack);
}
}
}
return(rpRulePack);
}
public static O2RulePack createRules_CallBacksOnControlFlowGraphs_And_ExternalSinks(String sCirDataFile)
{
var rpRulePack = new O2RulePack();
if (false == File.Exists(sCirDataFile))
{
DI.log.error(
"in createRules_CallBacksOnControlFlowGraphs_And_ExternalSinks, provide CirData file not found: {0}",
sCirDataFile);
}
else
{
List <String> lsFunctions = MiscUtils.getFunctionsSignaturesFrom02CirData(sCirDataFile);
// in this type of scan, there are two rules
// if functions have a ControlFlowGraph they are Callbacks
// everything else is a sink
ICirData fcdCirData = CirLoad.loadSerializedO2CirDataObject(sCirDataFile);
Dictionary <String, ICirFunction> dFunctionsWithControlFlowGraphs =
CirDataUtils.getFunctionsWithControlFlowGraph(fcdCirData);
foreach (string sFunction in lsFunctions)
{
if (dFunctionsWithControlFlowGraphs.ContainsKey(sFunction))
{
addRule(createRule(O2RuleType.Callback, sFunction, fcdCirData.sDbId), rpRulePack);
}
else
{
addRule(createRule(O2RuleType.Sink, sFunction, fcdCirData.sDbId), rpRulePack);
}
}
}
return(rpRulePack);
}
/*public static List<String> getListOfUniqueSignatures_UsingUniqueList(List<IO2Finding> o2Findings)
* {
* // mode 1 use one big list
* var uniqueSignatures = new List<String>();
* // calculate unique signatures
* var itemsProcessed = 0;
* var itemsToProcess = o2Findings.Count;
* foreach (var o2Finding in o2Findings)
* {
* getListOfUniqueSignatures(o2Finding.o2Traces, uniqueSignatures);
* if ((itemsProcessed++) % 5000 == 0)
* DI.log.info("on [{0}/{1}] there are {2} unique signatures", itemsProcessed, itemsToProcess, uniqueSignatures.Count);
* }
* DI.log.info("There are {0} unique signatures", uniqueSignatures.Count);
* return uniqueSignatures;
* }*/
internal static O2RulePack mapFindingsToCurrentRulePack(
O2RulePack currentO2RulePack, List <IO2Finding> o2Findings, string languageId, bool keepRulesLoadedFromDatabase)
{
var newCallbackSignature = "O2.AutoMapping";
DI.log.info("converting {0} Findings into rules", o2Findings.Count);
// get rules that we will keep (if there are rules loaded from Db and keepRulesLoadedFromDatabase is set)
var newO2Rules = (keepRulesLoadedFromDatabase)
? O2RulePackUtils.getRulesThatAreFromDB(currentO2RulePack)
: new List <IO2Rule>();
//update index
var indexedCurrentO2Rules = IndexedO2Rules.indexAll(newO2Rules);
// and if there are any rules from the database, remove any Tagged values
if (keepRulesLoadedFromDatabase)
{
foreach (var o2Rule in newO2Rules)
{
o2Rule.Tagged = false;
}
}
// create rulepack
var newRulePack = new O2RulePack("Rule Pack", newO2Rules);
// and call the function that does the mappings
mapInRulePack_FindingsSourcesAndSinks(newRulePack, indexedCurrentO2Rules, o2Findings, languageId);
DI.log.info("{0} rules created", newRulePack.o2Rules.Count);
return(newRulePack);
}
public List <IO2Rule> addRulesToDatabase(bool bDeleteDatabase, O2RulePack o2rulePack)
{
var rulesNotProcessed = new List <IO2Rule>();
if (bDeleteDatabase)
{
Lddb_OunceV6.action_DeleteAllRules();
}
//Utils.debugBreak();
//removeRulesFromCache();
DI.log.info("Adding {0} rules to database", o2rulePack.o2Rules.Count);
foreach (var o2Rule in o2rulePack.o2Rules)
{
if (false == addRuleToDatabase(o2Rule))
{
rulesNotProcessed.Add(o2Rule);
}
}
DI.log.info("Completed adding {0} rules to database", o2rulePack.o2Rules.Count);
if (rulesNotProcessed.Count > 0)
{
DI.log.info("Total number of rules NOT processed: {0}", rulesNotProcessed.Count);
}
return(rulesNotProcessed);
}
/// <summary>
/// recursive mapping of traces tracetype (required since there could be multiple sources
/// </summary>
/// <param name="o2RulePack"></param>
/// <param name="indexedO2Rules"></param>
/// <param name="o2Traces"></param>
/// <param name="languageDBId"></param>
/// <param name="o2NewRuleType"></param>
private static void updateO2RulePackWithFindingsTraceTypes(O2RulePack o2RulePack, IDictionary <string, List <IO2Rule> > indexedO2Rules, List <IO2Trace> o2Traces, string languageDBId, O2RuleType o2NewRuleType)
{
foreach (var o2Trace in o2Traces)
{
updateSourceOrSinkRule(o2RulePack, indexedO2Rules, o2Trace.signature, languageDBId, o2Trace.traceType);
updateO2RulePackWithFindingsTraceTypes(o2RulePack, indexedO2Rules, o2Trace.childTraces, languageDBId, o2NewRuleType);
}
}
public static string saveRulePack(string rulePackFile, O2RulePack o2rulePackToSave)
{
if (o2rulePackToSave.o2Rules.Count > 0)
{
Serialize.createSerializedXmlFileFromObject(o2rulePackToSave, rulePackFile);
DI.log.info("Saved O2RulePack {0} with {1} rules", rulePackFile, o2rulePackToSave.o2Rules.Count);
return(rulePackFile);
}
return("");
}
public static String saveRulePack(String pathToSaveFile, String sTypeOfPack, O2RulePack o2rulePackToSave)
{
if (false == File.Exists(pathToSaveFile) && false == Directory.Exists(Path.GetDirectoryName(pathToSaveFile)))
{
pathToSaveFile = DI.config.TempFileNameInTempDirectory + "_" + pathToSaveFile;
}
string rulePackFile = pathToSaveFile + "_" + o2rulePackToSave.RulePackName + "_" + "(" +
o2rulePackToSave.o2Rules.Count + ")_" + sTypeOfPack + ".O2RulePack";
return(saveRulePack(rulePackFile, o2rulePackToSave));
}
public void test_createRulePackWithSourcesAndSinksRulesFromLddbFor_DotNet()
{
var ruleDbId = MiscUtils_OunceV6.getIdForSuportedLanguage(SupportedLanguage.DotNet).ToString();
// create just sources and sinks
var sourcesRulePack = new O2RulePack("MySql_Sources", mySqlRules_OunceV6.getRules_Sources(ruleDbId));
var sourcesRulePackFile = O2RulePackUtils.saveRulePack(sourcesRulePack);
Assert.That(File.Exists(sourcesRulePackFile), "sourcesRulePackFile doesn't exist");
var sinksRulePack = new O2RulePack("MySql_Sinks", mySqlRules_OunceV6.getRules_Sinks(ruleDbId));
var sinksRulePackFile = O2RulePackUtils.saveRulePack(sinksRulePack);
Assert.That(File.Exists(sinksRulePackFile), "sinksRulePackFile doesn't exist");
}
public static void mapInRulePack_FindingsSourcesAndSinks(O2RulePack o2RulePack, Dictionary <string, List <IO2Rule> > indexedO2Rules, List <IO2Finding> o2Findings, string languageDBId)
{
var timer = new O2Timer("mapInRulePack_FindingsSourcesAndSinks").start();
foreach (O2Finding o2Finding in o2Findings)
{
updateO2RulePackWithFindingsTraceTypes(o2RulePack, indexedO2Rules, o2Finding.o2Traces, languageDBId,
O2RuleType.Source);
}
/*updateSourceOrSinkRule(o2RulePack,indexedO2Rules, o2Finding.Source, languageDBId, O2RuleType.Source);
* updateSourceOrSinkRule(o2RulePack,indexedO2Rules, o2Finding.KnownSink, languageDBId, O2RuleType.Sink);
* updateSourceOrSinkRule(o2RulePack, indexedO2Rules, o2Finding.LostSink, languageDBId, O2RuleType.LostSink); */
timer.stop();
}
public void test_createRulePackWithAllRulesFromLddbFor_DotNet()
{
var numberOfRules = Lddb_OunceV6.getNumberOfRulesInRecTable();
Assert.That(numberOfRules > 0, "numberOfRules == 0");
DI.log.info("There {0} rules", numberOfRules);
var o2Rules = mySqlRules_OunceV6.createO2RulesForAllLddbEntriesForLanguage(SupportedLanguage.DotNet);
var o2RulePack = new O2RulePack("MySql_Dump", o2Rules);
Assert.That(o2Rules.Count > 0, "o2Rules.Count ==0");
var rulePackFile = O2RulePackUtils.saveRulePack(o2RulePack);
Assert.That(File.Exists(rulePackFile), "rulePacklFile file didn't exist: " + rulePackFile);
DI.log.info("Rule pack (with {0} rules) saved to {1}", o2Rules.Count, rulePackFile);
}
public void calculateRulePack_forFile(string ozamstFileToTest, O2RulePack o2RulePackToUse)
{
DI.log.debug("\n\ntesting file: {0}", ozamstFileToTest);
var timer = new O2Timer("Calculate Rule Pack ").start();
// load assesment file and get unique signatures
var o2Findings = new O2AssessmentLoad_OunceV6().loadFile(ozamstFileToTest).o2Findings;
var uniqueSignatures = RulesAndFindingsUtils.getListOfUniqueSignatures(o2Findings);
Assert.That(uniqueSignatures.Count > 0, "uniqueSignatures ==0");
// calculate rulepack for this assessment
var o2RulePackForOzasmt = RulesAndFindingsUtils.createRulePackWithSignatures(o2RulePackToUse, uniqueSignatures,
Path.GetFileName(ozamstFileToTest), true, testLanguageDBId);
Assert.That(o2RulePackForOzasmt.o2Rules.Count > 0, "There were no rules in created o2RulePackForOzasmt");
timer.stop();
}
public void createRulePacks()
{
String sCirDataFile = scCurrentScanTarget.ApplicationFile + ".CirData";
//_SourcesAndSinks
O2RulePack o2RulePack_SourcesAndSinks = O2RulePackUtils.createRules_SourcesAndSinks(sCirDataFile);
O2RulePackUtils.saveRulePack(sCirDataFile, "_SourcesAndSinks", o2RulePack_SourcesAndSinks);
//_CallBacksOnEdges_And_ExternalSinks
O2RulePack o2RulePack_CallBacksOnEdges_And_ExternalSinks = O2RulePackUtils.createRules_CallBacksOnEdges_And_ExternalSinks(sCirDataFile);
O2RulePackUtils.saveRulePack(sCirDataFile, "_CallBacksOnEdges_And_ExternalSinks", o2RulePack_CallBacksOnEdges_And_ExternalSinks);
//_CallBacksOnControlFlowGraphs_And_ExternalSinks
O2RulePack o2RulePack_CallBacksOnControlFlowGraphs_And_ExternalSinks = O2RulePackUtils.createRules_CallBacksOnControlFlowGraphs_And_ExternalSinks(sCirDataFile);
O2RulePackUtils.saveRulePack(sCirDataFile, "_CallBacksOnControlFlowGraphs_And_ExternalSinks", o2RulePack_CallBacksOnControlFlowGraphs_And_ExternalSinks);
adManualTestTempFiles.refreshDirectoryView();
DI.log.debug("Rule Packs creation complete");
}
public static O2RulePack createRulePackWithSignatures(O2RulePack o2rulePack, List <String> signaturesToFind, string rulePackName, bool addNonMatchingSignaturesAsNewRule, string languageDBId)
{
DI.log.info("in createRulePackWithSignatures");
var o2RulesThatMatchSignatures = new List <IO2Rule>();
var indexedRules = IndexedO2Rules.indexAll(o2rulePack.getIO2Rules());
foreach (var signature in signaturesToFind)
{
if (indexedRules.ContainsKey(signature))
{
foreach (var o2Rule in indexedRules[signature])
{
o2RulesThatMatchSignatures.Add(o2Rule);
}
}
else
if (addNonMatchingSignaturesAsNewRule)
{
o2RulesThatMatchSignatures.Add(O2RulePackUtils.createRule(O2RuleType.NotMapped, signature, languageDBId));
}
}
DI.log.info("createRulePackWithSignatures completed, there were {0} rules found", o2RulesThatMatchSignatures.Count);
return(new O2RulePack(rulePackName, o2RulesThatMatchSignatures));
}
public static O2RulePack createRules_CallBacksOnEdges_And_ExternalSinks(String sCirDataFile)
{
var rpRulePack = new O2RulePack();
if (false == File.Exists(sCirDataFile))
{
DI.log.error(
"in createRules_CallBacksOnEdges_And_ExternalSinks, provide CirData file not found: {0}",
sCirDataFile);
}
else
{
List <String> lsFunctions = MiscUtils.getFunctionsSignaturesFrom02CirData(sCirDataFile);
// in this type of scan, there are two rules
// if functions make no calls it is a Sink
// if nobody calls the function it is a callback
// sinks have preference (for the cases there there are no calls into and from
ICirData fcdCirData = CirLoad.loadSerializedO2CirDataObject(sCirDataFile);
foreach (string sFunction in lsFunctions)
{
ICirFunction cfCirFunction = fcdCirData.dFunctions_bySignature[sFunction];
if (cfCirFunction.FunctionsCalledUniqueList.Count == 0)
{
addRule(createRule(O2RuleType.Sink, sFunction, fcdCirData.sDbId), rpRulePack);
}
// DI.log.error(" Make no Calls (make sink): {0}" , cfCirFunction.sSignature);
else if (cfCirFunction.FunctionIsCalledBy.Count == 0)
{
addRule(createRule(O2RuleType.Callback, sFunction, fcdCirData.sDbId), rpRulePack);
}
}
}
return(rpRulePack);
}
public static void addRule(IO2Rule rRule, O2RulePack o2rulePack)
{
o2rulePack.o2Rules.Add((O2Rule)rRule);
}
public static O2RulePack createRulePackThatMatchFindings(O2RulePack o2rulePack, List <IO2Finding> o2Findings, string rulePackName, string languageDBId)
{
var uniqueSignatures = getListOfUniqueSignatures(o2Findings);
return(createRulePackWithSignatures(o2rulePack, uniqueSignatures, rulePackName, true, languageDBId));
}
//rename this method since we do more than just Source or Sinks here
private static void updateSourceOrSinkRule(O2RulePack o2RulePack, IDictionary <string, List <IO2Rule> > indexedO2Rules, string ruleSignature, string languageDBId, TraceType traceType)
{
if (indexedO2Rules != null && ruleSignature != null)
{
//bool createNotMappedRule = false;
var createRuleWithType = O2RuleType.NotMapped;
// only proccess Sources, Sinks and Lost Sinks
if (traceType == TraceType.Source || traceType == TraceType.Known_Sink || traceType == TraceType.Lost_Sink)
{
O2RuleType o2NewRuleType = O2RulePackUtils.getO2RuleTypeFromRuleType(traceType);
createRuleWithType = o2NewRuleType;
/* if (indexedO2Rules != null && false == string.IsNullOrEmpty(ruleSignature))
* if (indexedO2Rules.ContainsKey(ruleSignature))
* {
* bool thereIsAlreadyARuleWithTheSameRuleType = false;
* foreach (var o2Rule in indexedO2Rules[ruleSignature])
* {
* if (o2Rule.RuleType == O2RuleType.NotMapped)
* // if it is not mapped change it to o2NewRuleType
* {
* o2Rule.RuleType = o2NewRuleType;
* return;
* }
* if (o2Rule.RuleType == o2NewRuleType)
* // if it is already a rule of type o2NewRuleType, mark it so we can ignore it below
* thereIsAlreadyARuleWithTheSameRuleType = true;
* }
* if (false == thereIsAlreadyARuleWithTheSameRuleType)
* // if we got this far, create a new rule of o2NewRuleType
* {
* var newRule = O2RulePackUtils.createRule(o2NewRuleType, ruleSignature, languageDBId);
* indexedO2Rules[ruleSignature].Add(newRule);
* // add it to the index so that we don't have to calculate it again
* o2RulePack.o2Rules.Add(newRule);
* }
* }
*/
}
bool createNewRule = true;
IO2Rule notMappedRule = null;
if (indexedO2Rules.ContainsKey(ruleSignature))
{
foreach (var o2Rule in indexedO2Rules[ruleSignature])
{
if (o2Rule.RuleType == createRuleWithType) // dont create if there is already a rule of this type
{
o2Rule.Tagged = true;
createNewRule = false;
}
if (o2Rule.RuleType == O2RuleType.NotMapped)
{
notMappedRule = o2Rule;
}
}
}
// handle the case where we have already added a signature but it is not a NotMapped one
if (createRuleWithType == O2RuleType.NotMapped && createNewRule && notMappedRule == null && indexedO2Rules.ContainsKey(ruleSignature))
{
createNewRule = false;
}
// if required, Create rule
if (createNewRule)
{
var vulnType = "O2.FindingRule." + createRuleWithType.ToString();
var newRule = new O2Rule(createRuleWithType, vulnType, ruleSignature, languageDBId, true);
o2RulePack.o2Rules.Add(newRule);
if (false == indexedO2Rules.ContainsKey(ruleSignature))
{
indexedO2Rules.Add(ruleSignature, new List <IO2Rule>());
}
indexedO2Rules[ruleSignature].Add(newRule);
if (notMappedRule != null)
{
indexedO2Rules[ruleSignature].Remove(notMappedRule);
}
}
}
}
public static O2RulePack createRulePackThatMatchFindings(O2RulePack o2rulePack, List <IO2Finding> o2Findings, string languageDBId)
{
return(createRulePackThatMatchFindings(o2rulePack, o2Findings, "RulePackThatMatchFindings", languageDBId));
}
public static string saveRulePack(O2RulePack o2rulePackToSave)
{
return(saveRulePack(DI.config.TempFileNameInTempDirectory, "",
o2rulePackToSave));
}
public static List <IO2Rule> getRulesThatAreFromDB(O2RulePack o2RulePack)
{
return((from IO2Rule o2Rule in o2RulePack.o2Rules where o2Rule.FromDb select o2Rule).ToList());
}
public bool scanApplication(String applicationToScan)
{
if (false == File.Exists(applicationToScan))
{
DI.log.error("in scanApplication, could not file application file: {0}", applicationToScan);
return(false);
}
sApplicationToScan = applicationToScan;
// set standard savedfilelocations
sAssessmentFile_ScanWithExistingRules = sApplicationToScan + sAssessmentFile_ScanWithExistingRules;
sAssessmentFile_ScanWithNoRules = sApplicationToScan + sAssessmentFile_ScanWithNoRules;
sAssessmentFile_CallBacksOnControlFlowGraphs_And_ExternalSinks = sApplicationToScan +
sAssessmentFile_CallBacksOnControlFlowGraphs_And_ExternalSinks;
sAssessmentFile_CallBacksOnEdges_And_ExternalSinks = sApplicationToScan +
sAssessmentFile_CallBacksOnEdges_And_ExternalSinks;
sAssessmentFile_SourcesAndSinks = sApplicationToScan + sAssessmentFile_SourcesAndSinks;
sAssessmentFile_CirDataScan = sApplicationToScan + sAssessmentFile_CirDataScan;
bool bContinueWithScans = true;
try
{
DI.log.debug("Scanning Application: {0}", applicationToScan);
sTargetScan = applicationToScan;
// Utils.debugBreak();
if (bScanWithExistingRules)
{
_scanApplication(applicationToScan, sAssessmentFile_ScanWithExistingRules);
bContinueWithScans = false;
// no need to do anything else since we don't want to change the existing rule set (which is what we need the CirDump for)
}
if (bContinueWithScans && bScanWithNoRules)
{
mySqlRules_OunceV6.DeleteAllRulesFromDatabase();
_scanApplication(applicationToScan, sAssessmentFile_ScanWithNoRules);
bContinueWithScans = false; // since this is only used to tests (to make sure we can scan it
}
// CreateCirDataFile (using rules so that the CIR creation process is as quick as possible)
if (bContinueWithScans)
{
if (bCreateCirDataFile)
{
if (sPathToCirDumpFiles == "")
{
sPathToCirDumpFiles =
Files.checkIfDirectoryExistsAndCreateIfNot(
Path.Combine(DI.config.O2TempDir, "_CirDumps"));
}
if (CirDumps.preCirDumpGeneration(sPathToCirDumpFiles))
{
if (bDeleteAllRulesForCirCreation)
{
mySqlRules_OunceV6.DeleteAllRulesFromDatabase();
}
_scanApplication(applicationToScan, sAssessmentFile_CirDataScan);
}
}
// CallBacksOnControlFlowGraphs_And_ExternalSinks
if (bCallBacksOnControlFlowGraphs_And_ExternalSinks)
{
//O2RulePack orpO2RulePack = OunceRules.createRules_CallBacksOnControlFlowGraphs_And_ExternalSinks(this.sCirDataFile);
String sRulePackFile = sCirDataFile + "_CallBacksOnControlFlowGraphs_And_ExternalSinks" +
".O2RulePack";
if (File.Exists(sRulePackFile) == false)
{
DI.log.error("in scanApplication: Could not file rule pack to load :{0}", sRulePackFile);
}
else
{
O2RulePack orpO2RulePack = O2RulePackUtils.loadRulePack(sRulePackFile);
mySqlRules_OunceV6.DeleteAllRulesFromDatabase();
mySqlRules_OunceV6.addRulesToDatabase(true, orpO2RulePack);
_scanApplication(applicationToScan,
sAssessmentFile_CallBacksOnControlFlowGraphs_And_ExternalSinks);
}
}
// CallBacksOnEdges_And_ExternalSinks
if (bCallBacksOnEdges_And_ExternalSinks)
{
//O2RulePack orpO2RulePack = OunceRules.createRules_CallBacksOnEdges_And_ExternalSinks(this.sCirDataFile);
String sRulePackFile = sCirDataFile + "_CallBacksOnEdges_And_ExternalSinks" + ".O2RulePack";
if (File.Exists(sRulePackFile) == false)
{
DI.log.error("in scanApplication: Could not file rule pack to load :{0}", sRulePackFile);
}
else
{
O2RulePack orpO2RulePack = O2RulePackUtils.loadRulePack(sRulePackFile);
mySqlRules_OunceV6.DeleteAllRulesFromDatabase();
mySqlRules_OunceV6.addRulesToDatabase(true, orpO2RulePack);
_scanApplication(applicationToScan, sAssessmentFile_CallBacksOnEdges_And_ExternalSinks);
}
}
// bSourcesAndSinks
if (bSourcesAndSinks)
{
//O2RulePack orpO2RulePack = OunceRules.createRules_SourcesAndSinks(this.sCirDataFile);
String sRulePackFile = sCirDataFile + "_SourcesAndSinks" + ".O2RulePack";
if (File.Exists(sRulePackFile) == false)
{
DI.log.error("in scanApplication: Could not file rule pack to load :{0}", sRulePackFile);
}
else
{
O2RulePack orpO2RulePack = O2RulePackUtils.loadRulePack(sRulePackFile);
mySqlRules_OunceV6.DeleteAllRulesFromDatabase();
mySqlRules_OunceV6.addRulesToDatabase(true, orpO2RulePack);
_scanApplication(applicationToScan, sAssessmentFile_SourcesAndSinks);
}
}
}
}
catch (Exception ex)
{
DI.log.error("in scanApplication:{0}", ex.Message);
return(false);
}
if (bRestartIISAfterScan)
{
new Thread(Processes.resetIIS).Start();
}
if (dProcessCompletionCallback != null)
{
dProcessCompletionCallback.Invoke(this);
}
return(true);
}