private void OutputProjectInfoRecords(IOutputTransaction trx, ProjectDescriptor project)
{
var flat = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(project, flat);
flat.Add("LastCrawlDate", DateTime.Now);
flat.Add(PropertyKeys.KEY_PRESET, project.PresetName);
flat.Add("Policies", project.Policies);
foreach (var lastScanProduct in project.LatestScanDateByProduct.Keys)
{
flat.Add($"{lastScanProduct}_LastScanDate",
project.LatestScanDateByProduct[lastScanProduct]);
}
foreach (var scanCountProduct in project.ScanCountByProduct.Keys)
{
flat.Add($"{scanCountProduct}_Scans",
project.ScanCountByProduct[scanCountProduct]);
}
if (project.CustomFields != null && project.CustomFields.Count > 0)
{
flat.Add(PropertyKeys.KEY_CUSTOMFIELDS, project.CustomFields);
}
trx.write(ProjectInfoOut, flat);
}
private void OutputRecords(SqlDataReader reader, IOutputTransaction trx, IRecordRef record,
Dictionary <String, Func <Object, Object> > customColumnConverters = null)
{
int count = 0;
while (reader.Read())
{
SortedDictionary <String, Object> rec = new SortedDictionary <string, object>();
for (int x = 0; x < reader.FieldCount; x++)
{
var colName = reader.GetColumnSchema()[x].ColumnName;
var insertVal = reader[x];
if (insertVal.GetType() == typeof(System.DBNull))
{
continue;
}
if (customColumnConverters != null && customColumnConverters.ContainsKey(colName))
{
insertVal = customColumnConverters[colName](reader[x]);
}
rec.Add(reader.GetColumnSchema()[x].ColumnName, insertVal);
}
trx.write(record, rec);
count++;
}
_log.Trace($"Wrote {count} audit records.");
}
private void ScaReportOutput(IOutputTransaction trx, ScanDescriptor sd)
{
Dictionary <String, CxOsaLicenses.License> licenseIndex =
new Dictionary <string, CxOsaLicenses.License>();
Dictionary <String, int> licenseCount =
new Dictionary <string, int>();
try
{
var licenses = CxOsaLicenses.GetLicenses(RestContext, CancelToken, sd.ScanId);
foreach (var l in licenses)
{
licenseIndex.Add(l.LicenseId, l);
if (licenseCount.ContainsKey(l.RiskLevel))
{
licenseCount[l.RiskLevel]++;
}
else
{
licenseCount.Add(l.RiskLevel, 1);
}
}
}
catch (Exception ex)
{
_log.Warn($"Could not obtain license data for scan {sd.ScanId} in project " +
$"{sd.Project.ProjectId}: {sd.Project.ProjectName}. License data will not be" +
$" available.", ex);
}
Dictionary <String, CxOsaLibraries.Library> libraryIndex =
new Dictionary <string, CxOsaLibraries.Library>();
try
{
var libraries = CxOsaLibraries.GetLibraries(RestContext, CancelToken, sd.ScanId);
foreach (var lib in libraries)
{
libraryIndex.Add(lib.LibraryId, lib);
}
}
catch (Exception ex)
{
_log.Warn($"Could not obtain library data for scan {sd.ScanId} in project " +
$"{sd.Project.ProjectId}: {sd.Project.ProjectName}. Library data will not be" +
$" available.", ex);
}
OutputScaScanSummary(trx, sd, licenseCount);
OutputScaScanDetails(trx, sd, licenseIndex, libraryIndex);
}
private void OutputPolicyViolationDetails(IOutputTransaction trx, ScanDescriptor scan)
{
var header = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(scan.Project, header);
header.Add(PropertyKeys.KEY_SCANID, scan.ScanId);
header.Add(PropertyKeys.KEY_SCANPRODUCT, scan.ScanProduct.ToString());
header.Add(PropertyKeys.KEY_SCANTYPE, scan.ScanType);
var violatedRules = PolicyViolations[scan.Project.ProjectId].
GetViolatedRulesByScanId(scan.ScanId);
if (violatedRules != null)
{
foreach (var rule in violatedRules)
{
var flat = new SortedDictionary <String, Object>(header);
flat.Add("PolicyName", Policies.GetPolicyById(rule.PolicyId).Name);
flat.Add("RuleId", rule.RuleId);
flat.Add("RuleName", rule.Name);
flat.Add("RuleDescription", rule.Description);
flat.Add("RuleType", rule.RuleType);
flat.Add("RuleCreateDate", rule.CreatedOn);
flat.Add("FirstViolationDetectionDate", rule.FirstDetectionDate);
flat.Add("ViolationName", rule.ViolationName);
if (rule.ViolationOccured.HasValue)
{
flat.Add("ViolationOccurredDate", rule.ViolationOccured.Value);
}
if (rule.ViolationRiskScore.HasValue)
{
flat.Add("ViolationRiskScore", rule.ViolationRiskScore.Value);
}
flat.Add("ViolationSeverity", rule.ViolationSeverity);
if (rule.ViolationSource != null)
{
flat.Add("ViolationSource", rule.ViolationSource);
}
flat.Add("ViolationState", rule.ViolationState);
flat.Add("ViolationStatus", rule.ViolationStatus);
flat.Add("ViolationId", rule.ViolationId);
if (rule.ViolationType != null)
{
flat.Add("ViolationType", rule.ViolationType);
}
trx.write(PolicyViolationDetailOut, flat);
}
}
}
public void CxDB_accesscontrol_AuditTrail(DateTime sinceDate, IOutputTransaction trx, IRecordRef record)
{
Func <Object, Object> detailsConverter = (val) =>
{
var serializer = JsonSerializer.Create();
using (JsonTextReader jtr = new JsonTextReader(new StringReader(val as String)))
return(serializer.Deserialize <SortedDictionary <String, Object> >(jtr));
};
using (var reader = _db.FetchRecords_CxDB_accesscontrol_AuditTrail(sinceDate))
OutputRecords(reader, trx, record, new Dictionary <string, Func <object, object> >
{
{ "Details", detailsConverter }
});
}
private void OutputSASTScanSummary(IOutputTransaction trx, ScanDescriptor scanRecord)
{
if (SastScanSummaryOut == null)
{
return;
}
var flat = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(scanRecord.Project, flat);
flat.Add(PropertyKeys.KEY_SCANID, scanRecord.ScanId);
flat.Add(PropertyKeys.KEY_SCANPRODUCT, scanRecord.ScanProduct.ToString());
flat.Add(PropertyKeys.KEY_SCANTYPE, scanRecord.ScanType);
flat.Add(PropertyKeys.KEY_SCANFINISH, scanRecord.FinishedStamp);
flat.Add(PropertyKeys.KEY_SCANSTART, SastScanCache[scanRecord.ScanId].StartTime);
flat.Add(PropertyKeys.KEY_ENGINESTART, SastScanCache[scanRecord.ScanId].EngineStartTime);
flat.Add(PropertyKeys.KEY_ENGINEFINISH, SastScanCache[scanRecord.ScanId].EngineFinishTime);
flat.Add(PropertyKeys.KEY_SCANRISK, SastScanCache[scanRecord.ScanId].ScanRisk);
flat.Add(PropertyKeys.KEY_SCANRISKSEV, SastScanCache[scanRecord.ScanId].ScanRiskSeverity);
flat.Add("LinesOfCode", SastScanCache[scanRecord.ScanId].LinesOfCode);
flat.Add("FailedLinesOfCode", SastScanCache[scanRecord.ScanId].FailedLinesOfCode);
flat.Add("FileCount", SastScanCache[scanRecord.ScanId].FileCount);
flat.Add("CxVersion", SastScanCache[scanRecord.ScanId].CxVersion);
flat.Add("Languages", SastScanCache[scanRecord.ScanId].Languages);
flat.Add(PropertyKeys.KEY_PRESET, scanRecord.Preset);
flat.Add("Initiator", scanRecord.Initiator);
flat.Add("DeepLink", scanRecord.DeepLink);
flat.Add("ScanTime", scanRecord.ScanTime);
flat.Add("ReportCreationTime", scanRecord.ReportCreateTime);
flat.Add("ScanComments", scanRecord.Comments);
flat.Add("SourceOrigin", scanRecord.SourceOrigin);
foreach (var sev in scanRecord.SeverityCounts.Keys)
{
flat.Add(sev, scanRecord.SeverityCounts[sev]);
}
AddPolicyViolationProperties(scanRecord, flat);
trx.write(SastScanSummaryOut, flat);
}
private void OutputScaScanSummary(IOutputTransaction trx, ScanDescriptor sd, Dictionary <string, int> licenseCount)
{
var flat = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(sd.Project, flat);
AddPolicyViolationProperties(sd, flat);
flat.Add(PropertyKeys.KEY_SCANID, sd.ScanId);
flat.Add(PropertyKeys.KEY_SCANSTART, ScaScanCache[sd.ScanId].StartTime);
flat.Add(PropertyKeys.KEY_SCANFINISH, ScaScanCache[sd.ScanId].FinishTime);
foreach (var k in licenseCount.Keys)
{
flat.Add($"Legal{k}", licenseCount[k]);
}
try
{
var summary = CxOsaSummaryReport.GetReport(RestContext, CancelToken, sd.ScanId);
flat.Add("HighVulnerabilityLibraries", summary.HighVulnerabilityLibraries);
flat.Add("LowVulnerabilityLibraries", summary.LowVulnerabilityLibraries);
flat.Add("MediumVulnerabilityLibraries", summary.MediumVulnerabilityLibraries);
flat.Add("NonVulnerableLibraries", summary.NonVulnerableLibraries);
flat.Add("TotalHighVulnerabilities", summary.TotalHighVulnerabilities);
flat.Add("TotalLibraries", summary.TotalLibraries);
flat.Add("TotalLowVulnerabilities", summary.TotalLowVulnerabilities);
flat.Add("TotalMediumVulnerabilities", summary.TotalMediumVulnerabilities);
flat.Add("VulnerabilityScore", summary.VulnerabilityScore);
flat.Add("VulnerableAndOutdated", summary.VulnerableAndOutdated);
flat.Add("VulnerableAndUpdated", summary.VulnerableAndUpdated);
}
catch (Exception ex)
{
_log.Warn($"Error obtaining summary report for SCA scan {sd.ScanId} " +
$"in project {sd.Project.ProjectName}", ex);
}
trx.write(ScaScanSummaryOut, flat);
}
private void SastReportOutput(IOutputTransaction trx, ScanDescriptor scan)
{
_log.Debug($"Retrieving XML Report for scan {scan.ScanId}");
try
{
using (var report = CxSastXmlReport.GetXmlReport(RestContext,
CancelToken, scan.ScanId))
{
_log.Debug($"XML Report for scan {scan.ScanId} retrieved.");
ProcessSASTReport(trx, scan, report);
_log.Debug($"XML Report for scan {scan.ScanId} processed.");
}
OutputSASTScanSummary(trx, scan);
}
catch (AggregateException aex)
{
_log.Warn($"Multiple exceptions caught attempting to retrieve the SAST XML report for {scan.ScanId}" +
$" in project {scan.Project.ProjectId}: {scan.Project.ProjectName}. ");
_log.Warn("BEGIN exception report");
int exCount = 0;
aex.Handle((x) =>
{
_log.Warn($"Exception #{++exCount}", x);
return(true);
});
_log.Warn("END exception report");
}
catch (Exception ex)
{
_log.Warn($"Error attempting to retrieve the SAST XML report for {scan.ScanId}" +
$" in project {scan.Project.ProjectId}: {scan.Project.ProjectName}. ", ex);
}
}
private void InvokeCrawlMethod(String methodName, IAuditTrailCrawler crawler, IOutputTransaction trx, CancellationToken token)
{
var recordRef = _outMappings[methodName];
crawler.GetType().InvokeMember(methodName, BindingFlags.InvokeMethod, null, crawler, new object[]
{
SinceDate,
trx,
recordRef
});
}
public void CxActivity_dbo_Audit_Logins(DateTime sinceDate, IOutputTransaction trx, IRecordRef record)
{
using (var reader = _db.FetchRecords_CxActivity_dbo_Audit_Logins(sinceDate))
OutputRecords(reader, trx, record);
}
private void ProcessSASTReport(IOutputTransaction trx, ScanDescriptor scan, Stream report)
{
var reportRec = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(scan.Project, reportRec);
reportRec.Add(PropertyKeys.KEY_SCANID, scan.ScanId);
reportRec.Add(PropertyKeys.KEY_SCANPRODUCT, scan.ScanProduct.ToString());
reportRec.Add(PropertyKeys.KEY_SCANTYPE, scan.ScanType);
reportRec.Add(PropertyKeys.KEY_SCANFINISH, scan.FinishedStamp);
SortedDictionary <String, Object> curResultRec = null;
SortedDictionary <String, Object> curQueryRec = null;
SortedDictionary <String, Object> curPath = null;
SortedDictionary <String, Object> curPathNode = null;
bool inSnippet = false;
using (XmlReader xr = XmlReader.Create(report))
{
while (xr.Read())
{
if (xr.NodeType == XmlNodeType.Element)
{
if (xr.Name.CompareTo("CxXMLResults") == 0)
{
_log.Trace($"[Scan: {scan.ScanId}] Processing attributes in CxXMLResults.");
scan.Preset = xr.GetAttribute("Preset");
scan.Initiator = xr.GetAttribute("InitiatorName");
scan.DeepLink = xr.GetAttribute("DeepLink");
scan.ScanTime = xr.GetAttribute("ScanTime");
scan.ReportCreateTime = DateTime.Parse(xr.GetAttribute("ReportCreationTime"));
scan.Comments = xr.GetAttribute("ScanComments");
scan.SourceOrigin = xr.GetAttribute("SourceOrigin");
continue;
}
if (xr.Name.CompareTo("Query") == 0)
{
_log.Trace($"[Scan: {scan.ScanId}] Processing attributes in Query " +
$"[{xr.GetAttribute("id")} - {xr.GetAttribute("name")}].");
curQueryRec = new SortedDictionary <String, Object>
(reportRec);
curQueryRec.Add("QueryCategories", xr.GetAttribute("categories"));
curQueryRec.Add("QueryId", xr.GetAttribute("id"));
curQueryRec.Add("QueryCweId", xr.GetAttribute("cweId"));
curQueryRec.Add("QueryName", xr.GetAttribute("name"));
curQueryRec.Add("QueryGroup", xr.GetAttribute("group"));
curQueryRec.Add("QuerySeverity", xr.GetAttribute("Severity"));
curQueryRec.Add("QueryLanguage", xr.GetAttribute("Language"));
curQueryRec.Add("QueryVersionCode", xr.GetAttribute("QueryVersionCode"));
continue;
}
if (xr.Name.CompareTo("Result") == 0)
{
_log.Trace($"[Scan: {scan.ScanId}] Processing attributes in Result " +
$"[{xr.GetAttribute("NodeId")}].");
scan.IncrementSeverity(xr.GetAttribute("Severity"));
curResultRec = new SortedDictionary <String, Object>(curQueryRec);
curResultRec.Add("VulnerabilityId", xr.GetAttribute("NodeId"));
curResultRec.Add("SinkFileName", xr.GetAttribute("FileName"));
curResultRec.Add("Status", xr.GetAttribute("Status"));
curResultRec.Add("SinkLine", xr.GetAttribute("Line"));
curResultRec.Add("SinkColumn", xr.GetAttribute("Column"));
curResultRec.Add("FalsePositive", xr.GetAttribute("FalsePositive"));
curResultRec.Add("ResultSeverity", xr.GetAttribute("Severity"));
// TODO: Translate state number to an appropriate string
curResultRec.Add("State", xr.GetAttribute("state"));
curResultRec.Add("Remark", xr.GetAttribute("Remark"));
curResultRec.Add("ResultDeepLink", xr.GetAttribute("DeepLink"));
continue;
}
if (xr.Name.CompareTo("Path") == 0)
{
curPath = new SortedDictionary <String, Object>(curResultRec);
curPath.Add("ResultId", xr.GetAttribute("ResultId"));
curPath.Add("PathId", xr.GetAttribute("PathId"));
curPath.Add(PropertyKeys.KEY_SIMILARITYID, xr.GetAttribute("SimilarityId"));
continue;
}
if (xr.Name.CompareTo("PathNode") == 0)
{
curPathNode = new SortedDictionary <String, Object>(curPath);
continue;
}
if (xr.Name.CompareTo("FileName") == 0 && curPathNode != null)
{
curPathNode.Add("NodeFileName", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Line") == 0 && curPathNode != null && !inSnippet)
{
curPathNode.Add("NodeLine", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Column") == 0 && curPathNode != null)
{
curPathNode.Add("NodeColumn", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("NodeId") == 0 && curPathNode != null)
{
curPathNode.Add("NodeId", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Name") == 0 && curPathNode != null)
{
curPathNode.Add("NodeName", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Type") == 0 && curPathNode != null)
{
curPathNode.Add("NodeType", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Length") == 0 && curPathNode != null)
{
curPathNode.Add("NodeLength", xr.ReadElementContentAsString());
continue;
}
if (xr.Name.CompareTo("Snippet") == 0 && curPathNode != null)
{
inSnippet = true;
continue;
}
if (xr.Name.CompareTo("Code") == 0 && curPathNode != null)
{
curPathNode.Add("NodeCodeSnippet", xr.ReadElementContentAsString());
continue;
}
}
if (xr.NodeType == XmlNodeType.EndElement)
{
if (xr.Name.CompareTo("CxXMLResults") == 0)
{
_log.Trace($"[Scan: {scan.ScanId}] Finished processing CxXMLResults");
continue;
}
if (xr.Name.CompareTo("Query") == 0)
{
curQueryRec = null;
continue;
}
if (xr.Name.CompareTo("Result") == 0)
{
curResultRec = null;
continue;
}
if (xr.Name.CompareTo("Path") == 0)
{
curPath = null;
continue;
}
if (xr.Name.CompareTo("PathNode") == 0)
{
trx.write(SastScanDetailOut, curPathNode);
curPathNode = null;
continue;
}
if (xr.Name.CompareTo("Snippet") == 0)
{
inSnippet = false;
continue;
}
}
}
}
}
private void OutputScaScanDetails(IOutputTransaction trx, ScanDescriptor sd, Dictionary <string, CxOsaLicenses.License> licenseIndex,
Dictionary <string, CxOsaLibraries.Library> libraryIndex)
{
try
{
var vulns = CxOsaVulnerabilities.GetVulnerabilities(RestContext,
CancelToken, sd.ScanId);
var header = new SortedDictionary <String, Object>();
AddPrimaryKeyElements(sd.Project, header);
header.Add(PropertyKeys.KEY_SCANFINISH, sd.FinishedStamp);
foreach (var vuln in vulns)
{
var flat = new SortedDictionary <String, Object>(header);
flat.Add(PropertyKeys.KEY_SCANID, sd.ScanId);
flat.Add("VulnerabilityId", vuln.VulerabilityId);
flat.Add(PropertyKeys.KEY_SIMILARITYID, vuln.SimilarityId);
flat.Add("CVEName", vuln.CVEName);
flat.Add("CVEDescription", vuln.CVEDescription);
flat.Add("CVEUrl", vuln.CVEUrl);
flat.Add("CVEPubDate", vuln.CVEPublishDate);
flat.Add("CVEScore", vuln.CVEScore);
flat.Add("Recommendation", vuln.Recommendations);
flat.Add(PropertyKeys.KEY_SCANRISKSEV, vuln.Severity.Name);
flat.Add("State", vuln.State.StateName);
flat.Add("LibraryId", vuln.LibraryId);
var lib = libraryIndex[vuln.LibraryId];
if (lib != null)
{
flat.Add("LibraryName", lib.LibraryName);
flat.Add("LibraryVersion", lib.LibraryVersion);
flat.Add("LibraryReleaseDate", lib.ReleaseDate);
flat.Add("LibraryLatestVersion", lib.LatestVersion);
flat.Add("LibraryLatestReleaseDate", lib.LatestVersionReleased);
}
StringBuilder licenseStr = new StringBuilder();
foreach (var license in lib.Licenses)
{
if (licenseStr.Length > 0)
{
licenseStr.Append(";");
}
licenseStr.Append(licenseIndex[license].LicenseName);
flat.Add($"LibraryLegalRisk_{licenseIndex[license].LicenseName.Replace(" ", "")}",
licenseIndex[license].RiskLevel);
}
flat.Add("LibraryLicenses", licenseStr.ToString());
trx.write(ScaScanDetailOut, flat);
}
}
catch (Exception ex)
{
_log.Warn($"Could not obtain vulnerability data for scan {sd.ScanId} in project " +
$"{sd.Project.ProjectId}: {sd.Project.ProjectName}. Vulnerability data will not be" +
$" available.", ex);
}
}