Пример #1
0
        private void OutputProjectInfoRecords(IOutputTransaction trx, ProjectDescriptor project)
        {
            var flat = new SortedDictionary <String, Object>();

            AddPrimaryKeyElements(project, flat);

            flat.Add("LastCrawlDate", DateTime.Now);

            flat.Add(PropertyKeys.KEY_PRESET, project.PresetName);
            flat.Add("Policies", project.Policies);

            foreach (var lastScanProduct in project.LatestScanDateByProduct.Keys)
            {
                flat.Add($"{lastScanProduct}_LastScanDate",
                         project.LatestScanDateByProduct[lastScanProduct]);
            }

            foreach (var scanCountProduct in project.ScanCountByProduct.Keys)
            {
                flat.Add($"{scanCountProduct}_Scans",
                         project.ScanCountByProduct[scanCountProduct]);
            }

            if (project.CustomFields != null && project.CustomFields.Count > 0)
            {
                flat.Add(PropertyKeys.KEY_CUSTOMFIELDS, project.CustomFields);
            }

            trx.write(ProjectInfoOut, flat);
        }
Пример #2
0
        private void OutputRecords(SqlDataReader reader, IOutputTransaction trx, IRecordRef record,
                                   Dictionary <String, Func <Object, Object> > customColumnConverters = null)
        {
            int count = 0;

            while (reader.Read())
            {
                SortedDictionary <String, Object> rec = new SortedDictionary <string, object>();

                for (int x = 0; x < reader.FieldCount; x++)
                {
                    var colName = reader.GetColumnSchema()[x].ColumnName;

                    var insertVal = reader[x];

                    if (insertVal.GetType() == typeof(System.DBNull))
                    {
                        continue;
                    }

                    if (customColumnConverters != null && customColumnConverters.ContainsKey(colName))
                    {
                        insertVal = customColumnConverters[colName](reader[x]);
                    }


                    rec.Add(reader.GetColumnSchema()[x].ColumnName, insertVal);
                }

                trx.write(record, rec);
                count++;
            }

            _log.Trace($"Wrote {count} audit records.");
        }
Пример #3
0
        private void ScaReportOutput(IOutputTransaction trx, ScanDescriptor sd)
        {
            Dictionary <String, CxOsaLicenses.License> licenseIndex =
                new Dictionary <string, CxOsaLicenses.License>();

            Dictionary <String, int> licenseCount =
                new Dictionary <string, int>();

            try
            {
                var licenses = CxOsaLicenses.GetLicenses(RestContext, CancelToken, sd.ScanId);

                foreach (var l in licenses)
                {
                    licenseIndex.Add(l.LicenseId, l);

                    if (licenseCount.ContainsKey(l.RiskLevel))
                    {
                        licenseCount[l.RiskLevel]++;
                    }
                    else
                    {
                        licenseCount.Add(l.RiskLevel, 1);
                    }
                }
            }
            catch (Exception ex)
            {
                _log.Warn($"Could not obtain license data for scan {sd.ScanId} in project " +
                          $"{sd.Project.ProjectId}: {sd.Project.ProjectName}.  License data will not be" +
                          $" available.", ex);
            }

            Dictionary <String, CxOsaLibraries.Library> libraryIndex =
                new Dictionary <string, CxOsaLibraries.Library>();


            try
            {
                var libraries = CxOsaLibraries.GetLibraries(RestContext, CancelToken, sd.ScanId);

                foreach (var lib in libraries)
                {
                    libraryIndex.Add(lib.LibraryId, lib);
                }
            }
            catch (Exception ex)
            {
                _log.Warn($"Could not obtain library data for scan {sd.ScanId} in project " +
                          $"{sd.Project.ProjectId}: {sd.Project.ProjectName}.  Library data will not be" +
                          $" available.", ex);
            }

            OutputScaScanSummary(trx, sd, licenseCount);

            OutputScaScanDetails(trx, sd, licenseIndex, libraryIndex);
        }
Пример #4
0
        private void OutputPolicyViolationDetails(IOutputTransaction trx, ScanDescriptor scan)
        {
            var header = new SortedDictionary <String, Object>();

            AddPrimaryKeyElements(scan.Project, header);
            header.Add(PropertyKeys.KEY_SCANID, scan.ScanId);
            header.Add(PropertyKeys.KEY_SCANPRODUCT, scan.ScanProduct.ToString());
            header.Add(PropertyKeys.KEY_SCANTYPE, scan.ScanType);

            var violatedRules = PolicyViolations[scan.Project.ProjectId].
                                GetViolatedRulesByScanId(scan.ScanId);

            if (violatedRules != null)
            {
                foreach (var rule in violatedRules)
                {
                    var flat = new SortedDictionary <String, Object>(header);
                    flat.Add("PolicyName", Policies.GetPolicyById(rule.PolicyId).Name);
                    flat.Add("RuleId", rule.RuleId);
                    flat.Add("RuleName", rule.Name);
                    flat.Add("RuleDescription", rule.Description);
                    flat.Add("RuleType", rule.RuleType);
                    flat.Add("RuleCreateDate", rule.CreatedOn);
                    flat.Add("FirstViolationDetectionDate", rule.FirstDetectionDate);
                    flat.Add("ViolationName", rule.ViolationName);
                    if (rule.ViolationOccured.HasValue)
                    {
                        flat.Add("ViolationOccurredDate", rule.ViolationOccured.Value);
                    }
                    if (rule.ViolationRiskScore.HasValue)
                    {
                        flat.Add("ViolationRiskScore", rule.ViolationRiskScore.Value);
                    }
                    flat.Add("ViolationSeverity", rule.ViolationSeverity);
                    if (rule.ViolationSource != null)
                    {
                        flat.Add("ViolationSource", rule.ViolationSource);
                    }
                    flat.Add("ViolationState", rule.ViolationState);
                    flat.Add("ViolationStatus", rule.ViolationStatus);
                    flat.Add("ViolationId", rule.ViolationId);
                    if (rule.ViolationType != null)
                    {
                        flat.Add("ViolationType", rule.ViolationType);
                    }

                    trx.write(PolicyViolationDetailOut, flat);
                }
            }
        }
Пример #5
0
        public void CxDB_accesscontrol_AuditTrail(DateTime sinceDate, IOutputTransaction trx, IRecordRef record)
        {
            Func <Object, Object> detailsConverter = (val) =>
            {
                var serializer = JsonSerializer.Create();

                using (JsonTextReader jtr = new JsonTextReader(new StringReader(val as String)))
                    return(serializer.Deserialize <SortedDictionary <String, Object> >(jtr));
            };

            using (var reader = _db.FetchRecords_CxDB_accesscontrol_AuditTrail(sinceDate))
                OutputRecords(reader, trx, record, new Dictionary <string, Func <object, object> >
                {
                    { "Details", detailsConverter }
                });
        }
Пример #6
0
        private void OutputSASTScanSummary(IOutputTransaction trx, ScanDescriptor scanRecord)
        {
            if (SastScanSummaryOut == null)
            {
                return;
            }

            var flat = new SortedDictionary <String, Object>();

            AddPrimaryKeyElements(scanRecord.Project, flat);
            flat.Add(PropertyKeys.KEY_SCANID, scanRecord.ScanId);
            flat.Add(PropertyKeys.KEY_SCANPRODUCT, scanRecord.ScanProduct.ToString());
            flat.Add(PropertyKeys.KEY_SCANTYPE, scanRecord.ScanType);
            flat.Add(PropertyKeys.KEY_SCANFINISH, scanRecord.FinishedStamp);
            flat.Add(PropertyKeys.KEY_SCANSTART, SastScanCache[scanRecord.ScanId].StartTime);
            flat.Add(PropertyKeys.KEY_ENGINESTART, SastScanCache[scanRecord.ScanId].EngineStartTime);
            flat.Add(PropertyKeys.KEY_ENGINEFINISH, SastScanCache[scanRecord.ScanId].EngineFinishTime);
            flat.Add(PropertyKeys.KEY_SCANRISK, SastScanCache[scanRecord.ScanId].ScanRisk);
            flat.Add(PropertyKeys.KEY_SCANRISKSEV, SastScanCache[scanRecord.ScanId].ScanRiskSeverity);
            flat.Add("LinesOfCode", SastScanCache[scanRecord.ScanId].LinesOfCode);
            flat.Add("FailedLinesOfCode", SastScanCache[scanRecord.ScanId].FailedLinesOfCode);
            flat.Add("FileCount", SastScanCache[scanRecord.ScanId].FileCount);
            flat.Add("CxVersion", SastScanCache[scanRecord.ScanId].CxVersion);
            flat.Add("Languages", SastScanCache[scanRecord.ScanId].Languages);
            flat.Add(PropertyKeys.KEY_PRESET, scanRecord.Preset);
            flat.Add("Initiator", scanRecord.Initiator);
            flat.Add("DeepLink", scanRecord.DeepLink);
            flat.Add("ScanTime", scanRecord.ScanTime);
            flat.Add("ReportCreationTime", scanRecord.ReportCreateTime);
            flat.Add("ScanComments", scanRecord.Comments);
            flat.Add("SourceOrigin", scanRecord.SourceOrigin);
            foreach (var sev in scanRecord.SeverityCounts.Keys)
            {
                flat.Add(sev, scanRecord.SeverityCounts[sev]);
            }

            AddPolicyViolationProperties(scanRecord, flat);

            trx.write(SastScanSummaryOut, flat);
        }
Пример #7
0
        private void OutputScaScanSummary(IOutputTransaction trx, ScanDescriptor sd, Dictionary <string, int> licenseCount)
        {
            var flat = new SortedDictionary <String, Object>();

            AddPrimaryKeyElements(sd.Project, flat);
            AddPolicyViolationProperties(sd, flat);
            flat.Add(PropertyKeys.KEY_SCANID, sd.ScanId);
            flat.Add(PropertyKeys.KEY_SCANSTART, ScaScanCache[sd.ScanId].StartTime);
            flat.Add(PropertyKeys.KEY_SCANFINISH, ScaScanCache[sd.ScanId].FinishTime);

            foreach (var k in licenseCount.Keys)
            {
                flat.Add($"Legal{k}", licenseCount[k]);
            }


            try
            {
                var summary = CxOsaSummaryReport.GetReport(RestContext, CancelToken, sd.ScanId);

                flat.Add("HighVulnerabilityLibraries", summary.HighVulnerabilityLibraries);
                flat.Add("LowVulnerabilityLibraries", summary.LowVulnerabilityLibraries);
                flat.Add("MediumVulnerabilityLibraries", summary.MediumVulnerabilityLibraries);
                flat.Add("NonVulnerableLibraries", summary.NonVulnerableLibraries);
                flat.Add("TotalHighVulnerabilities", summary.TotalHighVulnerabilities);
                flat.Add("TotalLibraries", summary.TotalLibraries);
                flat.Add("TotalLowVulnerabilities", summary.TotalLowVulnerabilities);
                flat.Add("TotalMediumVulnerabilities", summary.TotalMediumVulnerabilities);
                flat.Add("VulnerabilityScore", summary.VulnerabilityScore);
                flat.Add("VulnerableAndOutdated", summary.VulnerableAndOutdated);
                flat.Add("VulnerableAndUpdated", summary.VulnerableAndUpdated);
            }
            catch (Exception ex)
            {
                _log.Warn($"Error obtaining summary report for SCA scan {sd.ScanId} " +
                          $"in project {sd.Project.ProjectName}", ex);
            }

            trx.write(ScaScanSummaryOut, flat);
        }
Пример #8
0
        private void SastReportOutput(IOutputTransaction trx, ScanDescriptor scan)
        {
            _log.Debug($"Retrieving XML Report for scan {scan.ScanId}");
            try
            {
                using (var report = CxSastXmlReport.GetXmlReport(RestContext,
                                                                 CancelToken, scan.ScanId))
                {
                    _log.Debug($"XML Report for scan {scan.ScanId} retrieved.");
                    ProcessSASTReport(trx, scan, report);
                    _log.Debug($"XML Report for scan {scan.ScanId} processed.");
                }

                OutputSASTScanSummary(trx, scan);
            }
            catch (AggregateException aex)
            {
                _log.Warn($"Multiple exceptions caught attempting to retrieve the SAST XML report for {scan.ScanId}" +
                          $" in project {scan.Project.ProjectId}: {scan.Project.ProjectName}. ");

                _log.Warn("BEGIN exception report");

                int exCount = 0;

                aex.Handle((x) =>
                {
                    _log.Warn($"Exception #{++exCount}", x);

                    return(true);
                });

                _log.Warn("END exception report");
            }
            catch (Exception ex)
            {
                _log.Warn($"Error attempting to retrieve the SAST XML report for {scan.ScanId}" +
                          $" in project {scan.Project.ProjectId}: {scan.Project.ProjectName}. ", ex);
            }
        }
Пример #9
0
        private void InvokeCrawlMethod(String methodName, IAuditTrailCrawler crawler, IOutputTransaction trx, CancellationToken token)
        {
            var recordRef = _outMappings[methodName];

            crawler.GetType().InvokeMember(methodName, BindingFlags.InvokeMethod, null, crawler, new object[]
            {
                SinceDate,
                trx,
                recordRef
            });
        }
Пример #10
0
 public void CxActivity_dbo_Audit_Logins(DateTime sinceDate, IOutputTransaction trx, IRecordRef record)
 {
     using (var reader = _db.FetchRecords_CxActivity_dbo_Audit_Logins(sinceDate))
         OutputRecords(reader, trx, record);
 }
Пример #11
0
        private void ProcessSASTReport(IOutputTransaction trx, ScanDescriptor scan, Stream report)
        {
            var reportRec = new SortedDictionary <String, Object>();

            AddPrimaryKeyElements(scan.Project, reportRec);
            reportRec.Add(PropertyKeys.KEY_SCANID, scan.ScanId);
            reportRec.Add(PropertyKeys.KEY_SCANPRODUCT, scan.ScanProduct.ToString());
            reportRec.Add(PropertyKeys.KEY_SCANTYPE, scan.ScanType);
            reportRec.Add(PropertyKeys.KEY_SCANFINISH, scan.FinishedStamp);

            SortedDictionary <String, Object> curResultRec = null;
            SortedDictionary <String, Object> curQueryRec  = null;
            SortedDictionary <String, Object> curPath      = null;
            SortedDictionary <String, Object> curPathNode  = null;
            bool inSnippet = false;

            using (XmlReader xr = XmlReader.Create(report))
            {
                while (xr.Read())
                {
                    if (xr.NodeType == XmlNodeType.Element)
                    {
                        if (xr.Name.CompareTo("CxXMLResults") == 0)
                        {
                            _log.Trace($"[Scan: {scan.ScanId}] Processing attributes in CxXMLResults.");

                            scan.Preset           = xr.GetAttribute("Preset");
                            scan.Initiator        = xr.GetAttribute("InitiatorName");
                            scan.DeepLink         = xr.GetAttribute("DeepLink");
                            scan.ScanTime         = xr.GetAttribute("ScanTime");
                            scan.ReportCreateTime = DateTime.Parse(xr.GetAttribute("ReportCreationTime"));
                            scan.Comments         = xr.GetAttribute("ScanComments");
                            scan.SourceOrigin     = xr.GetAttribute("SourceOrigin");
                            continue;
                        }

                        if (xr.Name.CompareTo("Query") == 0)
                        {
                            _log.Trace($"[Scan: {scan.ScanId}] Processing attributes in Query " +
                                       $"[{xr.GetAttribute("id")} - {xr.GetAttribute("name")}].");

                            curQueryRec = new SortedDictionary <String, Object>
                                              (reportRec);

                            curQueryRec.Add("QueryCategories", xr.GetAttribute("categories"));
                            curQueryRec.Add("QueryId", xr.GetAttribute("id"));
                            curQueryRec.Add("QueryCweId", xr.GetAttribute("cweId"));
                            curQueryRec.Add("QueryName", xr.GetAttribute("name"));
                            curQueryRec.Add("QueryGroup", xr.GetAttribute("group"));
                            curQueryRec.Add("QuerySeverity", xr.GetAttribute("Severity"));
                            curQueryRec.Add("QueryLanguage", xr.GetAttribute("Language"));
                            curQueryRec.Add("QueryVersionCode", xr.GetAttribute("QueryVersionCode"));
                            continue;
                        }

                        if (xr.Name.CompareTo("Result") == 0)
                        {
                            _log.Trace($"[Scan: {scan.ScanId}] Processing attributes in Result " +
                                       $"[{xr.GetAttribute("NodeId")}].");

                            scan.IncrementSeverity(xr.GetAttribute("Severity"));

                            curResultRec = new SortedDictionary <String, Object>(curQueryRec);
                            curResultRec.Add("VulnerabilityId", xr.GetAttribute("NodeId"));
                            curResultRec.Add("SinkFileName", xr.GetAttribute("FileName"));
                            curResultRec.Add("Status", xr.GetAttribute("Status"));
                            curResultRec.Add("SinkLine", xr.GetAttribute("Line"));
                            curResultRec.Add("SinkColumn", xr.GetAttribute("Column"));
                            curResultRec.Add("FalsePositive", xr.GetAttribute("FalsePositive"));
                            curResultRec.Add("ResultSeverity", xr.GetAttribute("Severity"));
                            // TODO: Translate state number to an appropriate string
                            curResultRec.Add("State", xr.GetAttribute("state"));
                            curResultRec.Add("Remark", xr.GetAttribute("Remark"));
                            curResultRec.Add("ResultDeepLink", xr.GetAttribute("DeepLink"));
                            continue;
                        }

                        if (xr.Name.CompareTo("Path") == 0)
                        {
                            curPath = new SortedDictionary <String, Object>(curResultRec);
                            curPath.Add("ResultId", xr.GetAttribute("ResultId"));
                            curPath.Add("PathId", xr.GetAttribute("PathId"));
                            curPath.Add(PropertyKeys.KEY_SIMILARITYID, xr.GetAttribute("SimilarityId"));
                            continue;
                        }

                        if (xr.Name.CompareTo("PathNode") == 0)
                        {
                            curPathNode = new SortedDictionary <String, Object>(curPath);
                            continue;
                        }

                        if (xr.Name.CompareTo("FileName") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeFileName", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Line") == 0 && curPathNode != null && !inSnippet)
                        {
                            curPathNode.Add("NodeLine", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Column") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeColumn", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("NodeId") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeId", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Name") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeName", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Type") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeType", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Length") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeLength", xr.ReadElementContentAsString());
                            continue;
                        }

                        if (xr.Name.CompareTo("Snippet") == 0 && curPathNode != null)
                        {
                            inSnippet = true;
                            continue;
                        }

                        if (xr.Name.CompareTo("Code") == 0 && curPathNode != null)
                        {
                            curPathNode.Add("NodeCodeSnippet", xr.ReadElementContentAsString());
                            continue;
                        }
                    }


                    if (xr.NodeType == XmlNodeType.EndElement)
                    {
                        if (xr.Name.CompareTo("CxXMLResults") == 0)
                        {
                            _log.Trace($"[Scan: {scan.ScanId}] Finished processing CxXMLResults");
                            continue;
                        }

                        if (xr.Name.CompareTo("Query") == 0)
                        {
                            curQueryRec = null;
                            continue;
                        }

                        if (xr.Name.CompareTo("Result") == 0)
                        {
                            curResultRec = null;
                            continue;
                        }

                        if (xr.Name.CompareTo("Path") == 0)
                        {
                            curPath = null;
                            continue;
                        }

                        if (xr.Name.CompareTo("PathNode") == 0)
                        {
                            trx.write(SastScanDetailOut, curPathNode);
                            curPathNode = null;
                            continue;
                        }

                        if (xr.Name.CompareTo("Snippet") == 0)
                        {
                            inSnippet = false;
                            continue;
                        }
                    }
                }
            }
        }
Пример #12
0
        private void OutputScaScanDetails(IOutputTransaction trx, ScanDescriptor sd, Dictionary <string, CxOsaLicenses.License> licenseIndex,
                                          Dictionary <string, CxOsaLibraries.Library> libraryIndex)
        {
            try
            {
                var vulns = CxOsaVulnerabilities.GetVulnerabilities(RestContext,
                                                                    CancelToken, sd.ScanId);

                var header = new SortedDictionary <String, Object>();
                AddPrimaryKeyElements(sd.Project, header);
                header.Add(PropertyKeys.KEY_SCANFINISH, sd.FinishedStamp);

                foreach (var vuln in vulns)
                {
                    var flat = new SortedDictionary <String, Object>(header);

                    flat.Add(PropertyKeys.KEY_SCANID, sd.ScanId);

                    flat.Add("VulnerabilityId", vuln.VulerabilityId);
                    flat.Add(PropertyKeys.KEY_SIMILARITYID, vuln.SimilarityId);
                    flat.Add("CVEName", vuln.CVEName);
                    flat.Add("CVEDescription", vuln.CVEDescription);
                    flat.Add("CVEUrl", vuln.CVEUrl);
                    flat.Add("CVEPubDate", vuln.CVEPublishDate);
                    flat.Add("CVEScore", vuln.CVEScore);
                    flat.Add("Recommendation", vuln.Recommendations);
                    flat.Add(PropertyKeys.KEY_SCANRISKSEV, vuln.Severity.Name);
                    flat.Add("State", vuln.State.StateName);


                    flat.Add("LibraryId", vuln.LibraryId);

                    var lib = libraryIndex[vuln.LibraryId];
                    if (lib != null)
                    {
                        flat.Add("LibraryName", lib.LibraryName);
                        flat.Add("LibraryVersion", lib.LibraryVersion);
                        flat.Add("LibraryReleaseDate", lib.ReleaseDate);
                        flat.Add("LibraryLatestVersion", lib.LatestVersion);
                        flat.Add("LibraryLatestReleaseDate", lib.LatestVersionReleased);
                    }

                    StringBuilder licenseStr = new StringBuilder();

                    foreach (var license in lib.Licenses)
                    {
                        if (licenseStr.Length > 0)
                        {
                            licenseStr.Append(";");
                        }
                        licenseStr.Append(licenseIndex[license].LicenseName);

                        flat.Add($"LibraryLegalRisk_{licenseIndex[license].LicenseName.Replace(" ", "")}",
                                 licenseIndex[license].RiskLevel);
                    }

                    flat.Add("LibraryLicenses", licenseStr.ToString());

                    trx.write(ScaScanDetailOut, flat);
                }
            }
            catch (Exception ex)
            {
                _log.Warn($"Could not obtain vulnerability data for scan {sd.ScanId} in project " +
                          $"{sd.Project.ProjectId}: {sd.Project.ProjectName}.  Vulnerability data will not be" +
                          $" available.", ex);
            }
        }