private async Task <ProjectPolicyIndex> PopulatePolicies()
{
// Policies may not have data if M&O is not installed.
try
{
return(await Task.Run(
() =>
{
_log.Debug("Retrieving policies, if available.");
return new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(RestContext, CancelToken));
}, CancelToken));
}
catch (Exception ex)
{
String msg = "Policy data is not available. M&O was unreachable. You can omit the M&O URL in the configuration if M&O is not installed.";
if (_log.IsDebugEnabled)
{
_log.Debug(msg, ex);
}
else
{
_log.Warn(msg);
}
}
return(null);
}
private Transformer(CxRestContext ctx, CancellationToken token,
String previousStatePath)
{
RestContext = ctx;
CancelToken = token;
Policies = null;
_log.Debug("Retrieving policies, if available.");
// Policies may not have data if M&O is not installed.
try
{
Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
}
catch (Exception ex)
{
_log.Warn("Policy data is not available.", ex);
}
// Populate the data resolver with teams and presets
DataResolver dr = new DataResolver();
_log.Debug("Retrieving presets.");
var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);
foreach (var preset in presetEnum)
{
dr.addPreset(preset.PresetId, preset.PresetName);
}
_log.Debug("Retrieving teams.");
var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);
foreach (var team in teamEnum)
{
dr.addTeam(team.TeamId, team.TeamName);
}
_log.Debug("Resolving projects.");
// Now populate the project resolver with the projects
ProjectResolver pr = dr.Resolve(previousStatePath);
var projects = CxProjects.GetProjects(RestContext, CancelToken);
foreach (var p in projects)
{
String combinedPolicyNames = String.Empty;
if (Policies != null)
{
try
{
IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
(ctx, token, p.ProjectId);
Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
}
catch (Exception ex)
{
_log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
$"Policy statistics will be unavalable.", ex);
}
}
var cfDict = new SortedDictionary <String, String>();
p.CustomFields.ForEach((cf) => cfDict.Add(cf.FieldName, cf.FieldValue));
pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames, cfDict);
}
// Resolve projects to get the scan resolver.
ScanResolver sr = pr.Resolve(_mapActions);
try
{
_log.Debug("Retrieving SAST scans.");
var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);
foreach (var sastScan in sastScans)
{
_log.Debug($"SAST scan record: {sastScan}");
sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
sastScan.ScanId, sastScan.FinishTime);
SastScanCache.Add(sastScan.ScanId, sastScan);
}
_log.Debug("Retrieving OSA scans.");
foreach (var p in projects)
{
var scaScans = CxOsaScans.GetScans(ctx, token, p.ProjectId);
foreach (var scaScan in scaScans)
{
_log.Debug($"OSA scan record: {scaScan}");
sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
scaScan.FinishTime);
ScaScanCache.Add(scaScan.ScanId, scaScan);
}
}
ScanDescriptors = sr.Resolve(CheckTime);
}
catch (AggregateException aex)
{
_log.Error($"Multiple errors caught resolving scans.");
int count = 0;
aex.Handle(
(ex) =>
{
_log.Error($"Exception {++count}: ", ex);
return(true);
});
}
catch (Exception ex)
{
_log.Error($"Error resolving scans, server may be unavailable.", ex);
}
}
private Transformer(CxRestContext ctx, CancellationToken token,
String previousStatePath)
{
RestContext = ctx;
CancelToken = token;
Policies = null;
// Policies may not have data if M&O is not installed.
try
{
Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
}
catch (Exception ex)
{
_log.Warn("Policy data is not available.", ex);
}
// Populate the data resolver with teams and presets
DataResolver dr = new DataResolver();
var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);
foreach (var preset in presetEnum)
{
dr.addPreset(preset.PresetId, preset.PresetName);
}
var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);
foreach (var team in teamEnum)
{
dr.addTeam(team.TeamId, team.TeamName);
}
// Now populate the project resolver with the projects
ProjectResolver pr = dr.Resolve(previousStatePath);
var projects = CxProjects.GetProjects(RestContext, CancelToken);
foreach (var p in projects)
{
String combinedPolicyNames = String.Empty;
if (Policies != null)
{
try
{
IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
(ctx, token, p.ProjectId);
Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
}
catch (Exception ex)
{
_log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
$"Policy statistics will be unavalable.", ex);
}
}
pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames);
}
// Resolve projects to get the scan resolver.
ScanResolver sr = pr.Resolve(_mapActions);
try
{
var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);
foreach (var sastScan in sastScans)
{
sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
sastScan.ScanId, sastScan.FinishTime);
SastScanCache.Add(sastScan.ScanId, sastScan);
}
foreach (var p in projects)
{
var scaScans = CxScaScans.GetScans(ctx, token, p.ProjectId);
foreach (var scaScan in scaScans)
{
sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
scaScan.FinishTime);
ScaScanCache.Add(scaScan.ScanId, scaScan);
}
}
ScanDescriptors = sr.Resolve(CheckTime);
}
catch (Exception ex)
{
_log.Error($"Error resolving scans, server may be unavailable.", ex);
}
}
private async Task ResolveScans()
{
var policyTask = PopulatePolicies();
var presetsTask = PopulatePresets();
var teamsTask = PopulateTeams();
_log.Debug("Resolving projects.");
var projects = await Task.Run(() => CxProjects.GetProjects(RestContext, CancelToken), CancelToken);
Policies = await policyTask;
Teams = await teamsTask;
Presets = await presetsTask;
Parallel.ForEach(projects, new ParallelOptions {
CancellationToken = CancelToken
}, (p) =>
{
if (p.ProjectName == null)
{
return;
}
String combinedPolicyNames = String.Empty;
if (Policies != null)
{
try
{
IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
(RestContext, CancelToken, p.ProjectId);
if (projectPolicyList != null)
{
Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
}
}
catch (Exception ex)
{
_log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
$"Policy statistics will be unavalable.", ex);
}
}
var cfDict = new SortedDictionary <String, String>();
p.CustomFields.ForEach((cf) => cfDict.Add(cf.FieldName, cf.FieldValue));
// Load the projects
String teamName = Teams.ContainsKey(p.TeamId) ? Teams[p.TeamId] : String.Empty;
if (String.Empty == teamName)
{
_log.ErrorFormat("Unable to find a team name for team id [{0}] when adding project {1}:{2}", p.TeamId,
p.ProjectId, p.ProjectName);
return;
}
String presetName = Presets.ContainsKey(p.PresetId) ? Presets[p.PresetId] : String.Empty;
if (String.Empty == presetName)
{
_log.ErrorFormat("Unable to find a preset name for preset id [{0}] " +
"when adding project {1}:{2}; project may be assigned an invalid preset.", p.PresetId,
p.ProjectId, p.ProjectName);
return;
}
if (!Filter.Matches(teamName, p.ProjectName))
{
if (_log.IsDebugEnabled)
{
_log.Debug($"FILTERED: Team: [{teamName}] Project: [{p.ProjectName}]");
}
return;
}
if (!_loadedProjects.TryAdd(p.ProjectId,
new ProjectDescriptor()
{
ProjectId = p.ProjectId,
ProjectName = p.ProjectName,
TeamName = teamName,
TeamId = p.TeamId,
PresetId = p.PresetId,
PresetName = presetName,
Policies = combinedPolicyNames,
CustomFields = cfDict
}
))
{
_log.WarnFormat("Rejected changed when adding new project with duplicate id {0}: New name: [{1}] current name: [{2}].",
p.ProjectId, p.ProjectName, _loadedProjects[p.ProjectId].ProjectName);
return;
}
});
// _loadedProjects has a collection of projects loaded from the SAST system
_state.ConfirmProjects(new List <ProjectDescriptor>(_loadedProjects.Values));
_log.Info($"{_state.ProjectCount} projects are targets to check for new scans. Since last scan: {_state.DeletedProjects}"
+ $" projects removed, {_state.NewProjects} new projects.");
_log.Debug("Resolving scans.");
// Load the scans for each project
Parallel.ForEach(_state.Projects, new ParallelOptions {
CancellationToken = CancelToken
},
(p) =>
{
// SAST Scans
var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished, p.ProjectId);
foreach (var s in sastScans)
{
// Add to crawl state.
if (_log.IsTraceEnabled())
{
_log.Trace($"SAST scan record: {s}");
}
_state.AddScan(s.ProjectId, s.ScanType, ScanProductType.SAST, s.ScanId, s.FinishTime);
SastScanCache.TryAdd(s.ScanId, s);
}
// OSA scans
var osaScans = CxOsaScans.GetScans(RestContext, CancelToken, p.ProjectId);
foreach (var s in osaScans)
{
// Add to crawl state.
if (_log.IsTraceEnabled())
{
_log.Trace($"OSA scan record: {s}");
}
_state.AddScan(s.ProjectId, "Composition", ScanProductType.SCA, s.ScanId, s.FinishTime);
ScaScanCache.TryAdd(s.ScanId, s);
}
});
}