/// <summary>
/// Executes logic to determine which projects are candidates for querying for scan results
/// since the last run.
/// </summary>
/// <remarks>
/// Each run will have a list of projects currently in the system. There may be new projects
/// since the last run that have never been checked for scans. Some projects may have been
/// deleted since the last run and need not be checked.
/// </remarks>
/// <returns>An instance of <see cref="ScanResolver"/> that will assist with
/// picking scans that need to be used in the transformation.</returns>
public ScanResolver Resolve(Dictionary <String, Action <ScanDescriptor, Transformer> > productAction)
{
if (_res == null && !_disallowAdd)
{
_disallowAdd = true;
_targets = new Dictionary <int, ProjectDescriptorExt>();
int newProjects = 0;
foreach (int pid in _newProjects.Keys)
{
// Look for the project in the previous projects. If it exists, use that record to make
// the project a target for scan pulling.
if (_previousTargets.ContainsKey(pid))
{
// Make a new descriptor so that if there is a new preset/name/team/etc
// it gets updated based on data pulled from the web service.
_targets.Add(pid, new ProjectDescriptorExt(_newProjects[pid],
_previousTargets[pid]));
_previousTargets.Remove(pid);
}
else
{
// If the project was never seen before, it is a new project and a target for pulling scans.
_targets.Add(pid, ProjectDescriptorExt.newDetail(_newProjects[pid]));
newProjects++;
}
}
// scanTargets now contains the projects that were added via addProject. If they were checked before,
// the last check date is in the record. If not, the last check date is the epoch.
// _previousProjects now has projects that have likely been deleted, so throw some INFO in the log
// that these aren't going to be scanned.
_log.InfoFormat("{0} projects are targets for check for new scans. Since last scan: {1} projects removed, {2} new projects.",
_targets.Keys.Count, _previousTargets.Keys.Count, newProjects);
foreach (int pid in _previousTargets.Keys)
{
_log.InfoFormat("No longer tracking state for project {0}:[{1}] Team [{2}]",
pid, _previousTargets[pid].ProjectName, _previousTargets[pid].TeamName);
}
_res = new ScanResolver(this, productAction);
}
return(_res);
}
private Transformer(CxRestContext ctx, CancellationToken token,
String previousStatePath)
{
RestContext = ctx;
CancelToken = token;
Policies = null;
_log.Debug("Retrieving policies, if available.");
// Policies may not have data if M&O is not installed.
try
{
Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
}
catch (Exception ex)
{
_log.Warn("Policy data is not available.", ex);
}
// Populate the data resolver with teams and presets
DataResolver dr = new DataResolver();
_log.Debug("Retrieving presets.");
var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);
foreach (var preset in presetEnum)
{
dr.addPreset(preset.PresetId, preset.PresetName);
}
_log.Debug("Retrieving teams.");
var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);
foreach (var team in teamEnum)
{
dr.addTeam(team.TeamId, team.TeamName);
}
_log.Debug("Resolving projects.");
// Now populate the project resolver with the projects
ProjectResolver pr = dr.Resolve(previousStatePath);
var projects = CxProjects.GetProjects(RestContext, CancelToken);
foreach (var p in projects)
{
String combinedPolicyNames = String.Empty;
if (Policies != null)
{
try
{
IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
(ctx, token, p.ProjectId);
Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
}
catch (Exception ex)
{
_log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
$"Policy statistics will be unavalable.", ex);
}
}
var cfDict = new SortedDictionary <String, String>();
p.CustomFields.ForEach((cf) => cfDict.Add(cf.FieldName, cf.FieldValue));
pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames, cfDict);
}
// Resolve projects to get the scan resolver.
ScanResolver sr = pr.Resolve(_mapActions);
try
{
_log.Debug("Retrieving SAST scans.");
var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);
foreach (var sastScan in sastScans)
{
_log.Debug($"SAST scan record: {sastScan}");
sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
sastScan.ScanId, sastScan.FinishTime);
SastScanCache.Add(sastScan.ScanId, sastScan);
}
_log.Debug("Retrieving OSA scans.");
foreach (var p in projects)
{
var scaScans = CxOsaScans.GetScans(ctx, token, p.ProjectId);
foreach (var scaScan in scaScans)
{
_log.Debug($"OSA scan record: {scaScan}");
sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
scaScan.FinishTime);
ScaScanCache.Add(scaScan.ScanId, scaScan);
}
}
ScanDescriptors = sr.Resolve(CheckTime);
}
catch (AggregateException aex)
{
_log.Error($"Multiple errors caught resolving scans.");
int count = 0;
aex.Handle(
(ex) =>
{
_log.Error($"Exception {++count}: ", ex);
return(true);
});
}
catch (Exception ex)
{
_log.Error($"Error resolving scans, server may be unavailable.", ex);
}
}
private Transformer(CxRestContext ctx, CancellationToken token,
String previousStatePath)
{
RestContext = ctx;
CancelToken = token;
Policies = null;
// Policies may not have data if M&O is not installed.
try
{
Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
}
catch (Exception ex)
{
_log.Warn("Policy data is not available.", ex);
}
// Populate the data resolver with teams and presets
DataResolver dr = new DataResolver();
var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);
foreach (var preset in presetEnum)
{
dr.addPreset(preset.PresetId, preset.PresetName);
}
var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);
foreach (var team in teamEnum)
{
dr.addTeam(team.TeamId, team.TeamName);
}
// Now populate the project resolver with the projects
ProjectResolver pr = dr.Resolve(previousStatePath);
var projects = CxProjects.GetProjects(RestContext, CancelToken);
foreach (var p in projects)
{
String combinedPolicyNames = String.Empty;
if (Policies != null)
{
try
{
IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
(ctx, token, p.ProjectId);
Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
}
catch (Exception ex)
{
_log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
$"Policy statistics will be unavalable.", ex);
}
}
pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames);
}
// Resolve projects to get the scan resolver.
ScanResolver sr = pr.Resolve(_mapActions);
try
{
var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);
foreach (var sastScan in sastScans)
{
sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
sastScan.ScanId, sastScan.FinishTime);
SastScanCache.Add(sastScan.ScanId, sastScan);
}
foreach (var p in projects)
{
var scaScans = CxScaScans.GetScans(ctx, token, p.ProjectId);
foreach (var scaScan in scaScans)
{
sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
scaScan.FinishTime);
ScaScanCache.Add(scaScan.ScanId, scaScan);
}
}
ScanDescriptors = sr.Resolve(CheckTime);
}
catch (Exception ex)
{
_log.Error($"Error resolving scans, server may be unavailable.", ex);
}
}