Exemple #1
0
        private async Task <ProjectPolicyIndex> PopulatePolicies()
        {
            // Policies may not have data if M&O is not installed.
            try
            {
                return(await Task.Run(
                           () =>
                {
                    _log.Debug("Retrieving policies, if available.");

                    return new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(RestContext, CancelToken));
                }, CancelToken));
            }
            catch (Exception ex)
            {
                String msg = "Policy data is not available. M&O was unreachable.  You can omit the M&O URL in the configuration if M&O is not installed.";

                if (_log.IsDebugEnabled)
                {
                    _log.Debug(msg, ex);
                }
                else
                {
                    _log.Warn(msg);
                }
            }


            return(null);
        }
Exemple #2
0
        private Transformer(CxRestContext ctx, CancellationToken token,
                            String previousStatePath)
        {
            RestContext = ctx;
            CancelToken = token;

            Policies = null;

            _log.Debug("Retrieving policies, if available.");

            // Policies may not have data if M&O is not installed.
            try
            {
                Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
            }
            catch (Exception ex)
            {
                _log.Warn("Policy data is not available.", ex);
            }


            // Populate the data resolver with teams and presets
            DataResolver dr = new DataResolver();

            _log.Debug("Retrieving presets.");

            var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);

            foreach (var preset in presetEnum)
            {
                dr.addPreset(preset.PresetId, preset.PresetName);
            }

            _log.Debug("Retrieving teams.");

            var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);

            foreach (var team in teamEnum)
            {
                dr.addTeam(team.TeamId, team.TeamName);
            }

            _log.Debug("Resolving projects.");

            // Now populate the project resolver with the projects
            ProjectResolver pr = dr.Resolve(previousStatePath);

            var projects = CxProjects.GetProjects(RestContext, CancelToken);

            foreach (var p in projects)
            {
                String combinedPolicyNames = String.Empty;

                if (Policies != null)
                {
                    try
                    {
                        IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
                                                                  (ctx, token, p.ProjectId);

                        Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
                        combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
                    }
                    catch (Exception ex)
                    {
                        _log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
                                  $"Policy statistics will be unavalable.", ex);
                    }
                }

                var cfDict = new SortedDictionary <String, String>();
                p.CustomFields.ForEach((cf) => cfDict.Add(cf.FieldName, cf.FieldValue));

                pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames, cfDict);
            }

            // Resolve projects to get the scan resolver.
            ScanResolver sr = pr.Resolve(_mapActions);

            try
            {
                _log.Debug("Retrieving SAST scans.");

                var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);

                foreach (var sastScan in sastScans)
                {
                    _log.Debug($"SAST scan record: {sastScan}");

                    sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
                               sastScan.ScanId, sastScan.FinishTime);

                    SastScanCache.Add(sastScan.ScanId, sastScan);
                }


                _log.Debug("Retrieving OSA scans.");

                foreach (var p in projects)
                {
                    var scaScans = CxOsaScans.GetScans(ctx, token, p.ProjectId);
                    foreach (var scaScan in scaScans)
                    {
                        _log.Debug($"OSA scan record: {scaScan}");

                        sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
                                   scaScan.FinishTime);
                        ScaScanCache.Add(scaScan.ScanId, scaScan);
                    }
                }

                ScanDescriptors = sr.Resolve(CheckTime);
            }
            catch (AggregateException aex)
            {
                _log.Error($"Multiple errors caught resolving scans.");

                int count = 0;
                aex.Handle(
                    (ex) =>
                {
                    _log.Error($"Exception {++count}: ", ex);
                    return(true);
                });
            }
            catch (Exception ex)
            {
                _log.Error($"Error resolving scans, server may be unavailable.", ex);
            }
        }
Exemple #3
0
        private Transformer(CxRestContext ctx, CancellationToken token,
                            String previousStatePath)
        {
            RestContext = ctx;
            CancelToken = token;

            Policies = null;

            // Policies may not have data if M&O is not installed.
            try
            {
                Policies = new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(ctx, token));
            }
            catch (Exception ex)
            {
                _log.Warn("Policy data is not available.", ex);
            }


            // Populate the data resolver with teams and presets
            DataResolver dr = new DataResolver();

            var presetEnum = CxPresets.GetPresets(RestContext, CancelToken);

            foreach (var preset in presetEnum)
            {
                dr.addPreset(preset.PresetId, preset.PresetName);
            }

            var teamEnum = CxTeams.GetTeams(RestContext, CancelToken);

            foreach (var team in teamEnum)
            {
                dr.addTeam(team.TeamId, team.TeamName);
            }

            // Now populate the project resolver with the projects
            ProjectResolver pr = dr.Resolve(previousStatePath);

            var projects = CxProjects.GetProjects(RestContext, CancelToken);

            foreach (var p in projects)
            {
                String combinedPolicyNames = String.Empty;

                if (Policies != null)
                {
                    try
                    {
                        IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
                                                                  (ctx, token, p.ProjectId);

                        Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
                        combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
                    }
                    catch (Exception ex)
                    {
                        _log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
                                  $"Policy statistics will be unavalable.", ex);
                    }
                }

                pr.AddProject(p.TeamId, p.PresetId, p.ProjectId, p.ProjectName, combinedPolicyNames);
            }

            // Resolve projects to get the scan resolver.
            ScanResolver sr = pr.Resolve(_mapActions);

            try
            {
                var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished);
                foreach (var sastScan in sastScans)
                {
                    sr.AddScan(sastScan.ProjectId, sastScan.ScanType, SAST_PRODUCT_STRING,
                               sastScan.ScanId, sastScan.FinishTime);

                    SastScanCache.Add(sastScan.ScanId, sastScan);
                }


                foreach (var p in projects)
                {
                    var scaScans = CxScaScans.GetScans(ctx, token, p.ProjectId);
                    foreach (var scaScan in scaScans)
                    {
                        sr.AddScan(scaScan.ProjectId, "Composition", SCA_PRODUCT_STRING, scaScan.ScanId,
                                   scaScan.FinishTime);
                        ScaScanCache.Add(scaScan.ScanId, scaScan);
                    }
                }

                ScanDescriptors = sr.Resolve(CheckTime);
            }
            catch (Exception ex)
            {
                _log.Error($"Error resolving scans, server may be unavailable.", ex);
            }
        }
Exemple #4
0
        private async Task ResolveScans()
        {
            var policyTask = PopulatePolicies();

            var presetsTask = PopulatePresets();

            var teamsTask = PopulateTeams();

            _log.Debug("Resolving projects.");

            var projects = await Task.Run(() => CxProjects.GetProjects(RestContext, CancelToken), CancelToken);

            Policies = await policyTask;
            Teams    = await teamsTask;
            Presets  = await presetsTask;


            Parallel.ForEach(projects, new ParallelOptions {
                CancellationToken = CancelToken
            }, (p) =>
            {
                if (p.ProjectName == null)
                {
                    return;
                }

                String combinedPolicyNames = String.Empty;

                if (Policies != null)
                {
                    try
                    {
                        IEnumerable <int> projectPolicyList = CxMnoPolicies.GetPolicyIdsForProject
                                                                  (RestContext, CancelToken, p.ProjectId);

                        if (projectPolicyList != null)
                        {
                            Policies.CorrelateProjectToPolicies(p.ProjectId, projectPolicyList);
                            combinedPolicyNames = GetFlatPolicyNames(Policies, projectPolicyList);
                        }
                    }
                    catch (Exception ex)
                    {
                        _log.Warn($"Unable to correlate policies to project {p.ProjectId}: {p.ProjectName}. " +
                                  $"Policy statistics will be unavalable.", ex);
                    }
                }

                var cfDict = new SortedDictionary <String, String>();
                p.CustomFields.ForEach((cf) => cfDict.Add(cf.FieldName, cf.FieldValue));


                // Load the projects
                String teamName = Teams.ContainsKey(p.TeamId) ? Teams[p.TeamId] : String.Empty;
                if (String.Empty == teamName)
                {
                    _log.ErrorFormat("Unable to find a team name for team id [{0}] when adding project {1}:{2}", p.TeamId,
                                     p.ProjectId, p.ProjectName);

                    return;
                }

                String presetName = Presets.ContainsKey(p.PresetId) ? Presets[p.PresetId] : String.Empty;
                if (String.Empty == presetName)
                {
                    _log.ErrorFormat("Unable to find a preset name for preset id [{0}] " +
                                     "when adding project {1}:{2}; project may be assigned an invalid preset.", p.PresetId,
                                     p.ProjectId, p.ProjectName);

                    return;
                }


                if (!Filter.Matches(teamName, p.ProjectName))
                {
                    if (_log.IsDebugEnabled)
                    {
                        _log.Debug($"FILTERED: Team: [{teamName}] Project: [{p.ProjectName}]");
                    }

                    return;
                }

                if (!_loadedProjects.TryAdd(p.ProjectId,
                                            new ProjectDescriptor()
                {
                    ProjectId = p.ProjectId,
                    ProjectName = p.ProjectName,
                    TeamName = teamName,
                    TeamId = p.TeamId,
                    PresetId = p.PresetId,
                    PresetName = presetName,
                    Policies = combinedPolicyNames,
                    CustomFields = cfDict
                }
                                            ))
                {
                    _log.WarnFormat("Rejected changed when adding new project with duplicate id {0}: New name: [{1}] current name: [{2}].",
                                    p.ProjectId, p.ProjectName, _loadedProjects[p.ProjectId].ProjectName);

                    return;
                }
            });


            // _loadedProjects has a collection of projects loaded from the SAST system
            _state.ConfirmProjects(new List <ProjectDescriptor>(_loadedProjects.Values));

            _log.Info($"{_state.ProjectCount} projects are targets to check for new scans. Since last scan: {_state.DeletedProjects}"
                      + $" projects removed, {_state.NewProjects} new projects.");


            _log.Debug("Resolving scans.");

            // Load the scans for each project
            Parallel.ForEach(_state.Projects, new ParallelOptions {
                CancellationToken = CancelToken
            },
                             (p) =>
            {
                // SAST Scans
                var sastScans = CxSastScans.GetScans(RestContext, CancelToken, CxSastScans.ScanStatus.Finished, p.ProjectId);

                foreach (var s in sastScans)
                {
                    // Add to crawl state.
                    if (_log.IsTraceEnabled())
                    {
                        _log.Trace($"SAST scan record: {s}");
                    }
                    _state.AddScan(s.ProjectId, s.ScanType, ScanProductType.SAST, s.ScanId, s.FinishTime);
                    SastScanCache.TryAdd(s.ScanId, s);
                }

                // OSA scans
                var osaScans = CxOsaScans.GetScans(RestContext, CancelToken, p.ProjectId);
                foreach (var s in osaScans)
                {
                    // Add to crawl state.
                    if (_log.IsTraceEnabled())
                    {
                        _log.Trace($"OSA scan record: {s}");
                    }
                    _state.AddScan(s.ProjectId, "Composition", ScanProductType.SCA, s.ScanId, s.FinishTime);
                    ScaScanCache.TryAdd(s.ScanId, s);
                }
            });
        }