예제 #1
0
        // Extract selected results from original SarifLog.
        internal SarifLog GetPartitionedLog(IEnumerable <SarifErrorListItem> listItems)
        {
            int    runIndex = -1;
            string guid     = Guid.NewGuid().ToString();

            foreach (SarifErrorListItem item in listItems)
            {
                if (item.SarifResult != null)
                {
                    item.SarifResult.Guid = guid;
                    if (runIndex == -1)
                    {
                        runIndex = item.RunIndex;
                    }
                }
            }

            if (runIndex == -1 || !this.RunIndexToRunDataCache.TryGetValue(runIndex, out RunDataCache dataCache) || dataCache.SarifLog == null)
            {
                return(null);
            }

            // parition results in log
            PartitionFunction <string> partitionFunction = (result) => result.Guid ?? null;
            var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);

            partitioningVisitor.VisitSarifLog(dataCache.SarifLog);
            Dictionary <string, SarifLog> partitions = partitioningVisitor.GetPartitionLogs();

            return(partitions[guid]);
        }
예제 #2
0
        public static IEnumerable <SarifLog> Split(this SarifLog sarifLog, SplittingStrategy splittingStrategy)
        {
            PartitionFunction <string> partitionFunction = null;

            switch (splittingStrategy)
            {
            case SplittingStrategy.PerResult:
            {
                partitionFunction = (result) => result.RuleId;
                break;
            }

            case SplittingStrategy.PerRun:
            {
                foreach (Run run in sarifLog.Runs)
                {
                    run.SetRunOnResults();
                }

                partitionFunction = (result) => result.Run.GetHashCode().ToString();
                break;
            }

            default:
            {
                throw new NotImplementedException($"SplittingStrategy: {splittingStrategy}");
            }
            }

            var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);

            partitioningVisitor.VisitSarifLog(sarifLog);

            return(partitioningVisitor.GetPartitionLogs().Values);
        }
예제 #3
0
        /// <summary>
        /// Partition the specified SARIF log into a set of "partitioned logs" according to
        /// the specified partitioning function. Each partitioned log contains only those
        /// elements of run-level collections such as Run.Artifacts that are relevant to the
        /// subset of results in that log.
        /// </summary>
        /// <typeparam name="T">
        /// The type of the object returned by the partition function. It must be a reference
        /// type so that null is a valid value. It must override bool Equals(T other) so that
        /// two Ts can compare equal even if they are not reference equal.
        /// </typeparam>
        /// <param name="log">
        /// The SARIF log to be partitioned.
        /// </param>
        /// <param name="partitionFunction">
        /// A function that returns a value specifying which partitioned log a specified result
        /// belongs in, or null if the result should be discarded (not placed in any of the
        /// partitioned logs).
        /// </param>
        /// <param name="deepClone">
        /// A value that specifies how the partitioned logs are constructed from the original log.
        /// If <c>true</c>, each partitioned log is constructed from a deep clone of the
        /// original log; if <c>false</c>, each partitioned log is constructed from a shallow
        /// copy of the original log. Deep cloning ensures that the original and partitioned logs
        /// do not share any objects, so they can be modified safely, but at a cost of increased
        /// partitioning time and  working set. Shallow copying reduces partitioning time and
        /// working set, but it is not safe to modify any of the resulting logs because this class
        /// makes no guarantee about which objects are shared.
        /// </param>
        /// <returns>
        /// A dictionary whose keys are the values returned by <paramref name="partitionFunction"/>
        /// for the results in <paramref name="log"/> and whose values are the SARIF logs
        /// containing the results for which the partition function returns those values.
        /// </returns>
        public static IDictionary <T, SarifLog> Partition <T>(
            SarifLog log,
            PartitionFunction <T> partitionFunction,
            bool deepClone)
            where T : class, IEquatable <T>
        {
            var visitor = new PartitioningVisitor <T>(partitionFunction, deepClone);

            visitor.VisitSarifLog(log);

            return(visitor.GetPartitionLogs());
        }
예제 #4
0
        public virtual IReadOnlyList <SarifLog> SplitLogFile(SarifLog sarifLog)
        {
            IList <SarifLog> logsToProcess;

            using (Logger.BeginScopeContext(nameof(SplitLogFile)))
            {
                sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
                sarifLog.SetProperty("guid", Guid.NewGuid());

                this.FilingResult   = FilingResult.None;
                this.FiledWorkItems = new List <WorkItemModel>();

                sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));

                Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
                this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();

                OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
                if (optionallyEmittedData != OptionallyEmittedData.None)
                {
                    Logger.LogDebug("Removing optional data.");
                    var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
                    dataRemovingVisitor.Visit(sarifLog);
                }

                optionallyEmittedData = this.FilingContext.DataToInsert;
                if (optionallyEmittedData != OptionallyEmittedData.None)
                {
                    Logger.LogDebug("Inserting optional data.");
                    var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
                    dataInsertingVisitor.Visit(sarifLog);
                }

                using (Logger.BeginScopeContext("Splitting visitor"))
                {
                    SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;

                    Logger.LogInformation($"Splitting strategy - {splittingStrategy}");

                    if (splittingStrategy == SplittingStrategy.None)
                    {
                        return(new[] { sarifLog });
                    }

                    PartitionFunction <string> partitionFunction = null;

                    Stopwatch splittingStopwatch = Stopwatch.StartNew();

                    switch (splittingStrategy)
                    {
                    case SplittingStrategy.PerRun:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
                        break;
                    }

                    case SplittingStrategy.PerResult:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
                        break;
                    }

                    case SplittingStrategy.PerRunPerOrgPerEntityTypePerPartialFingerprint:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? result.GetFingerprintSplittingStrategyId() : null;
                        break;
                    }

                    case SplittingStrategy.PerRunPerOrgPerEntityTypePerRepositoryPerPartialFingerprint:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? result.GetPerRepositoryFingerprintSplittingStrategyId() : null;
                        break;
                    }

                    default:
                    {
                        throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
                    }
                    }

                    Logger.LogDebug("Begin splitting logs");
                    var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);
                    partitioningVisitor.VisitSarifLog(sarifLog);

                    Logger.LogDebug("Begin retrieving split logs");
                    logsToProcess = new List <SarifLog>(partitioningVisitor.GetPartitionLogs().Values);

                    Logger.LogDebug("End retrieving split logs");

                    var logsToProcessMetrics = new Dictionary <string, object>
                    {
                        { "splittingStrategy", splittingStrategy },
                        { "logsToProcessCount", logsToProcess.Count },
                        { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
                    };

                    this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
                    splittingStopwatch.Stop();
                }
            }

            if (logsToProcess != null && !this.FilingContext.ShouldFileUnchanged)
            {
                // Remove any logs that do not contain at least one result with a New or None baselinestate.
                logsToProcess = logsToProcess.Where(log => log?.Runs?.Any(run => run.Results?.Any(result => result.BaselineState == BaselineState.New || result.BaselineState == BaselineState.None) == true) == true).ToList();
            }

            return(logsToProcess.ToArray());
        }
예제 #5
0
        public virtual SarifLog FileWorkItems(SarifLog sarifLog)
        {
            sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));

            sarifLog.SetProperty("guid", Guid.NewGuid());

            using (Logger.BeginScope(nameof(FileWorkItems)))
            {
                this.FilingResult = FilingResult.None;
                this.FiledWorkItems = new List<WorkItemModel>();

                sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));

                Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
                this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();

                OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
                if (optionallyEmittedData != OptionallyEmittedData.None)
                {
                    var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
                    dataRemovingVisitor.Visit(sarifLog);
                }

                optionallyEmittedData = this.FilingContext.DataToInsert;
                if (optionallyEmittedData != OptionallyEmittedData.None)
                {
                    var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
                    dataInsertingVisitor.Visit(sarifLog);
                }

                SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;

                if (splittingStrategy == SplittingStrategy.None)
                {
                    FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient);
                    return sarifLog;
                }

                IList<SarifLog> logsToProcess;

                PartitionFunction<string> partitionFunction = null;

                Stopwatch splittingStopwatch = Stopwatch.StartNew();
                
                switch (splittingStrategy)
                {
                    case SplittingStrategy.PerRun:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
                        break;
                    }
                    case SplittingStrategy.PerResult:
                    {
                        partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
                        break;
                    }
                    default:
                    {
                        throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
                    }
                }

                var partitioningVisitor = new PartitioningVisitor<string>(partitionFunction, deepClone: false);
                partitioningVisitor.VisitSarifLog(sarifLog);

                logsToProcess = new List<SarifLog>(partitioningVisitor.GetPartitionLogs().Values);

                var logsToProcessMetrics = new Dictionary<string, object>
                {
                    { "splittingStrategy", splittingStrategy },
                    { "logsToProcessCount", logsToProcess.Count },
                    { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
                };

                this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
                splittingStopwatch.Stop();

                for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++)
                {
                    SarifLog splitLog = logsToProcess[splitFileIndex];
                    FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient);
                }
            }

            return sarifLog;
        }