public virtual IReadOnlyList <SarifLog> SplitLogFile(SarifLog sarifLog) { IList <SarifLog> logsToProcess; using (Logger.BeginScopeContext(nameof(SplitLogFile))) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); sarifLog.SetProperty("guid", Guid.NewGuid()); this.FilingResult = FilingResult.None; this.FiledWorkItems = new List <WorkItemModel>(); sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { Logger.LogDebug("Removing optional data."); var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { Logger.LogDebug("Inserting optional data."); var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } using (Logger.BeginScopeContext("Splitting visitor")) { SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; Logger.LogInformation($"Splitting strategy - {splittingStrategy}"); if (splittingStrategy == SplittingStrategy.None) { return(new[] { sarifLog }); } PartitionFunction <string> partitionFunction = null; Stopwatch splittingStopwatch = Stopwatch.StartNew(); switch (splittingStrategy) { case SplittingStrategy.PerRun: { partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null; break; } case SplittingStrategy.PerResult: { partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null; break; } case SplittingStrategy.PerRunPerOrgPerEntityTypePerPartialFingerprint: { partitionFunction = (result) => result.ShouldBeFiled() ? result.GetFingerprintSplittingStrategyId() : null; break; } case SplittingStrategy.PerRunPerOrgPerEntityTypePerRepositoryPerPartialFingerprint: { partitionFunction = (result) => result.ShouldBeFiled() ? result.GetPerRepositoryFingerprintSplittingStrategyId() : null; break; } default: { throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } } Logger.LogDebug("Begin splitting logs"); var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false); partitioningVisitor.VisitSarifLog(sarifLog); Logger.LogDebug("Begin retrieving split logs"); logsToProcess = new List <SarifLog>(partitioningVisitor.GetPartitionLogs().Values); Logger.LogDebug("End retrieving split logs"); var logsToProcessMetrics = new Dictionary <string, object> { { "splittingStrategy", splittingStrategy }, { "logsToProcessCount", logsToProcess.Count }, { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds }, }; this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics); splittingStopwatch.Stop(); } } if (logsToProcess != null && !this.FilingContext.ShouldFileUnchanged) { // Remove any logs that do not contain at least one result with a New or None baselinestate. logsToProcess = logsToProcess.Where(log => log?.Runs?.Any(run => run.Results?.Any(result => result.BaselineState == BaselineState.New || result.BaselineState == BaselineState.None) == true) == true).ToList(); } return(logsToProcess.ToArray()); }
public virtual void FileWorkItems(SarifLog sarifLog) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; if (splittingStrategy == SplittingStrategy.None) { FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient); return; } for (int runIndex = 0; runIndex < sarifLog.Runs?.Count; ++runIndex) { if (sarifLog.Runs[runIndex]?.Results?.Count > 0) { IList <SarifLog> logsToProcess = new List <SarifLog>(new SarifLog[] { sarifLog }); if (splittingStrategy != SplittingStrategy.PerRun) { SplittingVisitor visitor; switch (splittingStrategy) { case SplittingStrategy.PerRunPerRule: visitor = new PerRunPerRuleSplittingVisitor(); break; case SplittingStrategy.PerRunPerTarget: visitor = new PerRunPerTargetSplittingVisitor(); break; case SplittingStrategy.PerRunPerTargetPerRule: visitor = new PerRunPerTargetPerRuleSplittingVisitor(); break; // TODO: Implement PerResult and PerRun splittings strategies // // https://github.com/microsoft/sarif-sdk/issues/1763 // https://github.com/microsoft/sarif-sdk/issues/1762 // case SplittingStrategy.PerResult: case SplittingStrategy.PerRun: default: throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } visitor.VisitRun(sarifLog.Runs[runIndex]); logsToProcess = visitor.SplitSarifLogs; } for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++) { SarifLog splitLog = logsToProcess[splitFileIndex]; FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient); } } } }
public virtual SarifLog FileWorkItems(SarifLog sarifLog) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); sarifLog.SetProperty("guid", Guid.NewGuid()); using (Logger.BeginScope(nameof(FileWorkItems))) { this.FilingResult = FilingResult.None; this.FiledWorkItems = new List<WorkItemModel>(); sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; if (splittingStrategy == SplittingStrategy.None) { FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient); return sarifLog; } IList<SarifLog> logsToProcess; PartitionFunction<string> partitionFunction = null; Stopwatch splittingStopwatch = Stopwatch.StartNew(); switch (splittingStrategy) { case SplittingStrategy.PerRun: { partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null; break; } case SplittingStrategy.PerResult: { partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null; break; } default: { throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } } var partitioningVisitor = new PartitioningVisitor<string>(partitionFunction, deepClone: false); partitioningVisitor.VisitSarifLog(sarifLog); logsToProcess = new List<SarifLog>(partitioningVisitor.GetPartitionLogs().Values); var logsToProcessMetrics = new Dictionary<string, object> { { "splittingStrategy", splittingStrategy }, { "logsToProcessCount", logsToProcess.Count }, { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds }, }; this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics); splittingStopwatch.Stop(); for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++) { SarifLog splitLog = logsToProcess[splitFileIndex]; FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient); } } return sarifLog; }