public int Run(RewriteOptions rewriteOptions) { try { Console.WriteLine($"Rewriting '{rewriteOptions.InputFilePath}' => '{rewriteOptions.OutputFilePath}'..."); Stopwatch w = Stopwatch.StartNew(); bool valid = ValidateOptions(rewriteOptions); if (!valid) { return(FAILURE); } SarifLog actualLog = ReadSarifFile <SarifLog>(_fileSystem, rewriteOptions.InputFilePath); OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags(); IDictionary <string, ArtifactLocation> originalUriBaseIds = rewriteOptions.ConstructUriBaseIdsDictionary(); SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(actualLog); string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions); WriteSarifFile(_fileSystem, reformattedLog, fileName, rewriteOptions.Formatting); w.Stop(); Console.WriteLine($"Rewrite completed in {w.Elapsed}."); } catch (Exception ex) { Console.WriteLine(ex); return(FAILURE); } return(SUCCESS); }
public int Run(RewriteOptions rewriteOptions) { try { rewriteOptions = ValidateOptions(rewriteOptions); SarifLog actualLog = FileHelpers.ReadSarifFile <SarifLog>(_fileSystem, rewriteOptions.InputFilePath); OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags(); IDictionary <string, ArtifactLocation> originalUriBaseIds = rewriteOptions.ConstructUriBaseIdsDictionary(); SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(actualLog); string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions); var formatting = rewriteOptions.PrettyPrint ? Formatting.Indented : Formatting.None; FileHelpers.WriteSarifFile(_fileSystem, reformattedLog, fileName, formatting); } catch (Exception ex) { Console.WriteLine(ex); return(1); } return(0); }
public static int Run(RewriteOptions rewriteOptions) { try { rewriteOptions = ValidateOptions(rewriteOptions); SarifLog actualLog = MultitoolFileHelpers.ReadSarifFile <SarifLog>(rewriteOptions.InputFilePath); OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags(); SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(actualLog); string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions); var formatting = rewriteOptions.PrettyPrint ? Formatting.Indented : Formatting.None; MultitoolFileHelpers.WriteSarifFile(reformattedLog, fileName, formatting); } catch (Exception ex) { Console.WriteLine(ex); return(1); } return(0); }
public int Run(RewriteOptions options) { try { Console.WriteLine($"Rewriting '{options.InputFilePath}' => '{options.OutputFilePath}'..."); Stopwatch w = Stopwatch.StartNew(); bool valid = ValidateOptions(options); if (!valid) { return(FAILURE); } string actualOutputPath = CommandUtilities.GetTransformedOutputFileName(options); SarifLog actualLog = null; string inputVersion = SniffVersion(options.InputFilePath); if (!inputVersion.Equals(SarifUtilities.StableSarifVersion)) { actualLog = TransformFileToVersionTwo(options.InputFilePath, inputVersion); } else { actualLog = ReadSarifFile <SarifLog>(_fileSystem, options.InputFilePath); } OptionallyEmittedData dataToInsert = options.DataToInsert.ToFlags(); OptionallyEmittedData dataToRemove = options.DataToRemove.ToFlags(); IDictionary <string, ArtifactLocation> originalUriBaseIds = options.ConstructUriBaseIdsDictionary(); SarifLog reformattedLog = new RemoveOptionalDataVisitor(dataToRemove).VisitSarifLog(actualLog); reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(reformattedLog); if (options.SarifOutputVersion == SarifVersion.OneZeroZero) { var visitor = new SarifCurrentToVersionOneVisitor(); visitor.VisitSarifLog(reformattedLog); WriteSarifFile(_fileSystem, visitor.SarifLogVersionOne, actualOutputPath, options.Minify, SarifContractResolverVersionOne.Instance); } else { WriteSarifFile(_fileSystem, reformattedLog, actualOutputPath, options.Minify); } w.Stop(); Console.WriteLine($"Rewrite completed in {w.Elapsed}."); } catch (Exception ex) { Console.WriteLine(ex); return(FAILURE); } return(SUCCESS); }
public SarifLogger( TextWriter textWriter, LogFilePersistenceOptions logFilePersistenceOptions = DefaultLogFilePersistenceOptions, OptionallyEmittedData dataToInsert = OptionallyEmittedData.None, OptionallyEmittedData dataToRemove = OptionallyEmittedData.None, Tool tool = null, Run run = null, IEnumerable <string> analysisTargets = null, IEnumerable <string> invocationTokensToRedact = null, IEnumerable <string> invocationPropertiesToLog = null, string defaultFileEncoding = null, bool closeWriterOnDispose = true, IEnumerable <FailureLevel> levels = null, IEnumerable <ResultKind> kinds = null) : this(textWriter, logFilePersistenceOptions, closeWriterOnDispose, levels, kinds) { if (dataToInsert.HasFlag(OptionallyEmittedData.Hashes)) { AnalysisTargetToHashDataMap = HashUtilities.MultithreadedComputeTargetFileHashes(analysisTargets); } _run = run ?? new Run(); if (dataToInsert.HasFlag(OptionallyEmittedData.RegionSnippets) || dataToInsert.HasFlag(OptionallyEmittedData.ContextRegionSnippets)) { _insertOptionalDataVisitor = new InsertOptionalDataVisitor(dataToInsert, _run); } EnhanceRun( analysisTargets, dataToInsert, dataToRemove, invocationTokensToRedact, invocationPropertiesToLog, defaultFileEncoding, AnalysisTargetToHashDataMap); tool = tool ?? Tool.CreateFromAssemblyData(); _run.Tool = tool; _dataToInsert = dataToInsert; _dataToRemove = dataToRemove; _issueLogJsonWriter.Initialize(_run); // Map existing Rules to ensure duplicates aren't created if (_run.Tool.Driver?.Rules != null) { for (int i = 0; i < _run.Tool.Driver.Rules.Count; ++i) { RuleToIndexMap[_run.Tool.Driver.Rules[i]] = i; } } _persistArtifacts = (_dataToInsert & OptionallyEmittedData.Hashes) != 0 || (_dataToInsert & OptionallyEmittedData.TextFiles) != 0 || (_dataToInsert & OptionallyEmittedData.BinaryFiles) != 0; }
public int Run(MergeOptions mergeOptions) { try { string outputDirectory = mergeOptions.OutputFolderPath ?? Environment.CurrentDirectory; string outputFilePath = Path.Combine(outputDirectory, GetOutputFileName(mergeOptions)); if (!DriverUtilities.ReportWhetherOutputFileCanBeCreated(outputFilePath, mergeOptions.Force, _fileSystem)) { return(FAILURE); } HashSet <string> sarifFiles = CreateTargetsSet(mergeOptions.TargetFileSpecifiers, mergeOptions.Recurse, _fileSystem); IEnumerable <SarifLog> allRuns = ParseFiles(sarifFiles); // Build one SarifLog with all the Runs. SarifLog combinedLog = allRuns.Merge(); // If there were no input files, the Merge operation set combinedLog.Runs to null. Although // null is valid in certain error cases, it is not valid here. Here, the correct value is // an empty list. See the SARIF spec, §3.13.4, "runs property". combinedLog.Runs = combinedLog.Runs ?? new List <Run>(); combinedLog.Version = SarifVersion.Current; combinedLog.SchemaUri = combinedLog.Version.ConvertToSchemaUri(); OptionallyEmittedData dataToInsert = mergeOptions.DataToInsert.ToFlags(); if (dataToInsert != OptionallyEmittedData.None) { combinedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(combinedLog); } // Write output to file. Formatting formatting = mergeOptions.PrettyPrint ? Formatting.Indented : Formatting.None; _fileSystem.CreateDirectory(outputDirectory); WriteSarifFile(_fileSystem, combinedLog, outputFilePath, formatting); } catch (Exception ex) { Console.WriteLine(ex); return(FAILURE); } return(SUCCESS); }
private static void EnrichSarifLog(string actualFilePath) { JsonSerializerSettings settings = new JsonSerializerSettings() { Formatting = Formatting.Indented, }; string logText = File.ReadAllText(actualFilePath); SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(logText, settings); OptionallyEmittedData dataToInsert = OptionallyEmittedData.Hashes | OptionallyEmittedData.RegionSnippets | OptionallyEmittedData.ContextRegionSnippets | OptionallyEmittedData.ComprehensiveRegionProperties; SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(actualLog); File.WriteAllText(actualFilePath, JsonConvert.SerializeObject(reformattedLog, settings)); }
public int Run(MergeOptions mergeOptions) { try { var sarifFiles = CreateTargetsSet(mergeOptions.TargetFileSpecifiers, mergeOptions.Recurse, _fileSystem); var allRuns = ParseFiles(sarifFiles); // Build one SarifLog with all the Runs. SarifLog combinedLog = allRuns.Merge(); combinedLog.Version = SarifVersion.Current; combinedLog.SchemaUri = combinedLog.Version.ConvertToSchemaUri(); OptionallyEmittedData dataToInsert = mergeOptions.DataToInsert.ToFlags(); if (dataToInsert != OptionallyEmittedData.None) { combinedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(combinedLog); } string outputDirectory = mergeOptions.OutputFolderPath ?? Environment.CurrentDirectory; // Write output to file. string outputName = Path.Combine(outputDirectory, GetOutputFileName(mergeOptions)); var formatting = mergeOptions.PrettyPrint ? Formatting.Indented : Formatting.None; Directory.CreateDirectory(outputDirectory); WriteSarifFile(_fileSystem, combinedLog, outputName, formatting); } catch (Exception ex) { Console.WriteLine(ex); return(1); } return(0); }
public virtual void FileWorkItems(SarifLog sarifLog) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; if (splittingStrategy == SplittingStrategy.None) { FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient); return; } for (int runIndex = 0; runIndex < sarifLog.Runs?.Count; ++runIndex) { if (sarifLog.Runs[runIndex]?.Results?.Count > 0) { IList <SarifLog> logsToProcess = new List <SarifLog>(new SarifLog[] { sarifLog }); if (splittingStrategy != SplittingStrategy.PerRun) { SplittingVisitor visitor; switch (splittingStrategy) { case SplittingStrategy.PerRunPerRule: visitor = new PerRunPerRuleSplittingVisitor(); break; case SplittingStrategy.PerRunPerTarget: visitor = new PerRunPerTargetSplittingVisitor(); break; case SplittingStrategy.PerRunPerTargetPerRule: visitor = new PerRunPerTargetPerRuleSplittingVisitor(); break; // TODO: Implement PerResult and PerRun splittings strategies // // https://github.com/microsoft/sarif-sdk/issues/1763 // https://github.com/microsoft/sarif-sdk/issues/1762 // case SplittingStrategy.PerResult: case SplittingStrategy.PerRun: default: throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } visitor.VisitRun(sarifLog.Runs[runIndex]); logsToProcess = visitor.SplitSarifLogs; } for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++) { SarifLog splitLog = logsToProcess[splitFileIndex]; FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient); } } } }
public virtual IReadOnlyList <SarifLog> SplitLogFile(SarifLog sarifLog) { IList <SarifLog> logsToProcess; using (Logger.BeginScopeContext(nameof(SplitLogFile))) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); sarifLog.SetProperty("guid", Guid.NewGuid()); this.FilingResult = FilingResult.None; this.FiledWorkItems = new List <WorkItemModel>(); sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { Logger.LogDebug("Removing optional data."); var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { Logger.LogDebug("Inserting optional data."); var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } using (Logger.BeginScopeContext("Splitting visitor")) { SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; Logger.LogInformation($"Splitting strategy - {splittingStrategy}"); if (splittingStrategy == SplittingStrategy.None) { return(new[] { sarifLog }); } PartitionFunction <string> partitionFunction = null; Stopwatch splittingStopwatch = Stopwatch.StartNew(); switch (splittingStrategy) { case SplittingStrategy.PerRun: { partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null; break; } case SplittingStrategy.PerResult: { partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null; break; } case SplittingStrategy.PerRunPerOrgPerEntityTypePerPartialFingerprint: { partitionFunction = (result) => result.ShouldBeFiled() ? result.GetFingerprintSplittingStrategyId() : null; break; } case SplittingStrategy.PerRunPerOrgPerEntityTypePerRepositoryPerPartialFingerprint: { partitionFunction = (result) => result.ShouldBeFiled() ? result.GetPerRepositoryFingerprintSplittingStrategyId() : null; break; } default: { throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } } Logger.LogDebug("Begin splitting logs"); var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false); partitioningVisitor.VisitSarifLog(sarifLog); Logger.LogDebug("Begin retrieving split logs"); logsToProcess = new List <SarifLog>(partitioningVisitor.GetPartitionLogs().Values); Logger.LogDebug("End retrieving split logs"); var logsToProcessMetrics = new Dictionary <string, object> { { "splittingStrategy", splittingStrategy }, { "logsToProcessCount", logsToProcess.Count }, { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds }, }; this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics); splittingStopwatch.Stop(); } } if (logsToProcess != null && !this.FilingContext.ShouldFileUnchanged) { // Remove any logs that do not contain at least one result with a New or None baselinestate. logsToProcess = logsToProcess.Where(log => log?.Runs?.Any(run => run.Results?.Any(result => result.BaselineState == BaselineState.New || result.BaselineState == BaselineState.None) == true) == true).ToList(); } return(logsToProcess.ToArray()); }
public virtual SarifLog FileWorkItems(SarifLog sarifLog) { sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); sarifLog.SetProperty("guid", Guid.NewGuid()); using (Logger.BeginScope(nameof(FileWorkItems))) { this.FilingResult = FilingResult.None; this.FiledWorkItems = new List<WorkItemModel>(); sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog)); Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization); this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait(); OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData); dataRemovingVisitor.Visit(sarifLog); } optionallyEmittedData = this.FilingContext.DataToInsert; if (optionallyEmittedData != OptionallyEmittedData.None) { var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData); dataInsertingVisitor.Visit(sarifLog); } SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy; if (splittingStrategy == SplittingStrategy.None) { FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient); return sarifLog; } IList<SarifLog> logsToProcess; PartitionFunction<string> partitionFunction = null; Stopwatch splittingStopwatch = Stopwatch.StartNew(); switch (splittingStrategy) { case SplittingStrategy.PerRun: { partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null; break; } case SplittingStrategy.PerResult: { partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null; break; } default: { throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}"); } } var partitioningVisitor = new PartitioningVisitor<string>(partitionFunction, deepClone: false); partitioningVisitor.VisitSarifLog(sarifLog); logsToProcess = new List<SarifLog>(partitioningVisitor.GetPartitionLogs().Values); var logsToProcessMetrics = new Dictionary<string, object> { { "splittingStrategy", splittingStrategy }, { "logsToProcessCount", logsToProcess.Count }, { "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds }, }; this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics); splittingStopwatch.Stop(); for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++) { SarifLog splitLog = logsToProcess[splitFileIndex]; FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient); } } return sarifLog; }
public static IActionWrapper <SarifLog> GetActionStage(SarifLogAction action, params string[] args) { switch (action) { case SarifLogAction.None: { return(new GenericMappingAction <SarifLog>(a => a)); } case SarifLogAction.MakeUrisAbsolute: { return(new GenericMappingAction <SarifLog>(log => { MakeUrisAbsoluteVisitor visitor = new MakeUrisAbsoluteVisitor(); return visitor.VisitSarifLog(log); })); } case SarifLogAction.InsertOptionalData: { return(new GenericMappingAction <SarifLog>(log => { bool optionalDataArgValid = Enum.TryParse(args[0], out OptionallyEmittedData optionalData); Debug.Assert(optionalDataArgValid); if (optionalData != 0) { var visitor = new InsertOptionalDataVisitor(optionalData); return visitor.VisitSarifLog(log); } return log; })); } case SarifLogAction.RemoveOptionalData: { return(new GenericMappingAction <SarifLog>(log => { bool optionalDataArgValid = Enum.TryParse(args[0], out OptionallyEmittedData optionalData); Debug.Assert(optionalDataArgValid); if (optionalData != 0) { var visitor = new InsertOptionalDataVisitor(optionalData); return visitor.VisitSarifLog(log); } return log; })); } case SarifLogAction.RebaseUri: { return(new GenericMappingAction <SarifLog>(log => { bool rebaseRelativeUrisValid = bool.TryParse(args[1], out bool rebaseRelativeUris); Debug.Assert(rebaseRelativeUrisValid); var visitor = new RebaseUriVisitor(args[0], new Uri(args[2]), rebaseRelativeUris); return visitor.VisitSarifLog(log); })); } case SarifLogAction.Merge: { bool mergeEmptyLogsArgValid = bool.TryParse(args.Length == 0 ? "true" : args[0], out bool mergeEmptyLogs); Debug.Assert(mergeEmptyLogsArgValid); return(new GenericFoldAction <SarifLog>((accumulator, nextLog) => { if (nextLog.Runs == null) { return accumulator; } if (accumulator.Runs == null) { accumulator.Runs = new List <Run>(); } foreach (Run run in nextLog.Runs) { if (run != null && (mergeEmptyLogs || run?.Results.Count > 0)) { accumulator.Runs.Add(run); } } return accumulator; })); } case SarifLogAction.Sort: { throw new NotImplementedException(); } case SarifLogAction.MakeDeterministic: { throw new NotImplementedException(); } default: throw new ArgumentException($"Unknown/Not Supported Action {action}.", nameof(action)); } }