public int Run(RewriteOptions rewriteOptions)
{
try
{
Console.WriteLine($"Rewriting '{rewriteOptions.InputFilePath}' => '{rewriteOptions.OutputFilePath}'...");
Stopwatch w = Stopwatch.StartNew();
bool valid = ValidateOptions(rewriteOptions);
if (!valid)
{
return(FAILURE);
}
SarifLog actualLog = ReadSarifFile <SarifLog>(_fileSystem, rewriteOptions.InputFilePath);
OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags();
IDictionary <string, ArtifactLocation> originalUriBaseIds = rewriteOptions.ConstructUriBaseIdsDictionary();
SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(actualLog);
string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions);
WriteSarifFile(_fileSystem, reformattedLog, fileName, rewriteOptions.Formatting);
w.Stop();
Console.WriteLine($"Rewrite completed in {w.Elapsed}.");
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(FAILURE);
}
return(SUCCESS);
}
public int Run(RewriteOptions rewriteOptions)
{
try
{
rewriteOptions = ValidateOptions(rewriteOptions);
SarifLog actualLog = FileHelpers.ReadSarifFile <SarifLog>(_fileSystem, rewriteOptions.InputFilePath);
OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags();
IDictionary <string, ArtifactLocation> originalUriBaseIds = rewriteOptions.ConstructUriBaseIdsDictionary();
SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(actualLog);
string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions);
var formatting = rewriteOptions.PrettyPrint
? Formatting.Indented
: Formatting.None;
FileHelpers.WriteSarifFile(_fileSystem, reformattedLog, fileName, formatting);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(1);
}
return(0);
}
public static int Run(RewriteOptions rewriteOptions)
{
try
{
rewriteOptions = ValidateOptions(rewriteOptions);
SarifLog actualLog = MultitoolFileHelpers.ReadSarifFile <SarifLog>(rewriteOptions.InputFilePath);
OptionallyEmittedData dataToInsert = rewriteOptions.DataToInsert.ToFlags();
SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(actualLog);
string fileName = CommandUtilities.GetTransformedOutputFileName(rewriteOptions);
var formatting = rewriteOptions.PrettyPrint
? Formatting.Indented
: Formatting.None;
MultitoolFileHelpers.WriteSarifFile(reformattedLog, fileName, formatting);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(1);
}
return(0);
}
public int Run(RewriteOptions options)
{
try
{
Console.WriteLine($"Rewriting '{options.InputFilePath}' => '{options.OutputFilePath}'...");
Stopwatch w = Stopwatch.StartNew();
bool valid = ValidateOptions(options);
if (!valid)
{
return(FAILURE);
}
string actualOutputPath = CommandUtilities.GetTransformedOutputFileName(options);
SarifLog actualLog = null;
string inputVersion = SniffVersion(options.InputFilePath);
if (!inputVersion.Equals(SarifUtilities.StableSarifVersion))
{
actualLog = TransformFileToVersionTwo(options.InputFilePath, inputVersion);
}
else
{
actualLog = ReadSarifFile <SarifLog>(_fileSystem, options.InputFilePath);
}
OptionallyEmittedData dataToInsert = options.DataToInsert.ToFlags();
OptionallyEmittedData dataToRemove = options.DataToRemove.ToFlags();
IDictionary <string, ArtifactLocation> originalUriBaseIds = options.ConstructUriBaseIdsDictionary();
SarifLog reformattedLog = new RemoveOptionalDataVisitor(dataToRemove).VisitSarifLog(actualLog);
reformattedLog = new InsertOptionalDataVisitor(dataToInsert, originalUriBaseIds).VisitSarifLog(reformattedLog);
if (options.SarifOutputVersion == SarifVersion.OneZeroZero)
{
var visitor = new SarifCurrentToVersionOneVisitor();
visitor.VisitSarifLog(reformattedLog);
WriteSarifFile(_fileSystem, visitor.SarifLogVersionOne, actualOutputPath, options.Minify, SarifContractResolverVersionOne.Instance);
}
else
{
WriteSarifFile(_fileSystem, reformattedLog, actualOutputPath, options.Minify);
}
w.Stop();
Console.WriteLine($"Rewrite completed in {w.Elapsed}.");
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(FAILURE);
}
return(SUCCESS);
}
public SarifLogger(
TextWriter textWriter,
LogFilePersistenceOptions logFilePersistenceOptions = DefaultLogFilePersistenceOptions,
OptionallyEmittedData dataToInsert = OptionallyEmittedData.None,
OptionallyEmittedData dataToRemove = OptionallyEmittedData.None,
Tool tool = null,
Run run = null,
IEnumerable <string> analysisTargets = null,
IEnumerable <string> invocationTokensToRedact = null,
IEnumerable <string> invocationPropertiesToLog = null,
string defaultFileEncoding = null,
bool closeWriterOnDispose = true,
IEnumerable <FailureLevel> levels = null,
IEnumerable <ResultKind> kinds = null) : this(textWriter, logFilePersistenceOptions, closeWriterOnDispose, levels, kinds)
{
if (dataToInsert.HasFlag(OptionallyEmittedData.Hashes))
{
AnalysisTargetToHashDataMap = HashUtilities.MultithreadedComputeTargetFileHashes(analysisTargets);
}
_run = run ?? new Run();
if (dataToInsert.HasFlag(OptionallyEmittedData.RegionSnippets) || dataToInsert.HasFlag(OptionallyEmittedData.ContextRegionSnippets))
{
_insertOptionalDataVisitor = new InsertOptionalDataVisitor(dataToInsert, _run);
}
EnhanceRun(
analysisTargets,
dataToInsert,
dataToRemove,
invocationTokensToRedact,
invocationPropertiesToLog,
defaultFileEncoding,
AnalysisTargetToHashDataMap);
tool = tool ?? Tool.CreateFromAssemblyData();
_run.Tool = tool;
_dataToInsert = dataToInsert;
_dataToRemove = dataToRemove;
_issueLogJsonWriter.Initialize(_run);
// Map existing Rules to ensure duplicates aren't created
if (_run.Tool.Driver?.Rules != null)
{
for (int i = 0; i < _run.Tool.Driver.Rules.Count; ++i)
{
RuleToIndexMap[_run.Tool.Driver.Rules[i]] = i;
}
}
_persistArtifacts =
(_dataToInsert & OptionallyEmittedData.Hashes) != 0 ||
(_dataToInsert & OptionallyEmittedData.TextFiles) != 0 ||
(_dataToInsert & OptionallyEmittedData.BinaryFiles) != 0;
}
public int Run(MergeOptions mergeOptions)
{
try
{
string outputDirectory = mergeOptions.OutputFolderPath ?? Environment.CurrentDirectory;
string outputFilePath = Path.Combine(outputDirectory, GetOutputFileName(mergeOptions));
if (!DriverUtilities.ReportWhetherOutputFileCanBeCreated(outputFilePath, mergeOptions.Force, _fileSystem))
{
return(FAILURE);
}
HashSet <string> sarifFiles = CreateTargetsSet(mergeOptions.TargetFileSpecifiers, mergeOptions.Recurse, _fileSystem);
IEnumerable <SarifLog> allRuns = ParseFiles(sarifFiles);
// Build one SarifLog with all the Runs.
SarifLog combinedLog = allRuns.Merge();
// If there were no input files, the Merge operation set combinedLog.Runs to null. Although
// null is valid in certain error cases, it is not valid here. Here, the correct value is
// an empty list. See the SARIF spec, §3.13.4, "runs property".
combinedLog.Runs = combinedLog.Runs ?? new List <Run>();
combinedLog.Version = SarifVersion.Current;
combinedLog.SchemaUri = combinedLog.Version.ConvertToSchemaUri();
OptionallyEmittedData dataToInsert = mergeOptions.DataToInsert.ToFlags();
if (dataToInsert != OptionallyEmittedData.None)
{
combinedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(combinedLog);
}
// Write output to file.
Formatting formatting = mergeOptions.PrettyPrint
? Formatting.Indented
: Formatting.None;
_fileSystem.CreateDirectory(outputDirectory);
WriteSarifFile(_fileSystem, combinedLog, outputFilePath, formatting);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(FAILURE);
}
return(SUCCESS);
}
private static void EnrichSarifLog(string actualFilePath)
{
JsonSerializerSettings settings = new JsonSerializerSettings()
{
Formatting = Formatting.Indented,
};
string logText = File.ReadAllText(actualFilePath);
SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(logText, settings);
OptionallyEmittedData dataToInsert =
OptionallyEmittedData.Hashes |
OptionallyEmittedData.RegionSnippets |
OptionallyEmittedData.ContextRegionSnippets |
OptionallyEmittedData.ComprehensiveRegionProperties;
SarifLog reformattedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(actualLog);
File.WriteAllText(actualFilePath, JsonConvert.SerializeObject(reformattedLog, settings));
}
public int Run(MergeOptions mergeOptions)
{
try
{
var sarifFiles = CreateTargetsSet(mergeOptions.TargetFileSpecifiers, mergeOptions.Recurse, _fileSystem);
var allRuns = ParseFiles(sarifFiles);
// Build one SarifLog with all the Runs.
SarifLog combinedLog = allRuns.Merge();
combinedLog.Version = SarifVersion.Current;
combinedLog.SchemaUri = combinedLog.Version.ConvertToSchemaUri();
OptionallyEmittedData dataToInsert = mergeOptions.DataToInsert.ToFlags();
if (dataToInsert != OptionallyEmittedData.None)
{
combinedLog = new InsertOptionalDataVisitor(dataToInsert).VisitSarifLog(combinedLog);
}
string outputDirectory = mergeOptions.OutputFolderPath ?? Environment.CurrentDirectory;
// Write output to file.
string outputName = Path.Combine(outputDirectory, GetOutputFileName(mergeOptions));
var formatting = mergeOptions.PrettyPrint
? Formatting.Indented
: Formatting.None;
Directory.CreateDirectory(outputDirectory);
WriteSarifFile(_fileSystem, combinedLog, outputName, formatting);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(1);
}
return(0);
}
public virtual void FileWorkItems(SarifLog sarifLog)
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
if (splittingStrategy == SplittingStrategy.None)
{
FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient);
return;
}
for (int runIndex = 0; runIndex < sarifLog.Runs?.Count; ++runIndex)
{
if (sarifLog.Runs[runIndex]?.Results?.Count > 0)
{
IList <SarifLog> logsToProcess = new List <SarifLog>(new SarifLog[] { sarifLog });
if (splittingStrategy != SplittingStrategy.PerRun)
{
SplittingVisitor visitor;
switch (splittingStrategy)
{
case SplittingStrategy.PerRunPerRule:
visitor = new PerRunPerRuleSplittingVisitor();
break;
case SplittingStrategy.PerRunPerTarget:
visitor = new PerRunPerTargetSplittingVisitor();
break;
case SplittingStrategy.PerRunPerTargetPerRule:
visitor = new PerRunPerTargetPerRuleSplittingVisitor();
break;
// TODO: Implement PerResult and PerRun splittings strategies
//
// https://github.com/microsoft/sarif-sdk/issues/1763
// https://github.com/microsoft/sarif-sdk/issues/1762
//
case SplittingStrategy.PerResult:
case SplittingStrategy.PerRun:
default:
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
visitor.VisitRun(sarifLog.Runs[runIndex]);
logsToProcess = visitor.SplitSarifLogs;
}
for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++)
{
SarifLog splitLog = logsToProcess[splitFileIndex];
FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient);
}
}
}
}
public virtual IReadOnlyList <SarifLog> SplitLogFile(SarifLog sarifLog)
{
IList <SarifLog> logsToProcess;
using (Logger.BeginScopeContext(nameof(SplitLogFile)))
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
sarifLog.SetProperty("guid", Guid.NewGuid());
this.FilingResult = FilingResult.None;
this.FiledWorkItems = new List <WorkItemModel>();
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
Logger.LogDebug("Removing optional data.");
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
Logger.LogDebug("Inserting optional data.");
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
using (Logger.BeginScopeContext("Splitting visitor"))
{
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
Logger.LogInformation($"Splitting strategy - {splittingStrategy}");
if (splittingStrategy == SplittingStrategy.None)
{
return(new[] { sarifLog });
}
PartitionFunction <string> partitionFunction = null;
Stopwatch splittingStopwatch = Stopwatch.StartNew();
switch (splittingStrategy)
{
case SplittingStrategy.PerRun:
{
partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
break;
}
case SplittingStrategy.PerResult:
{
partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
break;
}
case SplittingStrategy.PerRunPerOrgPerEntityTypePerPartialFingerprint:
{
partitionFunction = (result) => result.ShouldBeFiled() ? result.GetFingerprintSplittingStrategyId() : null;
break;
}
case SplittingStrategy.PerRunPerOrgPerEntityTypePerRepositoryPerPartialFingerprint:
{
partitionFunction = (result) => result.ShouldBeFiled() ? result.GetPerRepositoryFingerprintSplittingStrategyId() : null;
break;
}
default:
{
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
}
Logger.LogDebug("Begin splitting logs");
var partitioningVisitor = new PartitioningVisitor <string>(partitionFunction, deepClone: false);
partitioningVisitor.VisitSarifLog(sarifLog);
Logger.LogDebug("Begin retrieving split logs");
logsToProcess = new List <SarifLog>(partitioningVisitor.GetPartitionLogs().Values);
Logger.LogDebug("End retrieving split logs");
var logsToProcessMetrics = new Dictionary <string, object>
{
{ "splittingStrategy", splittingStrategy },
{ "logsToProcessCount", logsToProcess.Count },
{ "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
};
this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
splittingStopwatch.Stop();
}
}
if (logsToProcess != null && !this.FilingContext.ShouldFileUnchanged)
{
// Remove any logs that do not contain at least one result with a New or None baselinestate.
logsToProcess = logsToProcess.Where(log => log?.Runs?.Any(run => run.Results?.Any(result => result.BaselineState == BaselineState.New || result.BaselineState == BaselineState.None) == true) == true).ToList();
}
return(logsToProcess.ToArray());
}
public virtual SarifLog FileWorkItems(SarifLog sarifLog)
{
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
sarifLog.SetProperty("guid", Guid.NewGuid());
using (Logger.BeginScope(nameof(FileWorkItems)))
{
this.FilingResult = FilingResult.None;
this.FiledWorkItems = new List<WorkItemModel>();
sarifLog = sarifLog ?? throw new ArgumentNullException(nameof(sarifLog));
Logger.LogInformation("Connecting to filing client: {accountOrOrganization}", this.FilingClient.AccountOrOrganization);
this.FilingClient.Connect(this.FilingContext.PersonalAccessToken).Wait();
OptionallyEmittedData optionallyEmittedData = this.FilingContext.DataToRemove;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataRemovingVisitor = new RemoveOptionalDataVisitor(optionallyEmittedData);
dataRemovingVisitor.Visit(sarifLog);
}
optionallyEmittedData = this.FilingContext.DataToInsert;
if (optionallyEmittedData != OptionallyEmittedData.None)
{
var dataInsertingVisitor = new InsertOptionalDataVisitor(optionallyEmittedData);
dataInsertingVisitor.Visit(sarifLog);
}
SplittingStrategy splittingStrategy = this.FilingContext.SplittingStrategy;
if (splittingStrategy == SplittingStrategy.None)
{
FileWorkItemsHelper(sarifLog, this.FilingContext, this.FilingClient);
return sarifLog;
}
IList<SarifLog> logsToProcess;
PartitionFunction<string> partitionFunction = null;
Stopwatch splittingStopwatch = Stopwatch.StartNew();
switch (splittingStrategy)
{
case SplittingStrategy.PerRun:
{
partitionFunction = (result) => result.ShouldBeFiled() ? "Include" : null;
break;
}
case SplittingStrategy.PerResult:
{
partitionFunction = (result) => result.ShouldBeFiled() ? Guid.NewGuid().ToString() : null;
break;
}
default:
{
throw new ArgumentOutOfRangeException($"SplittingStrategy: {splittingStrategy}");
}
}
var partitioningVisitor = new PartitioningVisitor<string>(partitionFunction, deepClone: false);
partitioningVisitor.VisitSarifLog(sarifLog);
logsToProcess = new List<SarifLog>(partitioningVisitor.GetPartitionLogs().Values);
var logsToProcessMetrics = new Dictionary<string, object>
{
{ "splittingStrategy", splittingStrategy },
{ "logsToProcessCount", logsToProcess.Count },
{ "splittingDurationInMilliseconds", splittingStopwatch.ElapsedMilliseconds },
};
this.Logger.LogMetrics(EventIds.LogsToProcessMetrics, logsToProcessMetrics);
splittingStopwatch.Stop();
for (int splitFileIndex = 0; splitFileIndex < logsToProcess.Count; splitFileIndex++)
{
SarifLog splitLog = logsToProcess[splitFileIndex];
FileWorkItemsHelper(splitLog, this.FilingContext, this.FilingClient);
}
}
return sarifLog;
}
public static IActionWrapper <SarifLog> GetActionStage(SarifLogAction action, params string[] args)
{
switch (action)
{
case SarifLogAction.None:
{
return(new GenericMappingAction <SarifLog>(a => a));
}
case SarifLogAction.MakeUrisAbsolute:
{
return(new GenericMappingAction <SarifLog>(log =>
{
MakeUrisAbsoluteVisitor visitor = new MakeUrisAbsoluteVisitor();
return visitor.VisitSarifLog(log);
}));
}
case SarifLogAction.InsertOptionalData:
{
return(new GenericMappingAction <SarifLog>(log =>
{
bool optionalDataArgValid = Enum.TryParse(args[0], out OptionallyEmittedData optionalData);
Debug.Assert(optionalDataArgValid);
if (optionalData != 0)
{
var visitor = new InsertOptionalDataVisitor(optionalData);
return visitor.VisitSarifLog(log);
}
return log;
}));
}
case SarifLogAction.RemoveOptionalData:
{
return(new GenericMappingAction <SarifLog>(log =>
{
bool optionalDataArgValid = Enum.TryParse(args[0], out OptionallyEmittedData optionalData);
Debug.Assert(optionalDataArgValid);
if (optionalData != 0)
{
var visitor = new InsertOptionalDataVisitor(optionalData);
return visitor.VisitSarifLog(log);
}
return log;
}));
}
case SarifLogAction.RebaseUri:
{
return(new GenericMappingAction <SarifLog>(log =>
{
bool rebaseRelativeUrisValid = bool.TryParse(args[1], out bool rebaseRelativeUris);
Debug.Assert(rebaseRelativeUrisValid);
var visitor = new RebaseUriVisitor(args[0], new Uri(args[2]), rebaseRelativeUris);
return visitor.VisitSarifLog(log);
}));
}
case SarifLogAction.Merge:
{
bool mergeEmptyLogsArgValid = bool.TryParse(args.Length == 0 ? "true" : args[0], out bool mergeEmptyLogs);
Debug.Assert(mergeEmptyLogsArgValid);
return(new GenericFoldAction <SarifLog>((accumulator, nextLog) =>
{
if (nextLog.Runs == null)
{
return accumulator;
}
if (accumulator.Runs == null)
{
accumulator.Runs = new List <Run>();
}
foreach (Run run in nextLog.Runs)
{
if (run != null &&
(mergeEmptyLogs || run?.Results.Count > 0))
{
accumulator.Runs.Add(run);
}
}
return accumulator;
}));
}
case SarifLogAction.Sort:
{
throw new NotImplementedException();
}
case SarifLogAction.MakeDeterministic:
{
throw new NotImplementedException();
}
default:
throw new ArgumentException($"Unknown/Not Supported Action {action}.", nameof(action));
}
}
