public void SarifLogResultMatcher_PreservesPropertiesProperly()
{
// Verify that SarifLog matching keeps old or latest property values on matched Results
// depending on how it was configured.
// This test used to verify that old Artifact properties are also preserved, but this behavior
// doesn't make sense to us. If we need Artifacts to have property values from old runs, we
// will need to merge them carefully, keeping the latest URI, File Content, and Hashes, and only
// taking the older version of the Property Bag properties. (The latest merging code just took the
// old Artifact wholesale)
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
string baselinePropertyValue = Guid.NewGuid().ToString();
string currentPropertyValue = Guid.NewGuid().ToString();
SetPropertyOnAllResultObjects(baselineLog, "Key", baselinePropertyValue);
SetPropertyOnAllResultObjects(currentLog, "Key", currentPropertyValue);
SarifLog matchedLog = s_preserveOldestPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone());
matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count);
// Retain property bag values from most current run
matchedLog = s_preserveMostRecentPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone());
matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count);
}
public static int Run(BaselineOptions baselineOptions)
{
try
{
SarifLog baselineFile = MultitoolFileHelpers.ReadSarifFile <SarifLog>(baselineOptions.BaselineFilePath);
SarifLog currentFile = MultitoolFileHelpers.ReadSarifFile <SarifLog>(baselineOptions.CurrentFilePath);
if (baselineFile.Runs.Count != 1 || currentFile.Runs.Count != 1)
{
throw new ArgumentException("Invalid sarif logs, we can only baseline logs with a single run in them.");
}
ISarifLogBaseliner baseliner = SarifLogBaselinerFactory.CreateSarifLogBaseliner(baselineOptions.BaselineType);
Run diffedRun = baseliner.CreateBaselinedRun(baselineFile.Runs.First(), currentFile.Runs.First());
SarifLog output = currentFile.DeepClone();
output.Runs = new List <Run>();
output.Runs.Add(diffedRun);
var formatting = baselineOptions.PrettyPrint
? Formatting.Indented
: Formatting.None;
MultitoolFileHelpers.WriteSarifFile(output, baselineOptions.OutputFilePath, formatting);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(1);
}
return(0);
}
public void ResultMatchingBaseliner_BaselinesTwoSimpleSarifLogs()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
baselineLog.Runs[0].Id = new RunAutomationDetails {
InstanceGuid = Guid.NewGuid().ToString()
};
currentLog.Runs[0].Id = new RunAutomationDetails {
InstanceGuid = Guid.NewGuid().ToString()
};
if (currentLog.Runs[0].Results.Any())
{
currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag");
}
foreach (Result result in baselineLog.Runs[0].Results)
{
result.CorrelationGuid = Guid.NewGuid().ToString();
}
SarifLog calculatedNextBaseline = baseliner.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First();
calculatedNextBaseline.Runs.Should().HaveCount(1);
if (currentLog.Runs[0].Results.Any())
{
calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1);
calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue();
AbsentResultProperties.Should().ContainKey("Run");
AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].Id.InstanceGuid);
int existingCount = currentLog.Runs[0].Results.Count - 1;
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).Count().Should().Be(existingCount);
if (existingCount > 0)
{
// In the event that we generated a SARIF run of only a single result, we will not have an 'existing' match
// since we adjusted the sole result value by adding a property to it.
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue();
CurrentResultProperties.Should().ContainKey("Run");
CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].Id.InstanceGuid);
}
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue();
NewResultProperties.Should().ContainKey("Run");
NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].Id.InstanceGuid);
}
}
/// <summary>
/// Filter the specified SARIF log to create a new log containing only those results for
/// which the specified predicate returns true, and only those elements of run-level
/// collections such as Run.Artifacts that are relevant to the filtered results.
/// </summary>
/// <param name="log">
/// The log file to be filtered.
/// </param>
/// <param name="predicate">
/// The predicate that selects the results in the filtered log file.
/// </param>
/// <returns>
/// A new SARIF log containing only the filtered results, and only the relevant elements
/// of the run-level collections.
/// </returns>
public static SarifLog Filter(SarifLog log, FilteringVisitor.IncludeResultPredicate predicate)
{
SarifLog newLog = log.DeepClone();
var visitor = new FilteringVisitor(predicate);
return(visitor.VisitSarifLog(newLog));
}
public void SarifLogResultMatcher_MatchMultipleCurrentLogsWithNoBaseline()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog mostRecentLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog oldestLog = mostRecentLog.DeepClone();
string sharedPropertyName = nameof(sharedPropertyName);
string currentSharedPropertyValue = Guid.NewGuid().ToString();
string uniqueToMostRecentPropertyName = nameof(uniqueToMostRecentPropertyName);
string uniqueToMostRecentPropertyValue = Guid.NewGuid().ToString();
string uniqueToOldestPropertyName = nameof(uniqueToOldestPropertyName);
string uniqueToOldestPropertyValue = Guid.NewGuid().ToString();
mostRecentLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue);
oldestLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue);
mostRecentLog.Runs[0].SetProperty(uniqueToMostRecentPropertyName, uniqueToMostRecentPropertyValue);
oldestLog.Runs[0].SetProperty(uniqueToOldestPropertyName, uniqueToOldestPropertyValue);
SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match(
previousLogs: null,
currentLogs: new SarifLog[] { oldestLog, mostRecentLog }).First();
calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull();
calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2);
string value = null;
// The default property bag matching behavior is to retain the oldest property bag available. Since we have no
// baseline in this test, we expect the preserved property bag to be associated with the first (i.e., oldest)
// run in the currentLogs property
calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToOldestPropertyName, out value).Should().BeTrue();
calculatedNextBaseline.Runs[0].GetProperty(uniqueToOldestPropertyName).Should().Be(uniqueToOldestPropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToMostRecentPropertyName, out value).Should().BeFalse();
calculatedNextBaseline = s_preserveMostRecentPropertyBagMatcher.Match(
previousLogs: null,
currentLogs: new SarifLog[] { oldestLog, mostRecentLog }).First();
calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull();
calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2);
calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToMostRecentPropertyName, out value).Should().BeTrue();
calculatedNextBaseline.Runs[0].GetProperty(uniqueToMostRecentPropertyName).Should().Be(uniqueToMostRecentPropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToOldestPropertyName, out value).Should().BeFalse();
}
public void SarifLogResultMatcher_BaselinesTwoSimpleSarifLogs()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
baselineLog.Runs[0].InstanceGuid = Guid.NewGuid().ToString();
currentLog.Runs[0].InstanceGuid = Guid.NewGuid().ToString();
if (currentLog.Runs[0].Results.Any())
{
currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag");
}
foreach (Result result in baselineLog.Runs[0].Results)
{
result.CorrelationGuid = Guid.NewGuid().ToString();
}
SarifLog calculatedNextBaseline = baseliner.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First();
calculatedNextBaseline.Runs.Should().HaveCount(1);
if (currentLog.Runs[0].Results.Any())
{
calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1);
calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue();
AbsentResultProperties.Should().ContainKey("Run");
AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].InstanceGuid);
if (currentLog.Runs[0].Results.Count > 1)
{
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Existing).Should().HaveCount(currentLog.Runs[0].Results.Count - 1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Existing).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue();
CurrentResultProperties.Should().ContainKey("Run");
CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].InstanceGuid);
}
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue();
NewResultProperties.Should().ContainKey("Run");
NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].InstanceGuid);
}
}
public void RemoveGuids()
{
SarifLog log = _sampleLog.DeepClone();
RemoveOptionalDataVisitor v = new RemoveOptionalDataVisitor(OptionallyEmittedData.None);
v.Visit(log);
log.Runs[0].Results[0].Guid.Should().NotBeNull();
v = new RemoveOptionalDataVisitor(OptionallyEmittedData.Guids);
v.Visit(log);
log.Runs[0].Results[0].Guid.Should().BeNull();
}
public void SarifLogResultMatcher_PreservesPropertiesProperly()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
string baselinePropertyValue = Guid.NewGuid().ToString();
string currentPropertyValue = Guid.NewGuid().ToString();
SetPropertyOnAllFileAndResultObjects(baselineLog, "Key", baselinePropertyValue);
SetPropertyOnAllFileAndResultObjects(currentLog, "Key", currentPropertyValue);
SarifLog matchedLog = s_preserveOldestPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone());
matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count);
matchedLog.Runs[0].Artifacts?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Artifacts.Count);
// Retain property bag values from most current run
matchedLog = s_preserveMostRecentPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone());
matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count);
matchedLog.Runs[0].Artifacts?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Artifacts.Count);
}
public void TestMerge_WorksAsExpected()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
List <SarifLog> logs = new List <SarifLog>();
List <SarifLog> secondLogSet = new List <SarifLog>();
int count = random.Next(10) + 1;
for (int i = 0; i < count; i++)
{
SarifLog log = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, random.Next(1, 10));
logs.Add(log);
secondLogSet.Add(log.DeepClone());
}
SarifLog combinedLog = logs.Merge();
combinedLog.Runs.Count.Should().Be(secondLogSet.Select(l => l.Runs == null ? 0 : l.Runs.Count).Sum());
}
public void SarifLogResultMatcher_BaselinesSarifLogsWithProperties()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
string sharedPropertyName = nameof(sharedPropertyName);
string currentSharedPropertyValue = Guid.NewGuid().ToString();
string uniqueToBaselinePropertyName = nameof(uniqueToBaselinePropertyName);
string uniqueToBaselinePropertyValue = Guid.NewGuid().ToString();
string uniqueToCurrentPropertyName = nameof(uniqueToCurrentPropertyName);
string uniqueToCurrentPropertyValue = Guid.NewGuid().ToString();
baselineLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue);
currentLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue);
baselineLog.Runs[0].SetProperty(uniqueToBaselinePropertyName, uniqueToBaselinePropertyValue);
currentLog.Runs[0].SetProperty(uniqueToCurrentPropertyName, uniqueToCurrentPropertyValue);
SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First();
string value = null;
// The default property bag matching behavior is to retain the property bag in its entirety from the baseline
calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull();
calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2);
calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue);
calculatedNextBaseline.Runs[0].GetProperty(uniqueToBaselinePropertyName).Should().Be(uniqueToBaselinePropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToCurrentPropertyName, out value).Should().BeFalse();
calculatedNextBaseline = s_preserveMostRecentPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First();
// The default property bag matching behavior is to retain the property bag in its entirety from the baseline
calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull();
calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2);
calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue);
calculatedNextBaseline.Runs[0].GetProperty(uniqueToCurrentPropertyName).Should().Be(uniqueToCurrentPropertyValue);
calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToBaselinePropertyName, out value).Should().BeFalse();
}
public void SarifLogResultMatcher_BaselinesTwoSimpleSarifLogs()
{
Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output);
SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1);
SarifLog currentLog = baselineLog.DeepClone();
baselineLog.Runs[0].AutomationDetails = new RunAutomationDetails {
Guid = Guid.NewGuid().ToString()
};
currentLog.Runs[0].AutomationDetails = new RunAutomationDetails {
Guid = Guid.NewGuid().ToString()
};
// This code exists to force a result to diverge from the previous run. By modifying this tag,
// we ensure that at least one result will be regarded as new (which implies one result
// will be regarded as going absent).
if (currentLog.Runs[0].Results.Any())
{
currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag");
}
string propertyName = "WeLikePi";
float propertyValue = 3.14159F;
baselineLog.Runs[0].SetProperty(propertyName, propertyValue);
foreach (Result result in baselineLog.Runs[0].Results)
{
result.CorrelationGuid = Guid.NewGuid().ToString();
}
SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First();
calculatedNextBaseline.Runs.Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull();
calculatedNextBaseline.Runs[0].GetProperty <float>(propertyName).Should().Be(propertyValue);
if (currentLog.Runs[0].Results.Any())
{
calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1);
calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue();
AbsentResultProperties.Should().ContainKey("Run");
AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].AutomationDetails.Guid);
if (currentLog.Runs[0].Results.Count > 1)
{
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).Should().HaveCount(currentLog.Runs[0].Results.Count - 1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue();
CurrentResultProperties.Should().ContainKey("Run");
CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].AutomationDetails.Guid);
}
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1);
calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue();
NewResultProperties.Should().ContainKey("Run");
NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].AutomationDetails.Guid);
}
}
public void SarifLogResultMatcher_MultipleLogsDuplicateData_WorksAsExpected()
{
SarifLog current1 = new SarifLog()
{
Runs = new Run[]
{
new Run()
{
Tool = new Tool {
Driver = new ToolComponent {
Name = "TestTool"
}
},
Artifacts = new List <Artifact>
{
new Artifact()
{
Contents = new ArtifactContent()
{
Text = "TestFileContents"
}
}
},
Results = new List <Result>
{
new Result
{
Message = new Message {
Text = "Some testing occurred."
},
Locations = new List <Location>
{
new Location
{
PhysicalLocation = new PhysicalLocation
{
ArtifactLocation = new ArtifactLocation
{
Index = 0
}
}
}
}
}
}
}
}
};
SarifLog current2 = current1.DeepClone();
SarifLog result = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[0], new SarifLog[] { current1, current2 }).First();
// In first match operation, the file data objects are identical, they should be collapsed into a single array entry
result.Runs[0].Artifacts.Should().HaveCount(1);
current2 = current1.DeepClone();
// Now we differentiate a files object
current2.Runs[0].Artifacts[0].Contents.Text = Guid.NewGuid().ToString();
result = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[0], new SarifLog[] { current1, current2 }).First();
// Both file data objects should be present
result.Runs[0].Artifacts.Should().HaveCount(2);
// Merged results should each refer to differentiated file data object
result.Runs[0].Results[0].Locations[0].PhysicalLocation.ArtifactLocation.Index.Should().Be(0);
result.Runs[0].Results[1].Locations[0].PhysicalLocation.ArtifactLocation.Index.Should().Be(1);
}
