public void SarifLogResultMatcher_PreservesPropertiesProperly() { // Verify that SarifLog matching keeps old or latest property values on matched Results // depending on how it was configured. // This test used to verify that old Artifact properties are also preserved, but this behavior // doesn't make sense to us. If we need Artifacts to have property values from old runs, we // will need to merge them carefully, keeping the latest URI, File Content, and Hashes, and only // taking the older version of the Property Bag properties. (The latest merging code just took the // old Artifact wholesale) Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); string baselinePropertyValue = Guid.NewGuid().ToString(); string currentPropertyValue = Guid.NewGuid().ToString(); SetPropertyOnAllResultObjects(baselineLog, "Key", baselinePropertyValue); SetPropertyOnAllResultObjects(currentLog, "Key", currentPropertyValue); SarifLog matchedLog = s_preserveOldestPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone()); matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count); // Retain property bag values from most current run matchedLog = s_preserveMostRecentPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone()); matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count); }
public static int Run(BaselineOptions baselineOptions) { try { SarifLog baselineFile = MultitoolFileHelpers.ReadSarifFile <SarifLog>(baselineOptions.BaselineFilePath); SarifLog currentFile = MultitoolFileHelpers.ReadSarifFile <SarifLog>(baselineOptions.CurrentFilePath); if (baselineFile.Runs.Count != 1 || currentFile.Runs.Count != 1) { throw new ArgumentException("Invalid sarif logs, we can only baseline logs with a single run in them."); } ISarifLogBaseliner baseliner = SarifLogBaselinerFactory.CreateSarifLogBaseliner(baselineOptions.BaselineType); Run diffedRun = baseliner.CreateBaselinedRun(baselineFile.Runs.First(), currentFile.Runs.First()); SarifLog output = currentFile.DeepClone(); output.Runs = new List <Run>(); output.Runs.Add(diffedRun); var formatting = baselineOptions.PrettyPrint ? Formatting.Indented : Formatting.None; MultitoolFileHelpers.WriteSarifFile(output, baselineOptions.OutputFilePath, formatting); } catch (Exception ex) { Console.WriteLine(ex); return(1); } return(0); }
public void ResultMatchingBaseliner_BaselinesTwoSimpleSarifLogs() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); baselineLog.Runs[0].Id = new RunAutomationDetails { InstanceGuid = Guid.NewGuid().ToString() }; currentLog.Runs[0].Id = new RunAutomationDetails { InstanceGuid = Guid.NewGuid().ToString() }; if (currentLog.Runs[0].Results.Any()) { currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag"); } foreach (Result result in baselineLog.Runs[0].Results) { result.CorrelationGuid = Guid.NewGuid().ToString(); } SarifLog calculatedNextBaseline = baseliner.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First(); calculatedNextBaseline.Runs.Should().HaveCount(1); if (currentLog.Runs[0].Results.Any()) { calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1); calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue(); AbsentResultProperties.Should().ContainKey("Run"); AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].Id.InstanceGuid); int existingCount = currentLog.Runs[0].Results.Count - 1; calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).Count().Should().Be(existingCount); if (existingCount > 0) { // In the event that we generated a SARIF run of only a single result, we will not have an 'existing' match // since we adjusted the sole result value by adding a property to it. calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue(); CurrentResultProperties.Should().ContainKey("Run"); CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].Id.InstanceGuid); } calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue(); NewResultProperties.Should().ContainKey("Run"); NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].Id.InstanceGuid); } }
/// <summary> /// Filter the specified SARIF log to create a new log containing only those results for /// which the specified predicate returns true, and only those elements of run-level /// collections such as Run.Artifacts that are relevant to the filtered results. /// </summary> /// <param name="log"> /// The log file to be filtered. /// </param> /// <param name="predicate"> /// The predicate that selects the results in the filtered log file. /// </param> /// <returns> /// A new SARIF log containing only the filtered results, and only the relevant elements /// of the run-level collections. /// </returns> public static SarifLog Filter(SarifLog log, FilteringVisitor.IncludeResultPredicate predicate) { SarifLog newLog = log.DeepClone(); var visitor = new FilteringVisitor(predicate); return(visitor.VisitSarifLog(newLog)); }
public void SarifLogResultMatcher_MatchMultipleCurrentLogsWithNoBaseline() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog mostRecentLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog oldestLog = mostRecentLog.DeepClone(); string sharedPropertyName = nameof(sharedPropertyName); string currentSharedPropertyValue = Guid.NewGuid().ToString(); string uniqueToMostRecentPropertyName = nameof(uniqueToMostRecentPropertyName); string uniqueToMostRecentPropertyValue = Guid.NewGuid().ToString(); string uniqueToOldestPropertyName = nameof(uniqueToOldestPropertyName); string uniqueToOldestPropertyValue = Guid.NewGuid().ToString(); mostRecentLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue); oldestLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue); mostRecentLog.Runs[0].SetProperty(uniqueToMostRecentPropertyName, uniqueToMostRecentPropertyValue); oldestLog.Runs[0].SetProperty(uniqueToOldestPropertyName, uniqueToOldestPropertyValue); SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match( previousLogs: null, currentLogs: new SarifLog[] { oldestLog, mostRecentLog }).First(); calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull(); calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2); string value = null; // The default property bag matching behavior is to retain the oldest property bag available. Since we have no // baseline in this test, we expect the preserved property bag to be associated with the first (i.e., oldest) // run in the currentLogs property calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToOldestPropertyName, out value).Should().BeTrue(); calculatedNextBaseline.Runs[0].GetProperty(uniqueToOldestPropertyName).Should().Be(uniqueToOldestPropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToMostRecentPropertyName, out value).Should().BeFalse(); calculatedNextBaseline = s_preserveMostRecentPropertyBagMatcher.Match( previousLogs: null, currentLogs: new SarifLog[] { oldestLog, mostRecentLog }).First(); calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull(); calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2); calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToMostRecentPropertyName, out value).Should().BeTrue(); calculatedNextBaseline.Runs[0].GetProperty(uniqueToMostRecentPropertyName).Should().Be(uniqueToMostRecentPropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToOldestPropertyName, out value).Should().BeFalse(); }
public void SarifLogResultMatcher_BaselinesTwoSimpleSarifLogs() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); baselineLog.Runs[0].InstanceGuid = Guid.NewGuid().ToString(); currentLog.Runs[0].InstanceGuid = Guid.NewGuid().ToString(); if (currentLog.Runs[0].Results.Any()) { currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag"); } foreach (Result result in baselineLog.Runs[0].Results) { result.CorrelationGuid = Guid.NewGuid().ToString(); } SarifLog calculatedNextBaseline = baseliner.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First(); calculatedNextBaseline.Runs.Should().HaveCount(1); if (currentLog.Runs[0].Results.Any()) { calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1); calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue(); AbsentResultProperties.Should().ContainKey("Run"); AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].InstanceGuid); if (currentLog.Runs[0].Results.Count > 1) { calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Existing).Should().HaveCount(currentLog.Runs[0].Results.Count - 1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Existing).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue(); CurrentResultProperties.Should().ContainKey("Run"); CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].InstanceGuid); } calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue(); NewResultProperties.Should().ContainKey("Run"); NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].InstanceGuid); } }
public void RemoveGuids() { SarifLog log = _sampleLog.DeepClone(); RemoveOptionalDataVisitor v = new RemoveOptionalDataVisitor(OptionallyEmittedData.None); v.Visit(log); log.Runs[0].Results[0].Guid.Should().NotBeNull(); v = new RemoveOptionalDataVisitor(OptionallyEmittedData.Guids); v.Visit(log); log.Runs[0].Results[0].Guid.Should().BeNull(); }
public void SarifLogResultMatcher_PreservesPropertiesProperly() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); string baselinePropertyValue = Guid.NewGuid().ToString(); string currentPropertyValue = Guid.NewGuid().ToString(); SetPropertyOnAllFileAndResultObjects(baselineLog, "Key", baselinePropertyValue); SetPropertyOnAllFileAndResultObjects(currentLog, "Key", currentPropertyValue); SarifLog matchedLog = s_preserveOldestPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone()); matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count); matchedLog.Runs[0].Artifacts?.Where((r) => { return(r.GetProperty("Key") == baselinePropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Artifacts.Count); // Retain property bag values from most current run matchedLog = s_preserveMostRecentPropertyBagMatcher.Match(baselineLog.DeepClone(), currentLog.DeepClone()); matchedLog.Runs[0].Results?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Results.Count); matchedLog.Runs[0].Artifacts?.Where((r) => { return(r.GetProperty("Key") == currentPropertyValue); }).Count().Should().Be(matchedLog.Runs[0].Artifacts.Count); }
public void TestMerge_WorksAsExpected() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); List <SarifLog> logs = new List <SarifLog>(); List <SarifLog> secondLogSet = new List <SarifLog>(); int count = random.Next(10) + 1; for (int i = 0; i < count; i++) { SarifLog log = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, random.Next(1, 10)); logs.Add(log); secondLogSet.Add(log.DeepClone()); } SarifLog combinedLog = logs.Merge(); combinedLog.Runs.Count.Should().Be(secondLogSet.Select(l => l.Runs == null ? 0 : l.Runs.Count).Sum()); }
public void SarifLogResultMatcher_BaselinesSarifLogsWithProperties() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); string sharedPropertyName = nameof(sharedPropertyName); string currentSharedPropertyValue = Guid.NewGuid().ToString(); string uniqueToBaselinePropertyName = nameof(uniqueToBaselinePropertyName); string uniqueToBaselinePropertyValue = Guid.NewGuid().ToString(); string uniqueToCurrentPropertyName = nameof(uniqueToCurrentPropertyName); string uniqueToCurrentPropertyValue = Guid.NewGuid().ToString(); baselineLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue); currentLog.Runs[0].SetProperty(sharedPropertyName, currentSharedPropertyValue); baselineLog.Runs[0].SetProperty(uniqueToBaselinePropertyName, uniqueToBaselinePropertyValue); currentLog.Runs[0].SetProperty(uniqueToCurrentPropertyName, uniqueToCurrentPropertyValue); SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First(); string value = null; // The default property bag matching behavior is to retain the property bag in its entirety from the baseline calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull(); calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2); calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue); calculatedNextBaseline.Runs[0].GetProperty(uniqueToBaselinePropertyName).Should().Be(uniqueToBaselinePropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToCurrentPropertyName, out value).Should().BeFalse(); calculatedNextBaseline = s_preserveMostRecentPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First(); // The default property bag matching behavior is to retain the property bag in its entirety from the baseline calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull(); calculatedNextBaseline.Runs[0].Properties.Count.Should().Be(2); calculatedNextBaseline.Runs[0].GetProperty(sharedPropertyName).Should().Be(currentSharedPropertyValue); calculatedNextBaseline.Runs[0].GetProperty(uniqueToCurrentPropertyName).Should().Be(uniqueToCurrentPropertyValue); calculatedNextBaseline.Runs[0].TryGetProperty(uniqueToBaselinePropertyName, out value).Should().BeFalse(); }
public void SarifLogResultMatcher_BaselinesTwoSimpleSarifLogs() { Random random = RandomSarifLogGenerator.GenerateRandomAndLog(this.output); SarifLog baselineLog = RandomSarifLogGenerator.GenerateSarifLogWithRuns(random, 1); SarifLog currentLog = baselineLog.DeepClone(); baselineLog.Runs[0].AutomationDetails = new RunAutomationDetails { Guid = Guid.NewGuid().ToString() }; currentLog.Runs[0].AutomationDetails = new RunAutomationDetails { Guid = Guid.NewGuid().ToString() }; // This code exists to force a result to diverge from the previous run. By modifying this tag, // we ensure that at least one result will be regarded as new (which implies one result // will be regarded as going absent). if (currentLog.Runs[0].Results.Any()) { currentLog.Runs[0].Results[0].Tags.Add("New Unused Tag"); } string propertyName = "WeLikePi"; float propertyValue = 3.14159F; baselineLog.Runs[0].SetProperty(propertyName, propertyValue); foreach (Result result in baselineLog.Runs[0].Results) { result.CorrelationGuid = Guid.NewGuid().ToString(); } SarifLog calculatedNextBaseline = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[] { baselineLog }, new SarifLog[] { currentLog }).First(); calculatedNextBaseline.Runs.Should().HaveCount(1); calculatedNextBaseline.Runs[0].Properties.Should().NotBeNull(); calculatedNextBaseline.Runs[0].GetProperty <float>(propertyName).Should().Be(propertyValue); if (currentLog.Runs[0].Results.Any()) { calculatedNextBaseline.Runs[0].Results.Should().HaveCount(currentLog.Runs[0].Results.Count + 1); calculatedNextBaseline.Runs[0].Results.Where(r => string.IsNullOrEmpty(r.CorrelationGuid)).Should().HaveCount(0); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Absent).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> AbsentResultProperties).Should().BeTrue(); AbsentResultProperties.Should().ContainKey("Run"); AbsentResultProperties["Run"].Should().BeEquivalentTo(baselineLog.Runs[0].AutomationDetails.Guid); if (currentLog.Runs[0].Results.Count > 1) { calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).Should().HaveCount(currentLog.Runs[0].Results.Count - 1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.Unchanged).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> CurrentResultProperties).Should().BeTrue(); CurrentResultProperties.Should().ContainKey("Run"); CurrentResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].AutomationDetails.Guid); } calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).Should().HaveCount(1); calculatedNextBaseline.Runs[0].Results.Where(r => r.BaselineState == BaselineState.New).First().TryGetProperty(SarifLogResultMatcher.ResultMatchingResultPropertyName, out Dictionary <string, string> NewResultProperties).Should().BeTrue(); NewResultProperties.Should().ContainKey("Run"); NewResultProperties["Run"].Should().BeEquivalentTo(currentLog.Runs[0].AutomationDetails.Guid); } }
public void SarifLogResultMatcher_MultipleLogsDuplicateData_WorksAsExpected() { SarifLog current1 = new SarifLog() { Runs = new Run[] { new Run() { Tool = new Tool { Driver = new ToolComponent { Name = "TestTool" } }, Artifacts = new List <Artifact> { new Artifact() { Contents = new ArtifactContent() { Text = "TestFileContents" } } }, Results = new List <Result> { new Result { Message = new Message { Text = "Some testing occurred." }, Locations = new List <Location> { new Location { PhysicalLocation = new PhysicalLocation { ArtifactLocation = new ArtifactLocation { Index = 0 } } } } } } } } }; SarifLog current2 = current1.DeepClone(); SarifLog result = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[0], new SarifLog[] { current1, current2 }).First(); // In first match operation, the file data objects are identical, they should be collapsed into a single array entry result.Runs[0].Artifacts.Should().HaveCount(1); current2 = current1.DeepClone(); // Now we differentiate a files object current2.Runs[0].Artifacts[0].Contents.Text = Guid.NewGuid().ToString(); result = s_preserveOldestPropertyBagMatcher.Match(new SarifLog[0], new SarifLog[] { current1, current2 }).First(); // Both file data objects should be present result.Runs[0].Artifacts.Should().HaveCount(2); // Merged results should each refer to differentiated file data object result.Runs[0].Results[0].Locations[0].PhysicalLocation.ArtifactLocation.Index.Should().Be(0); result.Runs[0].Results[1].Locations[0].PhysicalLocation.ArtifactLocation.Index.Should().Be(1); }