protected override string ConstructTestOutputFromInputResource(string inputResourceName) { PrereleaseCompatibilityTransformer.UpdateToCurrentVersion( GetResourceText(inputResourceName), formatting: Formatting.Indented, out string transformedLog); SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(transformedLog, formatting: Formatting.None, out transformedLog); Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri; string uriString = originalUri.ToString(); // This code rewrites the log persisted URI to match the test environment string currentDirectory = Environment.CurrentDirectory; currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\")); uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory); actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) }; var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing matches actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = originalUri }; return(JsonConvert.SerializeObject(actualLog, Formatting.Indented)); }
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInResult() { Run run = CreateBasicRunForMessageStringLookupTesting(); run.Results.Add( new Result { RuleId = RuleId, RuleIndex = RuleIndex, Message = new Message { Id = UniqueGlobalMessageId } }); run.Results.Add( new Result { RuleId = RuleId, RuleIndex = RuleIndex, Message = new Message { Id = UniqueRuleMessageId } }); run.Results.Add( new Result { RuleId = RuleId, RuleIndex = RuleIndex, Message = new Message { Id = SharedMessageId } }); var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages); visitor.Visit(run); run.Results[0].Message.Text.Should().Be(UniqueGlobalMessageValue); run.Results[1].Message.Text.Should().Be(UniqueRuleMessageValue); // Prefer rule-specific value in the event of a message id collision run.Results[2].Message.Text.Should().Be(SharedKeyRuleMessageValue); }
protected override string ConstructTestOutputFromInputResource(string inputResourceName, object parameter) { SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion( GetResourceText(inputResourceName), formatting: Formatting.Indented, updatedLog: out _); // Some of the tests operate on SARIF files that mention the absolute path of the file // that was "analyzed" (InsertOptionalDataVisitor.txt). That path depends on the repo // root, and so can vary depending on the machine where the tests are run. To avoid // this problem, both the input files and the expected output files contain a fixed // string "REPLACED_AT_TEST_RUNTIME" in place of the directory portion of the path. But some // of the tests must read the contents of the analyzed file (for instance, when the // test requires snippets or file hashes to be inserted). Those test require the actual // path. Therefore we replace the fixed string with the actual path, execute the // visitor, and then restore the fixed string so the actual output can be compared // to the expected output. string enlistmentRoot = GitHelper.Default.GetRepositoryRoot(Environment.CurrentDirectory, useCache: false); if (inputResourceName == "Inputs.CoreTests-Relative.sarif") { Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri; string uriString = originalUri.ToString(); uriString = uriString.Replace(EnlistmentRoot, enlistmentRoot); actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) }; var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing succeeds. actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = originalUri }; // In some of the tests, the visitor added an originalUriBaseId for the repo root. // Adjust that one, too. string repoRootUriBaseId = InsertOptionalDataVisitor.GetUriBaseId(0); if (actualLog.Runs[0].OriginalUriBaseIds.TryGetValue(repoRootUriBaseId, out ArtifactLocation artifactLocation)) { Uri repoRootUri = artifactLocation.Uri; string repoRootString = repoRootUri.ToString(); repoRootString = repoRootString.Replace(enlistmentRoot.Replace(@"\", "/"), EnlistmentRoot); actualLog.Runs[0].OriginalUriBaseIds[repoRootUriBaseId] = new ArtifactLocation { Uri = new Uri(repoRootString, UriKind.Absolute) }; } } else if (inputResourceName == "Inputs.CoreTests-Absolute.sarif") { Uri originalUri = actualLog.Runs[0].Artifacts[0].Location.Uri; string uriString = originalUri.ToString(); uriString = uriString.Replace(EnlistmentRoot, enlistmentRoot); actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) }; var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing matches actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = originalUri }; } else { var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); } // Verify and remove Guids, because they'll vary with every run and can't be compared to a fixed expected output. if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.Guids)) { for (int i = 0; i < actualLog.Runs[0].Results.Count; ++i) { Result result = actualLog.Runs[0].Results[i]; result.Guid.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.Guids flag was set"); result.Guid = null; } } if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.VersionControlDetails)) { VersionControlDetails versionControlDetails = actualLog.Runs[0].VersionControlProvenance[0]; // Verify and replace the mapped directory (enlistment root), because it varies // from machine to machine. var mappedUri = new Uri(enlistmentRoot + @"\", UriKind.Absolute); versionControlDetails.MappedTo.Uri.Should().Be(mappedUri); versionControlDetails.MappedTo.Uri = new Uri($"file:///{EnlistmentRoot}/"); // When OptionallyEmittedData includes any file-related content, the visitor inserts // an artifact that points to the enlistment root. So we have to verify and adjust // that as well. IList<Artifact> artifacts = actualLog.Runs[0].Artifacts; if (artifacts.Count >= 2) { artifacts[1].Location.Uri.Should().Be(enlistmentRoot); artifacts[1].Location.Uri = new Uri($"file:///{EnlistmentRoot}/"); } // Verify and replace the remote repo URI, because it would be different in a fork. var gitHelper = new GitHelper(); Uri remoteUri = gitHelper.GetRemoteUri(enlistmentRoot); versionControlDetails.RepositoryUri.Should().Be(remoteUri); versionControlDetails.RepositoryUri = new Uri("https://REMOTE_URI"); // Verify and remove branch and revision id, because they vary from run to run. versionControlDetails.Branch.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.VersionControlInformation flag was set"); versionControlDetails.Branch = null; versionControlDetails.RevisionId.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.VersionControlInformation flag was set"); versionControlDetails.RevisionId = null; } return JsonConvert.SerializeObject(actualLog, Formatting.Indented); }
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInFix() { Run run = CreateBasicRunForMessageStringLookupTesting(); run.Results.Add( new Result { RuleId = RuleId, RuleIndex = RuleIndex, Message = new Message { Text = "Some testing occurred." }, Fixes = new List <Fix> { new Fix { Description = new Message { Id = UniqueGlobalMessageId } }, new Fix { Description = new Message { Id = UniqueRuleMessageId } }, new Fix { Description = new Message { Id = SharedMessageId } } } }); run.Results.Add( new Result { RuleId = "RuleWithNoRuleDescriptor", Message = new Message { Text = "Some testing occurred." }, Fixes = new List <Fix> { new Fix { Description = new Message { Id = SharedMessageId } } } }); var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages); visitor.Visit(run); run.Results[0].Fixes[0].Description.Text.Should().Be(UniqueGlobalMessageValue); run.Results[0].Fixes[1].Description.Text.Should().Be(UniqueRuleMessageValue); // Prefer rule-specific value in the event of a message id collision run.Results[0].Fixes[2].Description.Text.Should().Be(SharedKeyRuleMessageValue); // Prefer global value in the event of no rules metadata run.Results[1].Fixes[0].Description.Text.Should().Be(SharedKeyGlobalMessageValue); }
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInNotification() { Run run = CreateBasicRunForMessageStringLookupTesting(); IList <Notification> toolNotifications = run.Invocations[0].ToolExecutionNotifications; IList <Notification> configurationNotifications = run.Invocations[0].ToolConfigurationNotifications; // Shared message id with no overriding rule id toolNotifications.Add( new Notification { Descriptor = new ReportingDescriptorReference { Id = NotificationId }, Message = new Message { Id = SharedMessageId } }); configurationNotifications.Add(toolNotifications[0]); // Notification that refers to a rule that does not contain a message with // the same id as the specified notification id.In this case it is no surprise // that the message comes from the global string table. toolNotifications.Add( new Notification { Descriptor = new ReportingDescriptorReference { Id = NotificationId }, AssociatedRule = new ReportingDescriptorReference { Index = RuleIndex }, Message = new Message { Id = UniqueGlobalMessageId } }); configurationNotifications.Add(toolNotifications[1]); // Notification that refers to a rule that contains a message with the same // id as the specified notification message id. The message should still be // retrieved from the global strings table. toolNotifications.Add( new Notification { Descriptor = new ReportingDescriptorReference { Id = NotificationId }, AssociatedRule = new ReportingDescriptorReference { Index = RuleIndex }, Message = new Message { Id = SharedMessageId } }); configurationNotifications.Add(toolNotifications[2]); var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages); visitor.Visit(run); toolNotifications[0].Message.Text.Should().Be(SharedKeyGlobalMessageValue); configurationNotifications[0].Message.Text.Should().Be(SharedKeyGlobalMessageValue); toolNotifications[1].Message.Text.Should().Be(UniqueGlobalMessageValue); configurationNotifications[1].Message.Text.Should().Be(UniqueGlobalMessageValue); toolNotifications[2].Message.Text.Should().Be(SharedKeyGlobalMessageValue); configurationNotifications[2].Message.Text.Should().Be(SharedKeyGlobalMessageValue); }
private void RunTest(string testDirectory, string inputFileName, OptionallyEmittedData optionallyEmittedData) { var sb = new StringBuilder(); string optionsNameSuffix = "_" + NormalizeOptionallyEmittedDataToString(optionallyEmittedData); string expectedFileName = inputFileName + optionsNameSuffix + ".sarif"; string actualFileName = @"Actual\" + inputFileName + optionsNameSuffix + ".sarif"; inputFileName = inputFileName + ".sarif"; expectedFileName = Path.Combine(testDirectory, expectedFileName); actualFileName = Path.Combine(testDirectory, actualFileName); inputFileName = Path.Combine(testDirectory, inputFileName); string actualDirectory = Path.GetDirectoryName(actualFileName); if (!Directory.Exists(actualDirectory)) { Directory.CreateDirectory(actualDirectory); } File.Exists(inputFileName).Should().BeTrue(); SarifLog actualLog; JsonSerializerSettings settings = new JsonSerializerSettings() { ContractResolver = SarifContractResolver.Instance, Formatting = Formatting.Indented }; try { actualLog = JsonConvert.DeserializeObject <SarifLog>(File.ReadAllText(inputFileName), settings); Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"]; string uriString = originalUri.ToString(); // This code rewrites the log persisted URI to match the test environment string currentDirectory = Environment.CurrentDirectory; currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\")); uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory); actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new Uri(uriString, UriKind.Absolute); var visitor = new InsertOptionalDataVisitor(optionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing matches actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = originalUri; } catch (Exception ex) { sb.AppendFormat(CultureInfo.InvariantCulture, "Unhandled exception processing input '{0}' with the following options: '{1}'.\r\n", inputFileName, optionallyEmittedData); sb.AppendLine(ex.ToString()); ValidateResults(sb.ToString()); return; } string expectedSarif = File.Exists(expectedFileName) ? File.ReadAllText(expectedFileName) : null; string actualSarif = JsonConvert.SerializeObject(actualLog, settings); if (!AreEquivalentSarifLogs(actualSarif, expectedSarif)) { if (s_rebaseline) { // We rewrite to test output directory. This allows subsequent tests to // pass without requiring a rebuild that recopies SARIF test files File.WriteAllText(expectedFileName, actualSarif); string subdirectory = Path.GetFileName(testDirectory); string productTestDirectory = GetProductTestDataDirectory(subdirectory); expectedFileName = Path.GetFileName(expectedFileName); expectedFileName = Path.Combine(productTestDirectory, expectedFileName); // We also rewrite the checked in test baselines File.WriteAllText(expectedFileName, actualSarif); } else { File.WriteAllText(actualFileName, actualSarif); string errorMessage = "Expanding optional data for input '{0}' produced unexpected results for the following options: '{1}'."; sb.AppendLine(string.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName, optionallyEmittedData)); sb.AppendLine("Check individual differences with:"); sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName) + Environment.NewLine); sb.AppendLine("To compare all difference for this test suite:"); sb.AppendLine(GenerateDiffCommand(Path.GetDirectoryName(expectedFileName), Path.GetDirectoryName(actualFileName)) + Environment.NewLine); } } // Add this check to prevent us from unexpectedly checking in this static with the wrong value s_rebaseline.Should().BeFalse(); ValidateResults(sb.ToString()); }
protected override string ConstructTestOutputFromInputResource(string inputResourceName, object parameter) { PrereleaseCompatibilityTransformer.UpdateToCurrentVersion( GetResourceText(inputResourceName), formatting: Formatting.Indented, out string transformedLog); SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(transformedLog, formatting: Formatting.None, out transformedLog); // For CoreTests only - this code rewrites the log persisted URI to match the test environment if (inputResourceName == "Inputs.CoreTests-Relative.sarif") { Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri; string uriString = originalUri.ToString(); string currentDirectory = Environment.CurrentDirectory; currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\")); uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory); actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) }; var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing matches actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = originalUri }; } else if (inputResourceName == "Inputs.CoreTests-Absolute.sarif") { Uri originalUri = actualLog.Runs[0].Artifacts[0].Location.Uri; string uriString = originalUri.ToString(); string currentDirectory = Environment.CurrentDirectory; currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\")); uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory); actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) }; var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); // Restore the remanufactured URI so that file diffing matches actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = originalUri }; } else { var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData); visitor.Visit(actualLog.Runs[0]); } // Verify and remove Guids, because they'll vary with every run and can't be compared to a fixed expected output if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.Guids)) { for (int i = 0; i < actualLog.Runs[0].Results.Count; ++i) { Result result = actualLog.Runs[0].Results[i]; if (string.IsNullOrEmpty(result.Guid)) { Assert.True(false, $"Results[{i}] had no Guid assigned, but OptionallyEmittedData.Guids flag was set."); } result.Guid = null; } } return(JsonConvert.SerializeObject(actualLog, Formatting.Indented)); }