protected override string ConstructTestOutputFromInputResource(string inputResourceName)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
GetResourceText(inputResourceName),
formatting: Formatting.Indented,
out string transformedLog);
SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(transformedLog, formatting: Formatting.None, out transformedLog);
Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri;
string uriString = originalUri.ToString();
// This code rewrites the log persisted URI to match the test environment
string currentDirectory = Environment.CurrentDirectory;
currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\"));
uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory);
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation {
Uri = new Uri(uriString, UriKind.Absolute)
};
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing matches
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation {
Uri = originalUri
};
return(JsonConvert.SerializeObject(actualLog, Formatting.Indented));
}
public void InsertOptionalDataVisitor_CanVisitIndividualResultsInASuppliedRun()
{
const string TestFileContents =
@"One
Two
Three";
const string ExpectedSnippet = "Two";
using (var tempFile = new TempFile(".txt"))
{
string tempFilePath = tempFile.Name;
string tempFileName = Path.GetFileName(tempFilePath);
string tempFileDirectory = Path.GetDirectoryName(tempFilePath);
File.WriteAllText(tempFilePath, TestFileContents);
var run = new Run
{
OriginalUriBaseIds = new Dictionary <string, ArtifactLocation>
{
[TestData.TestRootBaseId] = new ArtifactLocation
{
Uri = new Uri(tempFileDirectory, UriKind.Absolute)
}
},
Results = new List <Result>
{
new Result
{
Locations = new List <Location>
{
new Location
{
PhysicalLocation = new PhysicalLocation
{
ArtifactLocation = new ArtifactLocation
{
Uri = new Uri(tempFileName, UriKind.Relative),
UriBaseId = TestData.TestRootBaseId
},
Region = new Region
{
StartLine = 2
}
}
}
}
}
}
};
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.RegionSnippets, run);
visitor.VisitResult(run.Results[0]);
run.Results[0].Locations[0].PhysicalLocation.Region.Snippet.Text.Should().Be(ExpectedSnippet);
}
}
public void InsertOptionalDataVisitorTests_ResolvesOriginalUriBaseIds()
{
string inputFileName = "InsertOptionalDataVisitor.txt";
string testDirectory = GetTestDirectory("InsertOptionalDataVisitor") + @"\";
string uriBaseId = "TEST_DIR";
string fileKey = "#" + uriBaseId + "#" + inputFileName;
IDictionary <string, ArtifactLocation> originalUriBaseIds = new Dictionary <string, ArtifactLocation> {
{ uriBaseId, new ArtifactLocation {
Uri = new Uri(testDirectory, UriKind.Absolute)
} }
};
Run run = new Run()
{
DefaultEncoding = "UTF-8",
OriginalUriBaseIds = null,
Results = new[]
{
new Result()
{
Locations = new []
{
new Location
{
PhysicalLocation = new PhysicalLocation
{
ArtifactLocation = new ArtifactLocation
{
Uri = new Uri(inputFileName, UriKind.Relative),
UriBaseId = uriBaseId
}
}
}
}
}
}
};
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.TextFiles);
visitor.VisitRun(run);
run.OriginalUriBaseIds.Should().BeNull();
run.Artifacts.Count.Should().Be(1);
run.Artifacts[0].Contents.Should().BeNull();
visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.TextFiles, originalUriBaseIds);
visitor.VisitRun(run);
run.OriginalUriBaseIds.Should().Equal(originalUriBaseIds);
run.Artifacts[0].Contents.Text.Should().Be(File.ReadAllText(Path.Combine(testDirectory, inputFileName)));
}
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInResult()
{
Run run = CreateBasicRunForMessageStringLookupTesting();
run.Results.Add(
new Result
{
RuleId = RuleId,
RuleIndex = RuleIndex,
Message = new Message
{
Id = UniqueGlobalMessageId
}
});
run.Results.Add(
new Result
{
RuleId = RuleId,
RuleIndex = RuleIndex,
Message = new Message
{
Id = UniqueRuleMessageId
}
});
run.Results.Add(
new Result
{
RuleId = RuleId,
RuleIndex = RuleIndex,
Message = new Message
{
Id = SharedMessageId
}
});
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages);
visitor.Visit(run);
run.Results[0].Message.Text.Should().Be(UniqueGlobalMessageValue);
run.Results[1].Message.Text.Should().Be(UniqueRuleMessageValue);
// Prefer rule-specific value in the event of a message id collision
run.Results[2].Message.Text.Should().Be(SharedKeyRuleMessageValue);
}
public void InsertOptionalDataVisitorTests_VisitDictionaryValueNullChecked_ValidEncoding()
{
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.OverwriteExistingData | OptionallyEmittedData.TextFiles);
visitor.VisitRun(new Run()); // VisitDictionaryValueNullChecked requires a non-null run
string uriString = "file:///C:/src/foo.cs";
FileData fileData = FileData.Create(new Uri(uriString), mimeType: "text/x-csharp", encoding: Encoding.UTF8);
fileData.Length = 12345;
FileData outputFileData = visitor.VisitDictionaryValueNullChecked(uriString, fileData);
outputFileData.MimeType.Should().Be(fileData.MimeType);
outputFileData.Encoding.Should().Be(fileData.Encoding);
outputFileData.Length.Should().Be(fileData.Length);
}
protected override string ConstructTestOutputFromInputResource(string inputResourceName, object parameter)
{
SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
GetResourceText(inputResourceName),
formatting: Formatting.Indented,
updatedLog: out _);
// Some of the tests operate on SARIF files that mention the absolute path of the file
// that was "analyzed" (InsertOptionalDataVisitor.txt). That path depends on the repo
// root, and so can vary depending on the machine where the tests are run. To avoid
// this problem, both the input files and the expected output files contain a fixed
// string "REPLACED_AT_TEST_RUNTIME" in place of the directory portion of the path. But some
// of the tests must read the contents of the analyzed file (for instance, when the
// test requires snippets or file hashes to be inserted). Those test require the actual
// path. Therefore we replace the fixed string with the actual path, execute the
// visitor, and then restore the fixed string so the actual output can be compared
// to the expected output.
string enlistmentRoot = GitHelper.Default.GetRepositoryRoot(Environment.CurrentDirectory, useCache: false);
if (inputResourceName == "Inputs.CoreTests-Relative.sarif")
{
Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri;
string uriString = originalUri.ToString();
uriString = uriString.Replace(EnlistmentRoot, enlistmentRoot);
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) };
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing succeeds.
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation { Uri = originalUri };
// In some of the tests, the visitor added an originalUriBaseId for the repo root.
// Adjust that one, too.
string repoRootUriBaseId = InsertOptionalDataVisitor.GetUriBaseId(0);
if (actualLog.Runs[0].OriginalUriBaseIds.TryGetValue(repoRootUriBaseId, out ArtifactLocation artifactLocation))
{
Uri repoRootUri = artifactLocation.Uri;
string repoRootString = repoRootUri.ToString();
repoRootString = repoRootString.Replace(enlistmentRoot.Replace(@"\", "/"), EnlistmentRoot);
actualLog.Runs[0].OriginalUriBaseIds[repoRootUriBaseId] = new ArtifactLocation { Uri = new Uri(repoRootString, UriKind.Absolute) };
}
}
else if (inputResourceName == "Inputs.CoreTests-Absolute.sarif")
{
Uri originalUri = actualLog.Runs[0].Artifacts[0].Location.Uri;
string uriString = originalUri.ToString();
uriString = uriString.Replace(EnlistmentRoot, enlistmentRoot);
actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = new Uri(uriString, UriKind.Absolute) };
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing matches
actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation { Uri = originalUri };
}
else
{
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
}
// Verify and remove Guids, because they'll vary with every run and can't be compared to a fixed expected output.
if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.Guids))
{
for (int i = 0; i < actualLog.Runs[0].Results.Count; ++i)
{
Result result = actualLog.Runs[0].Results[i];
result.Guid.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.Guids flag was set");
result.Guid = null;
}
}
if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.VersionControlDetails))
{
VersionControlDetails versionControlDetails = actualLog.Runs[0].VersionControlProvenance[0];
// Verify and replace the mapped directory (enlistment root), because it varies
// from machine to machine.
var mappedUri = new Uri(enlistmentRoot + @"\", UriKind.Absolute);
versionControlDetails.MappedTo.Uri.Should().Be(mappedUri);
versionControlDetails.MappedTo.Uri = new Uri($"file:///{EnlistmentRoot}/");
// When OptionallyEmittedData includes any file-related content, the visitor inserts
// an artifact that points to the enlistment root. So we have to verify and adjust
// that as well.
IList<Artifact> artifacts = actualLog.Runs[0].Artifacts;
if (artifacts.Count >= 2)
{
artifacts[1].Location.Uri.Should().Be(enlistmentRoot);
artifacts[1].Location.Uri = new Uri($"file:///{EnlistmentRoot}/");
}
// Verify and replace the remote repo URI, because it would be different in a fork.
var gitHelper = new GitHelper();
Uri remoteUri = gitHelper.GetRemoteUri(enlistmentRoot);
versionControlDetails.RepositoryUri.Should().Be(remoteUri);
versionControlDetails.RepositoryUri = new Uri("https://REMOTE_URI");
// Verify and remove branch and revision id, because they vary from run to run.
versionControlDetails.Branch.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.VersionControlInformation flag was set");
versionControlDetails.Branch = null;
versionControlDetails.RevisionId.Should().NotBeNullOrEmpty(because: "OptionallyEmittedData.VersionControlInformation flag was set");
versionControlDetails.RevisionId = null;
}
return JsonConvert.SerializeObject(actualLog, Formatting.Indented);
}
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInFix()
{
Run run = CreateBasicRunForMessageStringLookupTesting();
run.Results.Add(
new Result
{
RuleId = RuleId,
RuleIndex = RuleIndex,
Message = new Message
{
Text = "Some testing occurred."
},
Fixes = new List <Fix>
{
new Fix
{
Description = new Message
{
Id = UniqueGlobalMessageId
}
},
new Fix
{
Description = new Message
{
Id = UniqueRuleMessageId
}
},
new Fix
{
Description = new Message
{
Id = SharedMessageId
}
}
}
});
run.Results.Add(
new Result
{
RuleId = "RuleWithNoRuleDescriptor",
Message = new Message
{
Text = "Some testing occurred."
},
Fixes = new List <Fix>
{
new Fix
{
Description = new Message
{
Id = SharedMessageId
}
}
}
});
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages);
visitor.Visit(run);
run.Results[0].Fixes[0].Description.Text.Should().Be(UniqueGlobalMessageValue);
run.Results[0].Fixes[1].Description.Text.Should().Be(UniqueRuleMessageValue);
// Prefer rule-specific value in the event of a message id collision
run.Results[0].Fixes[2].Description.Text.Should().Be(SharedKeyRuleMessageValue);
// Prefer global value in the event of no rules metadata
run.Results[1].Fixes[0].Description.Text.Should().Be(SharedKeyGlobalMessageValue);
}
public void InsertOptionalDataVisitorTests_FlattensMessageStringsInNotification()
{
Run run = CreateBasicRunForMessageStringLookupTesting();
IList <Notification> toolNotifications = run.Invocations[0].ToolExecutionNotifications;
IList <Notification> configurationNotifications = run.Invocations[0].ToolConfigurationNotifications;
// Shared message id with no overriding rule id
toolNotifications.Add(
new Notification
{
Descriptor = new ReportingDescriptorReference
{
Id = NotificationId
},
Message = new Message {
Id = SharedMessageId
}
});
configurationNotifications.Add(toolNotifications[0]);
// Notification that refers to a rule that does not contain a message with
// the same id as the specified notification id.In this case it is no surprise
// that the message comes from the global string table.
toolNotifications.Add(
new Notification
{
Descriptor = new ReportingDescriptorReference
{
Id = NotificationId
},
AssociatedRule = new ReportingDescriptorReference
{
Index = RuleIndex
},
Message = new Message {
Id = UniqueGlobalMessageId
}
});
configurationNotifications.Add(toolNotifications[1]);
// Notification that refers to a rule that contains a message with the same
// id as the specified notification message id. The message should still be
// retrieved from the global strings table.
toolNotifications.Add(
new Notification
{
Descriptor = new ReportingDescriptorReference
{
Id = NotificationId
},
AssociatedRule = new ReportingDescriptorReference
{
Index = RuleIndex
},
Message = new Message {
Id = SharedMessageId
}
});
configurationNotifications.Add(toolNotifications[2]);
var visitor = new InsertOptionalDataVisitor(OptionallyEmittedData.FlattenedMessages);
visitor.Visit(run);
toolNotifications[0].Message.Text.Should().Be(SharedKeyGlobalMessageValue);
configurationNotifications[0].Message.Text.Should().Be(SharedKeyGlobalMessageValue);
toolNotifications[1].Message.Text.Should().Be(UniqueGlobalMessageValue);
configurationNotifications[1].Message.Text.Should().Be(UniqueGlobalMessageValue);
toolNotifications[2].Message.Text.Should().Be(SharedKeyGlobalMessageValue);
configurationNotifications[2].Message.Text.Should().Be(SharedKeyGlobalMessageValue);
}
private void RunTest(string testDirectory, string inputFileName, OptionallyEmittedData optionallyEmittedData)
{
var sb = new StringBuilder();
string optionsNameSuffix = "_" + NormalizeOptionallyEmittedDataToString(optionallyEmittedData);
string expectedFileName = inputFileName + optionsNameSuffix + ".sarif";
string actualFileName = @"Actual\" + inputFileName + optionsNameSuffix + ".sarif";
inputFileName = inputFileName + ".sarif";
expectedFileName = Path.Combine(testDirectory, expectedFileName);
actualFileName = Path.Combine(testDirectory, actualFileName);
inputFileName = Path.Combine(testDirectory, inputFileName);
string actualDirectory = Path.GetDirectoryName(actualFileName);
if (!Directory.Exists(actualDirectory))
{
Directory.CreateDirectory(actualDirectory);
}
File.Exists(inputFileName).Should().BeTrue();
SarifLog actualLog;
JsonSerializerSettings settings = new JsonSerializerSettings()
{
ContractResolver = SarifContractResolver.Instance,
Formatting = Formatting.Indented
};
try
{
actualLog = JsonConvert.DeserializeObject <SarifLog>(File.ReadAllText(inputFileName), settings);
Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"];
string uriString = originalUri.ToString();
// This code rewrites the log persisted URI to match the test environment
string currentDirectory = Environment.CurrentDirectory;
currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\"));
uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory);
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new Uri(uriString, UriKind.Absolute);
var visitor = new InsertOptionalDataVisitor(optionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing matches
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = originalUri;
}
catch (Exception ex)
{
sb.AppendFormat(CultureInfo.InvariantCulture, "Unhandled exception processing input '{0}' with the following options: '{1}'.\r\n", inputFileName, optionallyEmittedData);
sb.AppendLine(ex.ToString());
ValidateResults(sb.ToString());
return;
}
string expectedSarif = File.Exists(expectedFileName) ? File.ReadAllText(expectedFileName) : null;
string actualSarif = JsonConvert.SerializeObject(actualLog, settings);
if (!AreEquivalentSarifLogs(actualSarif, expectedSarif))
{
if (s_rebaseline)
{
// We rewrite to test output directory. This allows subsequent tests to
// pass without requiring a rebuild that recopies SARIF test files
File.WriteAllText(expectedFileName, actualSarif);
string subdirectory = Path.GetFileName(testDirectory);
string productTestDirectory = GetProductTestDataDirectory(subdirectory);
expectedFileName = Path.GetFileName(expectedFileName);
expectedFileName = Path.Combine(productTestDirectory, expectedFileName);
// We also rewrite the checked in test baselines
File.WriteAllText(expectedFileName, actualSarif);
}
else
{
File.WriteAllText(actualFileName, actualSarif);
string errorMessage = "Expanding optional data for input '{0}' produced unexpected results for the following options: '{1}'.";
sb.AppendLine(string.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName, optionallyEmittedData));
sb.AppendLine("Check individual differences with:");
sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName) + Environment.NewLine);
sb.AppendLine("To compare all difference for this test suite:");
sb.AppendLine(GenerateDiffCommand(Path.GetDirectoryName(expectedFileName), Path.GetDirectoryName(actualFileName)) + Environment.NewLine);
}
}
// Add this check to prevent us from unexpectedly checking in this static with the wrong value
s_rebaseline.Should().BeFalse();
ValidateResults(sb.ToString());
}
protected override string ConstructTestOutputFromInputResource(string inputResourceName, object parameter)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
GetResourceText(inputResourceName),
formatting: Formatting.Indented,
out string transformedLog);
SarifLog actualLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(transformedLog, formatting: Formatting.None, out transformedLog);
// For CoreTests only - this code rewrites the log persisted URI to match the test environment
if (inputResourceName == "Inputs.CoreTests-Relative.sarif")
{
Uri originalUri = actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"].Uri;
string uriString = originalUri.ToString();
string currentDirectory = Environment.CurrentDirectory;
currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\"));
uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory);
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation {
Uri = new Uri(uriString, UriKind.Absolute)
};
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing matches
actualLog.Runs[0].OriginalUriBaseIds["TESTROOT"] = new ArtifactLocation {
Uri = originalUri
};
}
else if (inputResourceName == "Inputs.CoreTests-Absolute.sarif")
{
Uri originalUri = actualLog.Runs[0].Artifacts[0].Location.Uri;
string uriString = originalUri.ToString();
string currentDirectory = Environment.CurrentDirectory;
currentDirectory = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\bld\"));
uriString = uriString.Replace("REPLACED_AT_TEST_RUNTIME", currentDirectory);
actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation {
Uri = new Uri(uriString, UriKind.Absolute)
};
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
// Restore the remanufactured URI so that file diffing matches
actualLog.Runs[0].Artifacts[0].Location = new ArtifactLocation {
Uri = originalUri
};
}
else
{
var visitor = new InsertOptionalDataVisitor(_currentOptionallyEmittedData);
visitor.Visit(actualLog.Runs[0]);
}
// Verify and remove Guids, because they'll vary with every run and can't be compared to a fixed expected output
if (_currentOptionallyEmittedData.HasFlag(OptionallyEmittedData.Guids))
{
for (int i = 0; i < actualLog.Runs[0].Results.Count; ++i)
{
Result result = actualLog.Runs[0].Results[i];
if (string.IsNullOrEmpty(result.Guid))
{
Assert.True(false, $"Results[{i}] had no Guid assigned, but OptionallyEmittedData.Guids flag was set.");
}
result.Guid = null;
}
}
return(JsonConvert.SerializeObject(actualLog, Formatting.Indented));
}
