private void CompareActualToExpected(
IList <string> inputResourceNames,
IDictionary <string, string> expectedOutputResourceNameDictionary,
IDictionary <string, string> expectedSarifTextDictionary,
IDictionary <string, string> actualSarifTextDictionary)
{
if (inputResourceNames.Count == 0)
{
throw new ArgumentException("No input resources were specified", nameof(inputResourceNames));
}
if (expectedOutputResourceNameDictionary.Count == 0)
{
throw new ArgumentException("No expected output resources were specified", nameof(expectedOutputResourceNameDictionary));
}
if (expectedSarifTextDictionary.Count != expectedOutputResourceNameDictionary.Count)
{
throw new ArgumentException($"The number of expected output files ({expectedSarifTextDictionary.Count}) does not match the number of expected output resources {expectedOutputResourceNameDictionary.Count}");
}
if (expectedSarifTextDictionary.Count != actualSarifTextDictionary.Count)
{
throw new ArgumentException($"The number of actual output files ({actualSarifTextDictionary.Count}) does not match the number of expected output files {expectedSarifTextDictionary.Count}");
}
bool passed = true;
if (RebaselineExpectedResults)
{
passed = false;
}
else
{
// Reify the list of keys because we're going to modify the dictionary in place.
List <string> keys = expectedSarifTextDictionary.Keys.ToList();
foreach (string key in keys)
{
if (_testProducesSarifCurrentVersion)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(expectedSarifTextDictionary[key], Formatting.Indented, out string transformedSarifText);
expectedSarifTextDictionary[key] = transformedSarifText;
passed &= AreEquivalent <SarifLog>(actualSarifTextDictionary[key], expectedSarifTextDictionary[key]);
}
else
{
passed &= AreEquivalent <SarifLogVersionOne>(actualSarifTextDictionary[key], expectedSarifTextDictionary[key], SarifContractResolverVersionOne.Instance);
}
}
}
if (!passed)
{
string errorMessage = string.Format(@"there should be no unexpected diffs detected comparing actual results to '{0}'.", string.Join(", ", inputResourceNames));
var sb = new StringBuilder(errorMessage);
if (!Utilities.RunningInAppVeyor)
{
string expectedRootDirectory = null;
string actualRootDirectory = null;
bool firstKey = true;
foreach (string key in expectedOutputResourceNameDictionary.Keys)
{
string expectedFilePath = GetOutputFilePath("ExpectedOutputs", expectedOutputResourceNameDictionary[key]);
string actualFilePath = GetOutputFilePath("ActualOutputs", expectedOutputResourceNameDictionary[key]);
if (firstKey)
{
expectedRootDirectory = Path.GetDirectoryName(expectedFilePath);
actualRootDirectory = Path.GetDirectoryName(actualFilePath);
Directory.CreateDirectory(expectedRootDirectory);
Directory.CreateDirectory(actualRootDirectory);
firstKey = false;
}
File.WriteAllText(expectedFilePath, expectedSarifTextDictionary[key]);
File.WriteAllText(actualFilePath, actualSarifTextDictionary[key]);
}
sb.AppendLine("To compare all difference for this test suite:");
sb.AppendLine(GenerateDiffCommand(TypeUnderTest, expectedRootDirectory, actualRootDirectory) + Environment.NewLine);
if (RebaselineExpectedResults)
{
string intermediateFolder = !string.IsNullOrEmpty(IntermediateTestFolder) ? IntermediateTestFolder + @"\" : string.Empty;
string testDirectory = Path.Combine(GetProductTestDataDirectory(TestBinaryName, intermediateFolder + TypeUnderTest), "ExpectedOutputs");
Directory.CreateDirectory(testDirectory);
// We retrieve all test strings from embedded resources. To rebaseline, we need to
// compute the enlistment location from which these resources are compiled.
foreach (string key in expectedOutputResourceNameDictionary.Keys)
{
string expectedFilePath = Path.Combine(testDirectory, expectedOutputResourceNameDictionary[key]);
File.WriteAllText(expectedFilePath, actualSarifTextDictionary[key]);
}
}
}
if (!RebaselineExpectedResults)
{
ValidateResults(sb.ToString());
}
}
RebaselineExpectedResults.Should().BeFalse();
}
protected virtual void RunTest(string inputResourceName, string expectedOutputResourceName = null)
{
expectedOutputResourceName = expectedOutputResourceName ?? inputResourceName;
// When retrieving constructed test content, we pass the resourceName as the test
// specified it. When constructing actual and expected file names from this data,
// however, we will ensure that the name has the ".sarif" extension. We do this
// for test classes such as the Fortify converter that operate again non-SARIF inputs.
string actualSarifText = ConstructTestOutputFromInputResource("Inputs." + inputResourceName);
expectedOutputResourceName = Path.GetFileNameWithoutExtension(expectedOutputResourceName) + ".sarif";
var sb = new StringBuilder();
string expectedSarifText = GetResourceText("ExpectedOutputs." + expectedOutputResourceName);
bool passed = false;
if (!RebaselineExpectedResults)
{
if (_testProducesSarifCurrentVersion)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(expectedSarifText, Formatting.Indented, out expectedSarifText);
passed = AreEquivalent <SarifLog>(actualSarifText, expectedSarifText);
}
else
{
passed = AreEquivalent <SarifLogVersionOne>(actualSarifText, expectedSarifText, SarifContractResolverVersionOne.Instance);
}
}
if (!passed)
{
string errorMessage = string.Format(@"there should be no unexpected diffs detected comparing actual results to '{0}'.", inputResourceName);
sb.AppendLine(errorMessage);
if (!Utilities.RunningInAppVeyor)
{
string expectedFilePath = GetOutputFilePath("ExpectedOutputs", expectedOutputResourceName);
string actualFilePath = GetOutputFilePath("ActualOutputs", expectedOutputResourceName);
string expectedRootDirectory = Path.GetDirectoryName(expectedFilePath);
string actualRootDirectory = Path.GetDirectoryName(actualFilePath);
Directory.CreateDirectory(expectedRootDirectory);
Directory.CreateDirectory(actualRootDirectory);
File.WriteAllText(expectedFilePath, expectedSarifText);
File.WriteAllText(actualFilePath, actualSarifText);
sb.AppendLine("To compare all difference for this test suite:");
sb.AppendLine(GenerateDiffCommand(TypeUnderTest, Path.GetDirectoryName(expectedFilePath), Path.GetDirectoryName(actualFilePath)) + Environment.NewLine);
if (RebaselineExpectedResults)
{
string intermediateFolder = !string.IsNullOrEmpty(IntermediateTestFolder) ? IntermediateTestFolder + @"\" : String.Empty;
string testDirectory = Path.Combine(GetProductTestDataDirectory(TestBinaryName, intermediateFolder + TypeUnderTest), "ExpectedOutputs");
Directory.CreateDirectory(testDirectory);
// We retrieve all test strings from embedded resources. To rebaseline, we need to
// compute the enlistment location from which these resources are compiled.
expectedFilePath = Path.Combine(testDirectory, expectedOutputResourceName);
File.WriteAllText(expectedFilePath, actualSarifText);
}
}
if (!RebaselineExpectedResults)
{
ValidateResults(sb.ToString());
}
}
RebaselineExpectedResults.Should().BeFalse();
}
