private void RunRules(StringBuilder sb, string inputFileName)
{
string fileName = Path.GetFileName(inputFileName);
string actualDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
string expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");
if (!Directory.Exists(actualDirectory))
{
Directory.CreateDirectory(actualDirectory);
}
string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
string actualFileName = Path.Combine(actualDirectory, fileName + ".sarif");
AnalyzeCommand command = new AnalyzeCommand();
AnalyzeOptions options = new AnalyzeOptions();
options.TargetFileSpecifiers = new string[] { inputFileName };
options.OutputFilePath = actualFileName;
options.Verbose = true;
options.Recurse = false;
options.ConfigurationFilePath = "default";
int result = command.Run(options);
// Note that we don't ensure a success code. That is because we
// are running end-to-end tests for valid and invalid files
JsonSerializerSettings settings = new JsonSerializerSettings()
{
ContractResolver = SarifContractResolver.Instance,
Formatting = Formatting.Indented
};
string expectedText = File.ReadAllText(expectedFileName);
string actualText = File.ReadAllText(actualFileName);
// Make sure we can successfully deserialize what was just generated
ResultLog expectedLog = JsonConvert.DeserializeObject <ResultLog>(expectedText, settings);
ResultLog actualLog = JsonConvert.DeserializeObject <ResultLog>(actualText, settings);
var visitor = new ResultDiffingVisitor(expectedLog);
if (!visitor.Diff(actualLog.RunLogs[0].Results))
{
string errorMessage = "The output of the tool did not match for input {0}.";
sb.AppendLine(String.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
sb.AppendLine("Check differences with:");
sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName));
}
}
private void RunRules(StringBuilder sb, string inputFileName)
{
string fileName = Path.GetFileName(inputFileName);
string actualDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
string expectedDirectory;
if (PlatformSpecificHelpers.RunningOnWindows())
{
expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");
}
else
{
expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "NonWindowsExpected");
}
if (!Directory.Exists(actualDirectory))
{
Directory.CreateDirectory(actualDirectory);
}
string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
string actualFileName = Path.Combine(actualDirectory, fileName + ".sarif");
var command = new AnalyzeCommand();
var options = new AnalyzeOptions
{
Force = true,
Verbose = true,
Recurse = false,
PrettyPrint = true,
ComputeFileHashes = true,
OutputFilePath = actualFileName,
ConfigurationFilePath = "default",
SarifOutputVersion = SarifVersion.Current,
TargetFileSpecifiers = new string[] { inputFileName }
};
int result = command.Run(options);
// Note that we don't ensure a success code. That is because we
// are running end-to-end tests for valid and invalid files
var settings = new JsonSerializerSettings()
{
Formatting = Newtonsoft.Json.Formatting.Indented
};
string expectedText = File.ReadAllText(expectedFileName);
string actualText = File.ReadAllText(actualFileName);
// Replace repository root absolute path with Z:\ for machine and enlistment independence
string repoRoot = Path.GetFullPath(Path.Combine(actualDirectory, "..", "..", "..", ".."));
actualText = actualText.Replace(repoRoot.Replace(@"\", @"\\"), @"Z:");
actualText = actualText.Replace(repoRoot.Replace(@"\", @"/"), @"Z:");
// Remove stack traces as they can change due to inlining differences by configuration and runtime.
actualText = Regex.Replace(actualText, @"\\r\\n at [^""]+", "");
actualText = actualText.Replace(@"""Sarif""", @"""BinSkim""");
actualText = actualText.Replace(@" ""fileVersion"": ""15.0.0""," + Environment.NewLine, string.Empty);
actualText = Regex.Replace(actualText, @"\s*""fullName""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""semanticVersion""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""sarifLoggerVersion""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""dottedQuadFileVersion""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""Comments""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""CompanyName""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""ProductName""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""time""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""endTimeUtc""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""startTimeUtc""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @"\s*""processId""[^\n]+?\n", Environment.NewLine);
actualText = Regex.Replace(actualText, @" ""id""[^,]+,\s+""tool""", @" ""tool""", RegexOptions.Multiline);
// Write back the normalized actual text so that the diff command given on failure shows what was actually compared.
Encoding utf8encoding = new UTF8Encoding(true);
using (var textWriter = new StreamWriter(actualFileName, false, utf8encoding))
{
textWriter.Write(actualText);
}
// Make sure we can successfully deserialize what was just generated
SarifLog expectedLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
expectedText,
settings.Formatting,
out expectedText);
SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(actualText, settings);
var visitor = new ResultDiffingVisitor(expectedLog);
if (!visitor.Diff(actualLog.Runs[0].Results))
{
string errorMessage = "The output of the tool did not match for input {0}.";
sb.AppendLine(string.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
sb.AppendLine("Check differences with:");
sb.AppendLine(this.GenerateDiffCommand(expectedFileName, actualFileName));
}
}
private void RunRules(StringBuilder sb, string inputFileName)
{
string fileName = Path.GetFileName(inputFileName);
string actualDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
string expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");
if (!Directory.Exists(actualDirectory))
{
Directory.CreateDirectory(actualDirectory);
}
string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
string actualFileName = Path.Combine(actualDirectory, fileName + ".sarif");
AnalyzeCommand command = new AnalyzeCommand();
AnalyzeOptions options = new AnalyzeOptions();
options.TargetFileSpecifiers = new string[] { inputFileName };
options.OutputFilePath = actualFileName;
options.Verbose = true;
options.Recurse = false;
options.ComputeFileHashes = true;
options.ConfigurationFilePath = "default";
int result = command.Run(options);
// Note that we don't ensure a success code. That is because we
// are running end-to-end tests for valid and invalid files
JsonSerializerSettings settings = new JsonSerializerSettings()
{
ContractResolver = SarifContractResolver.Instance,
Formatting = Formatting.Indented
};
string expectedText = File.ReadAllText(expectedFileName);
string actualText = File.ReadAllText(actualFileName);
// Replace repository root absolute path with Z:\ for machine and enlistment independence
string repoRoot = Path.GetFullPath(Path.Combine(actualDirectory, "..", "..", "..", ".."));
actualText = actualText.Replace(repoRoot.Replace(@"\", @"\\"), @"Z:");
actualText = actualText.Replace(repoRoot.Replace(@"\", @"/"), @"Z:");
// Remove stack traces as they can change due to inlining differences by configuration and runtime.
actualText = Regex.Replace(actualText, @"\\r\\n at [^""]+", "");
// Write back the normalized actual text so that the diff command given on failure shows what was actually compared.
File.WriteAllText(actualFileName, actualText);
// Make sure we can successfully deserialize what was just generated
SarifLog expectedLog = JsonConvert.DeserializeObject <SarifLog>(expectedText, settings);
SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(actualText, settings);
var visitor = new ResultDiffingVisitor(expectedLog);
if (!visitor.Diff(actualLog.Runs[0].Results))
{
string errorMessage = "The output of the tool did not match for input {0}.";
sb.AppendLine(String.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
sb.AppendLine("Check differences with:");
sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName));
}
}