protected void Verify(SarifValidationSkimmerBase skimmer, string testFileName)
{
string ruleName = skimmer.GetType().Name;
string testDirectory = Path.Combine(Environment.CurrentDirectory, TestDataDirectory, ruleName);
string targetPath = Path.Combine(testDirectory, testFileName);
string expectedFilePath = MakeExpectedFilePath(testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
JsonSerializerSettings settings = new JsonSerializerSettings
{
ContractResolver = SarifContractResolver.Instance
};
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents, settings);
using (var logger = new SarifLogger(
actualFilePath,
new string[] { targetPath },
verbose: false,
logEnvironment: false,
computeTargetsHash: false,
prereleaseInfo: null,
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
string expectedLogContents = File.ReadAllText(expectedFilePath);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
}
protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTransform)
{
string targetPath = Path.Combine(_testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(_testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
if (!disablePrereleaseCompatibilityTransform)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(inputLogContents, formatting: Formatting.Indented, out inputLogContents);
}
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents);
bool expectedResultsArePresent = inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults);
expectedResultsArePresent.Should().Be(true);
var skimmer = new TSkimmer();
using (var logger = new SarifLogger(
actualFilePath,
LoggingOptions.None,
tool: null,
run: null,
analysisTargets: new string[] { targetPath },
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
SarifLog outputLog = JsonConvert.DeserializeObject <SarifLog>(actualLogContents);
Verify(outputLog.Runs[0], expectedResults);
}
// For the moment, we support two different test designs.
//
// The new, preferred design (all new tests should be written this way):
//
// The test file itself contains a custom property that summarizes the expected
// results of running the rule on the test file.
//
// The old, deprecated design:
//
// To each test file there exists a corresponding file whose name ends in
// "_Expected.sarif" that contains the expected results of running the rule
// on the test file. We perform a "selective compare" of the expected and
// actual validation log file contents.
//
// As we migrate from the old to the new design, if the custom property exists,
// we use the new design, if the "expected" file exists, we use the old design,
// and if both the custom property and the "expected" file exist, we execute
// both the new and the old style tests.
protected void Verify(string testFileName)
{
string targetPath = Path.Combine(_testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(_testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents, _jsonSerializerSettings);
var skimmer = new TSkimmer();
using (var logger = new SarifLogger(
actualFilePath,
LoggingOptions.None,
tool: null,
run: null,
analysisTargets: new string[] { targetPath },
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
bool resultsWereVerified = false;
string expectedFilePath = MakeExpectedFilePath(_testDirectory, testFileName);
if (File.Exists(expectedFilePath))
{
// The "expected" file exists. Use the old, deprecated verification method.
string expectedLogContents = File.ReadAllText(expectedFilePath);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
resultsWereVerified = true;
}
if (inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults))
{
// The custom property exists. Use the new, preferred verification method.
SarifLog outputLog = JsonConvert.DeserializeObject <SarifLog>(actualLogContents, _jsonSerializerSettings);
Verify(outputLog.Runs[0], expectedResults);
resultsWereVerified = true;
}
// Temporary: make sure at least one of the verification methods executed.
resultsWereVerified.Should().BeTrue();
}
