protected void Verify(SarifValidationSkimmerBase skimmer, string testFileName)
{
string ruleName = skimmer.GetType().Name;
string testDirectory = Path.Combine(Environment.CurrentDirectory, TestDataDirectory, ruleName);
string targetPath = Path.Combine(testDirectory, testFileName);
string expectedFilePath = MakeExpectedFilePath(testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
JsonSerializerSettings settings = new JsonSerializerSettings
{
ContractResolver = SarifContractResolver.Instance
};
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents, settings);
using (var logger = new SarifLogger(
actualFilePath,
new string[] { targetPath },
verbose: false,
logEnvironment: false,
computeTargetsHash: false,
prereleaseInfo: null,
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
string expectedLogContents = File.ReadAllText(expectedFilePath);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
}
protected void Verify(string testFileName, bool disablePrereleaseCompatibilityTransform)
{
string targetPath = Path.Combine(_testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(_testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
if (!disablePrereleaseCompatibilityTransform)
{
PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(inputLogContents, formatting: Formatting.Indented, out inputLogContents);
}
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents);
bool expectedResultsArePresent = inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults);
expectedResultsArePresent.Should().Be(true);
var skimmer = new TSkimmer();
using (var logger = new SarifLogger(
actualFilePath,
LoggingOptions.None,
tool: null,
run: null,
analysisTargets: new string[] { targetPath },
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
SarifLog outputLog = JsonConvert.DeserializeObject <SarifLog>(actualLogContents);
Verify(outputLog.Runs[0], expectedResults);
}
/// <inheritdoc/>
public async Task <Stream> AnalyzeAsync(IEnumerable <string> targetFiles, CancellationToken cancellationToken)
{
targetFiles = targetFiles ?? throw new ArgumentNullException(nameof(targetFiles));
if (!targetFiles.Any())
{
return(Stream.Null);
}
string solutionDirectory = await GetSolutionDirectoryAsync().ConfigureAwait(continueOnCapturedContext: false);
var stream = new MemoryStream();
bool wasAnalyzed = false;
using (var writer = new StreamWriter(stream, Encoding.UTF8, DefaultBufferSize, leaveOpen: true))
{
using (SarifLogger sarifLogger = this.MakeSarifLogger(writer))
{
sarifLogger.AnalysisStarted();
foreach (string targetFile in targetFiles)
{
bool currentAnalysis;
var uri = new Uri(targetFile, UriKind.Absolute);
string text = File.ReadAllText(targetFile);
currentAnalysis = this.AnalyzeCore(uri, text, solutionDirectory, sarifLogger, cancellationToken);
if (!wasAnalyzed && currentAnalysis)
{
wasAnalyzed = currentAnalysis;
}
}
sarifLogger.AnalysisStopped(RuntimeConditions.None);
}
if (wasAnalyzed)
{
await writer.FlushAsync().ConfigureAwait(continueOnCapturedContext: false);
}
}
if (!wasAnalyzed)
{
stream.Dispose();
return(Stream.Null);
}
return(stream);
}
protected virtual void Dispose(bool disposing)
{
if (!_isDisposed)
{
if (disposing)
{
if (_logger != null)
{
_logger.AnalysisStopped(RuntimeConditions.NoErrors);
_logger.Dispose();
}
}
_isDisposed = true;
}
}
/// <inheritdoc/>
public async Task <Stream> AnalyzeAsync(string path, string text, CancellationToken cancellationToken)
{
text = text ?? throw new ArgumentNullException(nameof(text));
string solutionDirectory = await GetSolutionDirectoryAsync().ConfigureAwait(continueOnCapturedContext: false);
// If we don't have a solutionDirectory, then, we don't need to analyze.
if (string.IsNullOrEmpty(solutionDirectory))
{
return(Stream.Null);
}
var stream = new MemoryStream();
bool wasAnalyzed;
using (var writer = new StreamWriter(stream, Encoding.UTF8, DefaultBufferSize, leaveOpen: true))
{
using (SarifLogger sarifLogger = this.MakeSarifLogger(writer))
{
sarifLogger.AnalysisStarted();
// TODO: What do we do when path is null (text buffer with no backing file)?
Uri uri = path != null ? new Uri(path, UriKind.Absolute) : null;
wasAnalyzed = this.AnalyzeCore(uri, text, solutionDirectory, sarifLogger, cancellationToken);
sarifLogger.AnalysisStopped(RuntimeConditions.None);
}
if (wasAnalyzed)
{
await writer.FlushAsync().ConfigureAwait(continueOnCapturedContext: false);
}
}
if (!wasAnalyzed)
{
stream.Dispose();
return(Stream.Null);
}
return(stream);
}
// For the moment, we support two different test designs.
//
// The new, preferred design (all new tests should be written this way):
//
// The test file itself contains a custom property that summarizes the expected
// results of running the rule on the test file.
//
// The old, deprecated design:
//
// To each test file there exists a corresponding file whose name ends in
// "_Expected.sarif" that contains the expected results of running the rule
// on the test file. We perform a "selective compare" of the expected and
// actual validation log file contents.
//
// As we migrate from the old to the new design, if the custom property exists,
// we use the new design, if the "expected" file exists, we use the old design,
// and if both the custom property and the "expected" file exist, we execute
// both the new and the old style tests.
protected void Verify(string testFileName)
{
string targetPath = Path.Combine(_testDirectory, testFileName);
string actualFilePath = MakeActualFilePath(_testDirectory, testFileName);
string inputLogContents = File.ReadAllText(targetPath);
SarifLog inputLog = JsonConvert.DeserializeObject <SarifLog>(inputLogContents, _jsonSerializerSettings);
var skimmer = new TSkimmer();
using (var logger = new SarifLogger(
actualFilePath,
LoggingOptions.None,
tool: null,
run: null,
analysisTargets: new string[] { targetPath },
invocationTokensToRedact: null))
{
logger.AnalysisStarted();
var context = new SarifValidationContext
{
Rule = skimmer,
Logger = logger,
TargetUri = new Uri(targetPath),
SchemaFilePath = JsonSchemaFile,
InputLogContents = inputLogContents,
InputLog = inputLog
};
skimmer.Initialize(context);
context.Logger.AnalyzingTarget(context);
skimmer.Analyze(context);
logger.AnalysisStopped(RuntimeConditions.None);
}
string actualLogContents = File.ReadAllText(actualFilePath);
bool resultsWereVerified = false;
string expectedFilePath = MakeExpectedFilePath(_testDirectory, testFileName);
if (File.Exists(expectedFilePath))
{
// The "expected" file exists. Use the old, deprecated verification method.
string expectedLogContents = File.ReadAllText(expectedFilePath);
// We can't just compare the text of the log files because properties
// like start time, and absolute paths, will differ from run to run.
// Until SarifLogger has a "deterministic" option (see http://github.com/Microsoft/sarif-sdk/issues/500),
// we perform a selective compare of just the elements we care about.
SelectiveCompare(actualLogContents, expectedLogContents);
resultsWereVerified = true;
}
if (inputLog.Runs[0].TryGetProperty(ExpectedResultsPropertyName, out ExpectedValidationResults expectedResults))
{
// The custom property exists. Use the new, preferred verification method.
SarifLog outputLog = JsonConvert.DeserializeObject <SarifLog>(actualLogContents, _jsonSerializerSettings);
Verify(outputLog.Runs[0], expectedResults);
resultsWereVerified = true;
}
// Temporary: make sure at least one of the verification methods executed.
resultsWereVerified.Should().BeTrue();
}
