private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase)
{
// This makes sure that we will reinitialize the mock file system. This
// allows callers to reuse test case instances, by adjusting specific
// property values. The mock file system cannot be reused in this way, once
// a test executes, it must be reset.
testCase.FileSystem = null;
var options = new TestAnalyzeOptions
{
OutputFilePath = testCase.PersistLogFileToDisk ? Guid.NewGuid().ToString() : null,
TargetFileSpecifiers = new string[] { Guid.NewGuid().ToString() },
Verbose = testCase.Verbose,
ComputeFileHashes = false,
};
int expectedResultsCount = testCase.ExpectedWarningCount + testCase.ExpectedErrorCount;
Run runWithoutCaching = RunAnalyzeCommand(options, testCase);
options.ComputeFileHashes = true;
Run runWithCaching = RunAnalyzeCommand(options, testCase);
// Core static analysis results
runWithCaching.Results.Should().BeEquivalentTo(runWithoutCaching.Results);
// Tool configuration errors, such as 'Could not locate scan target PDB.'
runWithoutCaching.Invocations?[0].ToolConfigurationNotifications?.Should()
.BeEquivalentTo(runWithCaching.Invocations?[0].ToolConfigurationNotifications);
// Not yet explicitly tested
runWithoutCaching.Invocations?[0].ToolExecutionNotifications?.Should()
.BeEquivalentTo(runWithCaching.Invocations?[0].ToolExecutionNotifications);
}
public void AnalyzeCommandBase_CachesResultsWhenPersistingToLogFile()
{
var testCase = new ResultsCachingTestCase
{
Files = ComprehensiveKindAndLevelsByFileName,
PersistLogFileToDisk = true,
};
RunResultsCachingTestCase(testCase);
testCase.Verbose = true;
RunResultsCachingTestCase(testCase);
}
public void AnalyzeCommandBase_CachesNotificationsWithoutPersistingToLogFile()
{
var testCase = new ResultsCachingTestCase
{
Files = ComprehensiveKindAndLevelsByFileName,
TestRuleBehaviors = TestRuleBehaviors.RaiseTargetParseError,
ExpectedReturnCode = FAILURE,
};
RunResultsCachingTestCase(testCase);
testCase.Verbose = true;
RunResultsCachingTestCase(testCase);
}
public void AnalyzeCommandBase_CachesErrors()
{
// Produce two errors results
var testCase = new ResultsCachingTestCase()
{
Files = new List <string> {
"Error.dll", "Error.exe"
}
};
RunResultsCachingTestCase(testCase);
testCase.Verbose = true;
RunResultsCachingTestCase(testCase);
}
private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase)
{
// This makes sure that we will reinitialize the mock file system. This
// allows callers to reuse test case instances, by adjusting specific
// property values. The mock file system cannot be reused in this way, once
// a test executes, it must be reset.
testCase.FileSystem = null;
var options = new TestAnalyzeOptions
{
TestRuleBehaviors = testCase.TestRuleBehaviors,
OutputFilePath = testCase.PersistLogFileToDisk ? Guid.NewGuid().ToString() : null,
TargetFileSpecifiers = new string[] { Guid.NewGuid().ToString() },
Kind = new List <ResultKind> {
ResultKind.Fail
},
Level = new List <FailureLevel> {
FailureLevel.Warning, FailureLevel.Error
}
};
if (testCase.Verbose)
{
options.Kind = new List <ResultKind> {
ResultKind.Informational, ResultKind.Open, ResultKind.Review, ResultKind.Fail, ResultKind.Pass, ResultKind.NotApplicable, ResultKind.None
};
options.Level = new List <FailureLevel> {
FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note, FailureLevel.None
};
}
int expectedResultsCount = testCase.ExpectedWarningCount + testCase.ExpectedErrorCount;
Run runWithoutCaching = RunAnalyzeCommand(options, testCase);
options.DataToInsert = new OptionallyEmittedData[] { OptionallyEmittedData.Hashes };
Run runWithCaching = RunAnalyzeCommand(options, testCase);
// Core static analysis results
runWithCaching.Results.Should().BeEquivalentTo(runWithoutCaching.Results);
// Tool configuration errors, such as 'Could not locate scan target PDB.'
runWithoutCaching.Invocations?[0].ToolConfigurationNotifications?.Should()
.BeEquivalentTo(runWithCaching.Invocations?[0].ToolConfigurationNotifications);
// Not yet explicitly tested
runWithoutCaching.Invocations?[0].ToolExecutionNotifications?.Should()
.BeEquivalentTo(runWithCaching.Invocations?[0].ToolExecutionNotifications);
}
private static Run RunAnalyzeCommand(TestAnalyzeOptions options, ResultsCachingTestCase testCase)
{
Run run = null;
SarifLog sarifLog;
try
{
TestRule.s_testRuleBehaviors = testCase.TestRuleBehaviors.AccessibleOutsideOfContextOnly();
sarifLog = RunAnalyzeCommand(options, testCase.FileSystem, testCase.ExpectedReturnCode);
run = sarifLog.Runs[0];
run.Results.Count.Should().Be(testCase.ExpectedResultsCount);
}
finally
{
TestRule.s_testRuleBehaviors = TestRuleBehaviors.None;
}
return(run);
}
public void AnalyzeCommandBase_CachesNotes()
{
// Produce three results in verbose runs only
var testCase = new ResultsCachingTestCase()
{
Files = new List <string> {
"Note.dll", "Note.exe", "Note.sys"
}
};
// Notes are verbose only results
testCase.ExpectedResultsCount.Should().Be(0);
testCase.ExpectedNoteCount.Should().Be(0);
RunResultsCachingTestCase(testCase);
testCase.Verbose = true;
testCase.ExpectedResultsCount.Should().Be(testCase.Files.Count);
testCase.ExpectedNoteCount.Should().Be(testCase.Files.Count);
RunResultsCachingTestCase(testCase);
}