public void AnalyzeCommandBase_RunDefaultRules()
{
string location = GetThisTestAssemblyFilePath();
Run run = null;
try
{
TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError;
run = AnalyzeFile(location);
}
finally
{
TestRule.s_testRuleBehaviors = TestRuleBehaviors.None;
}
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// As configured by the inject TestRuleBehaviors value, we should see
// an error for every scan target (of which there is one file in this test).
resultCount.Should().Be(1);
run.Results[0].Level.Should().Be(FailureLevel.Error);
toolNotificationCount.Should().Be(0);
configurationNotificationCount.Should().Be(0);
}
public void AnalyzeCommandBase_DefaultEndToEndAnalysis()
{
string location = GetThisTestAssemblyFilePath();
Run run = null;
try
{
TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError;
run = AnalyzeFile(location);
}
finally
{
TestRule.s_testRuleBehaviors = TestRuleBehaviors.None;
}
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// As configured by injected TestRuleBehaviors, we should
// see an error per scan target (one file in this case).
resultCount.Should().Be(1);
run.Results[0].Kind.Should().Be(ResultKind.Fail);
toolNotificationCount.Should().Be(0);
configurationNotificationCount.Should().Be(0);
}
public void AnalyzeCommandBase_EndToEndAnalysisWithExplicitlyDisabledRules()
{
PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration;
string path = Path.GetTempFileName() + ".xml";
try
{
allRulesDisabledConfiguration.SaveToXml(path);
string location = GetThisTestAssemblyFilePath();
Run run = AnalyzeFile(
location,
configFileName: path,
runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded,
expectedReturnCode: TestAnalyzeCommand.FAILURE);
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// When rules are disabled, we expect a configuration warning for each
// disabled check that documents it was turned off for the analysis.
resultCount.Should().Be(0);
// Three notifications. One for each disabled rule, i.e. SimpleTestRule
// and SimpleTestRule + an error notification that all rules have been disabled
configurationNotificationCount.Should().Be(3);
run.Invocations.Should().NotBeNull();
run.Invocations.Count.Should().Be(1);
// Error: all rules were disabled
run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Error).Should().Be(1);
run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Descriptor.Id == Errors.ERR997_AllRulesExplicitlyDisabled).Should().Be(1);
// Warnings: one per disabled rule.
run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Warning).Should().Be(2);
run.Invocations[0].ToolConfigurationNotifications.Where((notification) => notification.Descriptor.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(2);
// We raised a notification error, which means the invocation failed.
run.Invocations[0].ExecutionSuccessful.Should().Be(false);
toolNotificationCount.Should().Be(0);
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
public void AnalyzeCommand_EndToEndAnalysisWithExplicitlyDisabledRules()
{
PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration;
string path = Path.GetTempFileName() + ".xml";
try
{
allRulesDisabledConfiguration.SaveToXml(path);
string location = GetThisTestAssemblyFilePath();
Run run = AnalyzeFile(
location,
configFileName: path,
runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded,
expectedReturnCode: TestAnalyzeCommand.FAILURE);
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// When rules are disabled, we expect a configuration warning for each
// disabled check that documents it was turned off for the analysis.
resultCount.Should().Be(0);
// Three notifications. One for each disabled rule. And an error
// notification that all rules have been disabled
configurationNotificationCount.Should().Be(4);
run.Invocations.Should().NotBeNull();
run.Invocations.Count.Should().Be(1);
run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Error).Count().Should().Be(1);
run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Warning).Count().Should().Be(3);
run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(3);
toolNotificationCount.Should().Be(0);
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
public void AnalyzeCommand_EndToEndAnalysisWithNoIssues()
{
Run run = AnalyzeFile(this.GetType().Assembly.Location);
int resultCount = 0;
int toolNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; });
Assert.Equal(0, resultCount);
Assert.Equal(1, toolNotificationCount);
}
public void AnalyzeCommand_FireAllRules()
{
PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration;
string path = Path.GetTempFileName() + ".xml";
configuration.SetProperty(SimpleTestRule.Behaviors, TestRuleBehaviors.LogError);
try
{
configuration.SaveToXml(path);
string location = GetThisTestAssemblyFilePath();
Run run = AnalyzeFile(location, configFileName: path);
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// By default, the exception raising rule produces a single error.
// The simple test rule doesn't raise anything without add'l configuration
resultCount.Should().Be(2);
run.Results.Where((result) => result.Level == ResultLevel.Error).Count().Should().Be(1);
run.Results.Where((result) => result.Level == ResultLevel.Warning).Count().Should().Be(1);
run.Results.Where((result) => result.Level == ResultLevel.NotApplicable).Count().Should().Be(0);
toolNotificationCount.Should().Be(1);
configurationNotificationCount.Should().Be(0);
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
public void AnalyzeCommandBase_FireAllRules()
{
PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration;
string path = Path.GetTempFileName() + ".xml";
configuration.SetProperty(TestRule.Behaviors, TestRuleBehaviors.LogError);
try
{
configuration.SaveToXml(path);
string location = GetThisTestAssemblyFilePath();
Run run = AnalyzeFile(location, configFileName: path);
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// As configured by context, we should see a single error raised.
resultCount.Should().Be(1);
run.Results.Count((result) => result.Level == FailureLevel.Error).Should().Be(1);
toolNotificationCount.Should().Be(0);
configurationNotificationCount.Should().Be(0);
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
public void AnalyzeCommand_FireDefaultRule()
{
string location = GetThisTestAssemblyFilePath();
Run run = AnalyzeFile(location);
int resultCount = 0;
int toolNotificationCount = 0;
int configurationNotificationCount = 0;
SarifHelpers.ValidateRun(
run,
(issue) => { resultCount++; },
(toolNotification) => { toolNotificationCount++; },
(configurationNotification) => { configurationNotificationCount++; });
// By default, the exception raising rule produces a single error.
// The simple test rule doesn't raise anything without add'l configuration
resultCount.Should().Be(1);
run.Results[0].Level.Should().Be(ResultLevel.Warning);
toolNotificationCount.Should().Be(1);
configurationNotificationCount.Should().Be(0);
}
