public void AnalyzeCommandBase_RunDefaultRules() { string location = GetThisTestAssemblyFilePath(); Run run = null; try { TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError; run = AnalyzeFile(location); } finally { TestRule.s_testRuleBehaviors = TestRuleBehaviors.None; } int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // As configured by the inject TestRuleBehaviors value, we should see // an error for every scan target (of which there is one file in this test). resultCount.Should().Be(1); run.Results[0].Level.Should().Be(FailureLevel.Error); toolNotificationCount.Should().Be(0); configurationNotificationCount.Should().Be(0); }
public void AnalyzeCommandBase_DefaultEndToEndAnalysis() { string location = GetThisTestAssemblyFilePath(); Run run = null; try { TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError; run = AnalyzeFile(location); } finally { TestRule.s_testRuleBehaviors = TestRuleBehaviors.None; } int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // As configured by injected TestRuleBehaviors, we should // see an error per scan target (one file in this case). resultCount.Should().Be(1); run.Results[0].Kind.Should().Be(ResultKind.Fail); toolNotificationCount.Should().Be(0); configurationNotificationCount.Should().Be(0); }
public void AnalyzeCommandBase_EndToEndAnalysisWithExplicitlyDisabledRules() { PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration; string path = Path.GetTempFileName() + ".xml"; try { allRulesDisabledConfiguration.SaveToXml(path); string location = GetThisTestAssemblyFilePath(); Run run = AnalyzeFile( location, configFileName: path, runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded, expectedReturnCode: TestAnalyzeCommand.FAILURE); int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // When rules are disabled, we expect a configuration warning for each // disabled check that documents it was turned off for the analysis. resultCount.Should().Be(0); // Three notifications. One for each disabled rule, i.e. SimpleTestRule // and SimpleTestRule + an error notification that all rules have been disabled configurationNotificationCount.Should().Be(3); run.Invocations.Should().NotBeNull(); run.Invocations.Count.Should().Be(1); // Error: all rules were disabled run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Error).Should().Be(1); run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Descriptor.Id == Errors.ERR997_AllRulesExplicitlyDisabled).Should().Be(1); // Warnings: one per disabled rule. run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Warning).Should().Be(2); run.Invocations[0].ToolConfigurationNotifications.Where((notification) => notification.Descriptor.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(2); // We raised a notification error, which means the invocation failed. run.Invocations[0].ExecutionSuccessful.Should().Be(false); toolNotificationCount.Should().Be(0); } finally { if (File.Exists(path)) { File.Delete(path); } } }
public void AnalyzeCommand_EndToEndAnalysisWithExplicitlyDisabledRules() { PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration; string path = Path.GetTempFileName() + ".xml"; try { allRulesDisabledConfiguration.SaveToXml(path); string location = GetThisTestAssemblyFilePath(); Run run = AnalyzeFile( location, configFileName: path, runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded, expectedReturnCode: TestAnalyzeCommand.FAILURE); int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // When rules are disabled, we expect a configuration warning for each // disabled check that documents it was turned off for the analysis. resultCount.Should().Be(0); // Three notifications. One for each disabled rule. And an error // notification that all rules have been disabled configurationNotificationCount.Should().Be(4); run.Invocations.Should().NotBeNull(); run.Invocations.Count.Should().Be(1); run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Error).Count().Should().Be(1); run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Warning).Count().Should().Be(3); run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(3); toolNotificationCount.Should().Be(0); } finally { if (File.Exists(path)) { File.Delete(path); } } }
public void AnalyzeCommand_EndToEndAnalysisWithNoIssues() { Run run = AnalyzeFile(this.GetType().Assembly.Location); int resultCount = 0; int toolNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }); Assert.Equal(0, resultCount); Assert.Equal(1, toolNotificationCount); }
public void AnalyzeCommand_FireAllRules() { PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration; string path = Path.GetTempFileName() + ".xml"; configuration.SetProperty(SimpleTestRule.Behaviors, TestRuleBehaviors.LogError); try { configuration.SaveToXml(path); string location = GetThisTestAssemblyFilePath(); Run run = AnalyzeFile(location, configFileName: path); int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // By default, the exception raising rule produces a single error. // The simple test rule doesn't raise anything without add'l configuration resultCount.Should().Be(2); run.Results.Where((result) => result.Level == ResultLevel.Error).Count().Should().Be(1); run.Results.Where((result) => result.Level == ResultLevel.Warning).Count().Should().Be(1); run.Results.Where((result) => result.Level == ResultLevel.NotApplicable).Count().Should().Be(0); toolNotificationCount.Should().Be(1); configurationNotificationCount.Should().Be(0); } finally { if (File.Exists(path)) { File.Delete(path); } } }
public void AnalyzeCommandBase_FireAllRules() { PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration; string path = Path.GetTempFileName() + ".xml"; configuration.SetProperty(TestRule.Behaviors, TestRuleBehaviors.LogError); try { configuration.SaveToXml(path); string location = GetThisTestAssemblyFilePath(); Run run = AnalyzeFile(location, configFileName: path); int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // As configured by context, we should see a single error raised. resultCount.Should().Be(1); run.Results.Count((result) => result.Level == FailureLevel.Error).Should().Be(1); toolNotificationCount.Should().Be(0); configurationNotificationCount.Should().Be(0); } finally { if (File.Exists(path)) { File.Delete(path); } } }
public void AnalyzeCommand_FireDefaultRule() { string location = GetThisTestAssemblyFilePath(); Run run = AnalyzeFile(location); int resultCount = 0; int toolNotificationCount = 0; int configurationNotificationCount = 0; SarifHelpers.ValidateRun( run, (issue) => { resultCount++; }, (toolNotification) => { toolNotificationCount++; }, (configurationNotification) => { configurationNotificationCount++; }); // By default, the exception raising rule produces a single error. // The simple test rule doesn't raise anything without add'l configuration resultCount.Should().Be(1); run.Results[0].Level.Should().Be(ResultLevel.Warning); toolNotificationCount.Should().Be(1); configurationNotificationCount.Should().Be(0); }