public void AnalyzeCommandBase_RunDefaultRules()
        {
            string location = GetThisTestAssemblyFilePath();

            Run run = null;

            try
            {
                TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError;
                run = AnalyzeFile(location);
            }
            finally
            {
                TestRule.s_testRuleBehaviors = TestRuleBehaviors.None;
            }

            int resultCount                    = 0;
            int toolNotificationCount          = 0;
            int configurationNotificationCount = 0;

            SarifHelpers.ValidateRun(
                run,
                (issue) => { resultCount++; },
                (toolNotification) => { toolNotificationCount++; },
                (configurationNotification) => { configurationNotificationCount++; });

            // As configured by the inject TestRuleBehaviors value, we should see
            // an error for every scan target (of which there is one file in this test).
            resultCount.Should().Be(1);
            run.Results[0].Level.Should().Be(FailureLevel.Error);

            toolNotificationCount.Should().Be(0);
            configurationNotificationCount.Should().Be(0);
        }
        public void AnalyzeCommandBase_DefaultEndToEndAnalysis()
        {
            string location = GetThisTestAssemblyFilePath();

            Run run = null;

            try
            {
                TestRule.s_testRuleBehaviors = TestRuleBehaviors.LogError;
                run = AnalyzeFile(location);
            }
            finally
            {
                TestRule.s_testRuleBehaviors = TestRuleBehaviors.None;
            }

            int resultCount                    = 0;
            int toolNotificationCount          = 0;
            int configurationNotificationCount = 0;

            SarifHelpers.ValidateRun(
                run,
                (issue) => { resultCount++; },
                (toolNotification) => { toolNotificationCount++; },
                (configurationNotification) => { configurationNotificationCount++; });

            // As configured by injected TestRuleBehaviors, we should
            // see an error per scan target (one file in this case).
            resultCount.Should().Be(1);
            run.Results[0].Kind.Should().Be(ResultKind.Fail);

            toolNotificationCount.Should().Be(0);
            configurationNotificationCount.Should().Be(0);
        }
Example #3
0
        public void AnalyzeCommandBase_EndToEndAnalysisWithExplicitlyDisabledRules()
        {
            PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration;
            string path = Path.GetTempFileName() + ".xml";

            try
            {
                allRulesDisabledConfiguration.SaveToXml(path);

                string location = GetThisTestAssemblyFilePath();

                Run run = AnalyzeFile(
                    location,
                    configFileName: path,
                    runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded,
                    expectedReturnCode: TestAnalyzeCommand.FAILURE);

                int resultCount                    = 0;
                int toolNotificationCount          = 0;
                int configurationNotificationCount = 0;

                SarifHelpers.ValidateRun(
                    run,
                    (issue) => { resultCount++; },
                    (toolNotification) => { toolNotificationCount++; },
                    (configurationNotification) => { configurationNotificationCount++; });

                // When rules are disabled, we expect a configuration warning for each
                // disabled check that documents it was turned off for the analysis.
                resultCount.Should().Be(0);

                // Three notifications. One for each disabled rule, i.e. SimpleTestRule
                // and SimpleTestRule + an error notification that all rules have been disabled
                configurationNotificationCount.Should().Be(3);

                run.Invocations.Should().NotBeNull();
                run.Invocations.Count.Should().Be(1);

                // Error: all rules were disabled
                run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Error).Should().Be(1);
                run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Descriptor.Id == Errors.ERR997_AllRulesExplicitlyDisabled).Should().Be(1);

                // Warnings: one per disabled rule.
                run.Invocations[0].ToolConfigurationNotifications.Count((notification) => notification.Level == FailureLevel.Warning).Should().Be(2);
                run.Invocations[0].ToolConfigurationNotifications.Where((notification) => notification.Descriptor.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(2);

                // We raised a notification error, which means the invocation failed.
                run.Invocations[0].ExecutionSuccessful.Should().Be(false);

                toolNotificationCount.Should().Be(0);
            }
            finally
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
            }
        }
Example #4
0
        public void AnalyzeCommand_EndToEndAnalysisWithExplicitlyDisabledRules()
        {
            PropertiesDictionary allRulesDisabledConfiguration = ExportConfigurationCommandBaseTests.s_allRulesDisabledConfiguration;
            string path = Path.GetTempFileName() + ".xml";

            try
            {
                allRulesDisabledConfiguration.SaveToXml(path);

                string location = GetThisTestAssemblyFilePath();

                Run run = AnalyzeFile(
                    location,
                    configFileName: path,
                    runtimeConditions: RuntimeConditions.RuleWasExplicitlyDisabled | RuntimeConditions.NoRulesLoaded,
                    expectedReturnCode: TestAnalyzeCommand.FAILURE);

                int resultCount                    = 0;
                int toolNotificationCount          = 0;
                int configurationNotificationCount = 0;

                SarifHelpers.ValidateRun(
                    run,
                    (issue) => { resultCount++; },
                    (toolNotification) => { toolNotificationCount++; },
                    (configurationNotification) => { configurationNotificationCount++; });

                // When rules are disabled, we expect a configuration warning for each
                // disabled check that documents it was turned off for the analysis.
                resultCount.Should().Be(0);

                // Three notifications. One for each disabled rule. And an error
                // notification that all rules have been disabled
                configurationNotificationCount.Should().Be(4);

                run.Invocations.Should().NotBeNull();
                run.Invocations.Count.Should().Be(1);

                run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Error).Count().Should().Be(1);
                run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Level == NotificationLevel.Warning).Count().Should().Be(3);

                run.Invocations[0].ConfigurationNotifications.Where((notification) => notification.Id == Warnings.Wrn999_RuleExplicitlyDisabled).Count().Should().Be(3);

                toolNotificationCount.Should().Be(0);
            }
            finally
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
            }
        }
        public void AnalyzeCommand_EndToEndAnalysisWithNoIssues()
        {
            Run run = AnalyzeFile(this.GetType().Assembly.Location);

            int resultCount           = 0;
            int toolNotificationCount = 0;

            SarifHelpers.ValidateRun(
                run,
                (issue) => { resultCount++; },
                (toolNotification) => { toolNotificationCount++; });

            Assert.Equal(0, resultCount);
            Assert.Equal(1, toolNotificationCount);
        }
        public void AnalyzeCommand_FireAllRules()
        {
            PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration;

            string path = Path.GetTempFileName() + ".xml";

            configuration.SetProperty(SimpleTestRule.Behaviors, TestRuleBehaviors.LogError);

            try
            {
                configuration.SaveToXml(path);

                string location = GetThisTestAssemblyFilePath();

                Run run = AnalyzeFile(location, configFileName: path);

                int resultCount                    = 0;
                int toolNotificationCount          = 0;
                int configurationNotificationCount = 0;

                SarifHelpers.ValidateRun(
                    run,
                    (issue) => { resultCount++; },
                    (toolNotification) => { toolNotificationCount++; },
                    (configurationNotification) => { configurationNotificationCount++; });

                // By default, the exception raising rule produces a single error.
                // The simple test rule doesn't raise anything without add'l configuration
                resultCount.Should().Be(2);
                run.Results.Where((result) => result.Level == ResultLevel.Error).Count().Should().Be(1);
                run.Results.Where((result) => result.Level == ResultLevel.Warning).Count().Should().Be(1);
                run.Results.Where((result) => result.Level == ResultLevel.NotApplicable).Count().Should().Be(0);

                toolNotificationCount.Should().Be(1);
                configurationNotificationCount.Should().Be(0);
            }
            finally
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
            }
        }
Example #7
0
        public void AnalyzeCommandBase_FireAllRules()
        {
            PropertiesDictionary configuration = ExportConfigurationCommandBaseTests.s_defaultConfiguration;

            string path = Path.GetTempFileName() + ".xml";

            configuration.SetProperty(TestRule.Behaviors, TestRuleBehaviors.LogError);

            try
            {
                configuration.SaveToXml(path);

                string location = GetThisTestAssemblyFilePath();

                Run run = AnalyzeFile(location, configFileName: path);

                int resultCount                    = 0;
                int toolNotificationCount          = 0;
                int configurationNotificationCount = 0;

                SarifHelpers.ValidateRun(
                    run,
                    (issue) => { resultCount++; },
                    (toolNotification) => { toolNotificationCount++; },
                    (configurationNotification) => { configurationNotificationCount++; });

                // As configured by context, we should see a single error raised.
                resultCount.Should().Be(1);
                run.Results.Count((result) => result.Level == FailureLevel.Error).Should().Be(1);

                toolNotificationCount.Should().Be(0);
                configurationNotificationCount.Should().Be(0);
            }
            finally
            {
                if (File.Exists(path))
                {
                    File.Delete(path);
                }
            }
        }
        public void AnalyzeCommand_FireDefaultRule()
        {
            string location = GetThisTestAssemblyFilePath();
            Run    run      = AnalyzeFile(location);

            int resultCount                    = 0;
            int toolNotificationCount          = 0;
            int configurationNotificationCount = 0;

            SarifHelpers.ValidateRun(
                run,
                (issue) => { resultCount++; },
                (toolNotification) => { toolNotificationCount++; },
                (configurationNotification) => { configurationNotificationCount++; });

            // By default, the exception raising rule produces a single error.
            // The simple test rule doesn't raise anything without add'l configuration
            resultCount.Should().Be(1);
            run.Results[0].Level.Should().Be(ResultLevel.Warning);

            toolNotificationCount.Should().Be(1);
            configurationNotificationCount.Should().Be(0);
        }