Exemple #1
0
        private async Task Run(TestCase testCase, DiscoveredTestData testData, TestRunContext testRunContext, StepBinder stepBinder, IFrameworkHandle frameworkHandle, SemaphoreSlim simultaneousTestCasesSemaphore)
        {
            await simultaneousTestCasesSemaphore
            .WaitAsync()
            .ConfigureAwait(false);

            try
            {
                frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Starting test \"{testCase.DisplayName}\"");

                frameworkHandle.RecordStart(testCase);

                var executor = stepsExecutorFactory(stepBinder);

                var testResult = await executor
                                 .Execute(testCase, testData, testRunContext, frameworkHandle)
                                 .ConfigureAwait(false);

                // https://github.com/Microsoft/vstest/blob/master/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs <- comments here seem to suggest that we need to call RecordEnd just before RecordResult
                frameworkHandle.RecordEnd(testCase, testResult.Outcome);
                frameworkHandle.RecordResult(testResult);

                frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Finished test \"{testCase.DisplayName}\"");
            }
            finally
            {
                simultaneousTestCasesSemaphore.Release();
            }
        }
        public async Task <TestResult> Execute(TestCase testCase, DiscoveredTestData testData, TestRunContext testRunContext, IMessageLogger logger)
        {
            const double SmallestTimeRecognisedByTestRunnerInSeconds = 0.0005;

            var testResult = new TestResult(testCase);
            var startTicks = Stopwatch.GetTimestamp();

            try
            {
                (testRunContext.Logger as TestLogAccessor)?.SetCurrentTestResult(testResult);

                if (testData == null)
                {
                    throw new ArgumentNullException(nameof(testData));
                }

                var hasAnySteps = testData.Feature.Background.Steps
                                  .Concat(testData.Scenario.Steps)
                                  .Concat(testData.Rule?.Background?.Steps ?? Enumerable.Empty <IStep>())
                                  .Any();

                if (!hasAnySteps)
                {
                    MarkTestAsSkipped(testResult);
                }
                else
                {
                    using (var serviceScope = testRunContext.ServiceProvider.CreateScope())
                    {
                        // Before Scenario hooks should run here (see https://docs.cucumber.io/gherkin/reference/#background)
                        // > A Background is run before each scenario, but after any Before hooks. In your feature file, put the Background before the first Scenario.

                        IEnumerable <IStep> allScenarioSteps = testData.Feature.Background.Steps;

                        if (testData.Rule != null)
                        {
                            allScenarioSteps = allScenarioSteps.Concat(testData.Rule.Background.Steps);
                        }

                        allScenarioSteps = allScenarioSteps.Concat(testData.Scenario.Steps);

                        await ExecuteSteps(serviceScope.ServiceProvider, testResult, allScenarioSteps, testData, testRunContext)
                        .ConfigureAwait(false);
                    }

                    testResult.Outcome = TestOutcome.Passed;
                }

                testResult.Duration = TimeSpan.FromSeconds(
                    Math.Max(
                        SmallestTimeRecognisedByTestRunnerInSeconds,
                        (Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency));
            }
            catch (Exception exception)
            {
                MarkTestAsFailed(testCase, testResult, exception, logger);
            }

            return(testResult);
        }
        private async Task ExecuteSteps(
            IServiceProvider serviceProvider,
            TestResult testResult,
            IEnumerable <IStep> steps,
            DiscoveredTestData testData,
            TestRunContext testRunContext)
        {
            var scenarioExecutionStartTime = DateTime.UtcNow;
            var stepsIterator = new StepsIterator(steps);

            foreach (var step in stepsIterator.Iterate())
            {
                testResult.Messages.Add(
                    new TestResultMessage(
                        TestResultMessage.StandardOutCategory,
                        $"{step.Title}{Environment.NewLine}"));

                var stepBinding       = stepBinder.GetBindingFor(step, testData.Assembly);
                var executionStrategy = StepExecutionStrategyFactory.GetFor(stepBinding);

                try
                {
                    await executionStrategy
                    .Execute(stepBinding, serviceProvider, testResult.Messages, testRunContext)
                    .ConfigureAwait(false);

                    testResult.Messages.Add(
                        new TestResultMessage(
                            TestResultMessage.StandardOutCategory,
                            $"{StepLogIndent}Completed at {DateTime.UtcNow:o}{Environment.NewLine}{Environment.NewLine}"));
                }
                catch (Exception exception)
                {
                    if (ScenarioFailureIsTerminal(
                            testData.Scenario,
                            scenarioExecutionStartTime,
                            stepsIterator))
                    {
                        testResult.Messages.Add(
                            new TestResultMessage(
                                TestResultMessage.StandardOutCategory,
                                $"{StepLogIndent}Failed at {DateTime.UtcNow:o}{Environment.NewLine}{Environment.NewLine}"));

                        testResult.Messages.Add(
                            new TestResultMessage(
                                TestResultMessage.StandardErrorCategory,
                                $"{exception}{Environment.NewLine}"));

                        throw;
                    }

                    testResult.Messages.Add(
                        new TestResultMessage(
                            TestResultMessage.StandardOutCategory,
                            $"{StepLogIndent}Failed at {DateTime.UtcNow:o}, waiting and retrying scenario from last When step{Environment.NewLine}"));

                    await Task.Delay(testData.Scenario.EventuallyConsistentConfiguration.RetryInterval).ConfigureAwait(false);
                }
            }
        }
Exemple #4
0
        private async Task <TestResultMessage[]> PerformEventuallyConsistentStepsTest(MethodInfo method)
        {
            var scenarioStep = new WhenStep("When an exception is thrown", DataTable.Empty, null);
            var testFeature  = new Feature(
                "Feature",
                null,
                Background.Empty,
                new[]
            {
                new Scenario(
                    "Scenario",
                    new[] { scenarioStep },
                    1,
                    Enumerable.Empty <Tag>())
            },
                Enumerable.Empty <ScenarioOutline>(),
                Enumerable.Empty <Rule>(),
                Enumerable.Empty <Tag>());

            var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
            var binding  = new StepBinding(scenarioStep, method, Array.Empty <object>());

            mockStepBinder
            .Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
            .Returns(binding);

            var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);

            return(testResult
                   .Messages
                   .Where(
                       message => message.Category == TestResultMessage.StandardOutCategory)
                   .ToArray());
        }
        public async Task SetThreadUICultureToMatchFeatureCulture()
        {
            var scenarioStep = new GivenStep("Scenario step", DataTable.Empty, null);
            var testFeature  = new Feature("Feature", null,
                                           Background.Empty,
                                           new[]
            {
                new Scenario(
                    "Scenario",
                    new[] { scenarioStep },
                    1,
                    Enumerable.Empty <Tag>())
            },
                                           Enumerable.Empty <ScenarioOutline>(),
                                           Enumerable.Empty <Rule>(),
                                           new[] {
                new Tag("culture(nb-NO)")
            });

            var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());

            string capturedCultureName     = null;
            var    mockScenarioStepMapping = new Mock <IStepBinding>();

            mockStepBinder
            .Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
            .Returns(mockScenarioStepMapping.Object)
            .Callback(() => capturedCultureName = CultureInfo.CurrentUICulture.Name);

            var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);

            Assert.AreEqual("nb-NO", capturedCultureName);
        }
Exemple #6
0
        private async Task <TestResultMessage[]> PerformEventuallyConsistentScenarioTest(MethodInfo givenMethod, MethodInfo whenMethod, MethodInfo thenMethod)
        {
            var givenStep   = new GivenStep("Given", DataTable.Empty, null);
            var whenStep    = new WhenStep("When", DataTable.Empty, null);
            var thenStep    = new ThenStep("Then", DataTable.Empty, null);
            var testFeature = new Feature(
                "Feature",
                null,
                Background.Empty,
                new[]
            {
                new Scenario(
                    "Scenario",
                    new IStep[] { givenStep, whenStep, thenStep },
                    1,
                    new List <Tag> {
                    new Tag("eventuallyConsistent(retryInterval=00:00:01;within=00:00:05)")
                })
            },
                Enumerable.Empty <ScenarioOutline>(),
                Enumerable.Empty <Rule>(),
                Enumerable.Empty <Tag>());

            var testData         = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
            var givenStepBinding = new StepBinding(givenStep, givenMethod, Array.Empty <object>());
            var whenStepBinding  = new StepBinding(whenStep, whenMethod, Array.Empty <object>());
            var thenStepBinding  = new StepBinding(thenStep, thenMethod, Array.Empty <object>());

            mockStepBinder
            .Setup(m => m.GetBindingFor(givenStep, testAssembly))
            .Returns(givenStepBinding);

            mockStepBinder
            .Setup(m => m.GetBindingFor(whenStep, testAssembly))
            .Returns(whenStepBinding);

            mockStepBinder
            .Setup(m => m.GetBindingFor(thenStep, testAssembly))
            .Returns(thenStepBinding);

            var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);

            return(testResult
                   .Messages
                   .Where(
                       message => message.Category == TestResultMessage.StandardOutCategory)
                   .ToArray());
        }
Exemple #7
0
        public async Task ExecuteFeatureBackgroundStepsThenScenarioSteps()
        {
            var backgroundStep = new GivenStep("Feature background step", DataTable.Empty, null);
            var scenarioStep   = new GivenStep("Scenario step", DataTable.Empty, null);

            var testFeature = new Feature("Feature", null,
                                          new Background(
                                              new[] { backgroundStep },
                                              0),
                                          new[]
            {
                new Scenario(
                    "Scenario",
                    new[] { scenarioStep },
                    1,
                    Enumerable.Empty <Tag>())
            },
                                          Enumerable.Empty <ScenarioOutline>(),
                                          Enumerable.Empty <Rule>(),
                                          Enumerable.Empty <Tag>());

            var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());

            var invocationOrder           = 0;
            var mockBackgroundStepMapping = new Mock <IStepBinding>();

            mockStepBinder
            .Setup(m => m.GetBindingFor(backgroundStep, testAssembly))
            .Returns(mockBackgroundStepMapping.Object)
            .Callback(() => Assert.AreEqual(0, invocationOrder++));

            var mockScenarioStepMapping = new Mock <IStepBinding>();

            mockStepBinder
            .Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
            .Returns(mockScenarioStepMapping.Object)
            .Callback(() => Assert.AreEqual(1, invocationOrder++));

            var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);

            mockBackgroundStepMapping.Verify(m => m.Execute(It.IsAny <IServiceProvider>(), It.IsAny <Collection <TestResultMessage> >()), Times.Once);
            mockScenarioStepMapping.Verify(m => m.Execute(It.IsAny <IServiceProvider>(), It.IsAny <Collection <TestResultMessage> >()), Times.Once);
        }
        public async Task ExecuteSteps(IServiceProvider serviceProvider, TestResult testResult, IEnumerable <IStep> steps, DiscoveredTestData testData, TestRunContext testRunContext)
        {
            var startTime = DateTime.UtcNow;

            for (var stepIndex = 0; stepIndex < steps.Count(); stepIndex++)
            {
                var step = steps.ElementAt(stepIndex);

                testResult.Messages.Add(
                    new TestResultMessage(
                        TestResultMessage.StandardOutCategory,
                        $"{step.Title}{Environment.NewLine}"));

                var stepBinding       = stepBinder.GetBindingFor(step, testData.Assembly);
                var executionStrategy = StepExecutionStrategyFactory.GetFor(stepBinding);

                try
                {
                    await executionStrategy
                    .Execute(stepBinding, serviceProvider, testResult.Messages, testRunContext)
                    .ConfigureAwait(false);
                }
                catch (Exception exception)
                {
                    if (!testData.Scenario.IsEventuallyConsistent ||
                        DateTime.UtcNow > startTime.Add(testData.Scenario.EventuallyConsistentConfiguration.Within))
                    {
                        testResult.Messages.Add(
                            new TestResultMessage(
                                TestResultMessage.StandardOutCategory,
                                $"{StepsExecutor.StepLogIndent}Failed{Environment.NewLine}{Environment.NewLine}"));

                        testResult.Messages.Add(
                            new TestResultMessage(
                                TestResultMessage.StandardErrorCategory,
                                $"{exception}{Environment.NewLine}"));

                        throw;
                    }

                    var previousStepsInReverse = steps.Take(stepIndex + 1).Reverse();
                    var lastWhenStep           = previousStepsInReverse.First(s => s is WhenStep);
                    stepIndex = steps.ToList().IndexOf(lastWhenStep);
                    await Task.Delay(testData.Scenario.EventuallyConsistentConfiguration.RetryInterval).ConfigureAwait(false);

                    continue;
                }

                testResult.Messages.Add(
                    new TestResultMessage(
                        TestResultMessage.StandardOutCategory,
                        $"{StepsExecutor.StepLogIndent}Completed{Environment.NewLine}{Environment.NewLine}"));
            }
        }