private async Task Run(TestCase testCase, DiscoveredTestData testData, TestRunContext testRunContext, StepBinder stepBinder, IFrameworkHandle frameworkHandle, SemaphoreSlim simultaneousTestCasesSemaphore)
{
await simultaneousTestCasesSemaphore
.WaitAsync()
.ConfigureAwait(false);
try
{
frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Starting test \"{testCase.DisplayName}\"");
frameworkHandle.RecordStart(testCase);
var executor = stepsExecutorFactory(stepBinder);
var testResult = await executor
.Execute(testCase, testData, testRunContext, frameworkHandle)
.ConfigureAwait(false);
// https://github.com/Microsoft/vstest/blob/master/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs <- comments here seem to suggest that we need to call RecordEnd just before RecordResult
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Finished test \"{testCase.DisplayName}\"");
}
finally
{
simultaneousTestCasesSemaphore.Release();
}
}
public async Task <TestResult> Execute(TestCase testCase, DiscoveredTestData testData, TestRunContext testRunContext, IMessageLogger logger)
{
const double SmallestTimeRecognisedByTestRunnerInSeconds = 0.0005;
var testResult = new TestResult(testCase);
var startTicks = Stopwatch.GetTimestamp();
try
{
(testRunContext.Logger as TestLogAccessor)?.SetCurrentTestResult(testResult);
if (testData == null)
{
throw new ArgumentNullException(nameof(testData));
}
var hasAnySteps = testData.Feature.Background.Steps
.Concat(testData.Scenario.Steps)
.Concat(testData.Rule?.Background?.Steps ?? Enumerable.Empty <IStep>())
.Any();
if (!hasAnySteps)
{
MarkTestAsSkipped(testResult);
}
else
{
using (var serviceScope = testRunContext.ServiceProvider.CreateScope())
{
// Before Scenario hooks should run here (see https://docs.cucumber.io/gherkin/reference/#background)
// > A Background is run before each scenario, but after any Before hooks. In your feature file, put the Background before the first Scenario.
IEnumerable <IStep> allScenarioSteps = testData.Feature.Background.Steps;
if (testData.Rule != null)
{
allScenarioSteps = allScenarioSteps.Concat(testData.Rule.Background.Steps);
}
allScenarioSteps = allScenarioSteps.Concat(testData.Scenario.Steps);
await ExecuteSteps(serviceScope.ServiceProvider, testResult, allScenarioSteps, testData, testRunContext)
.ConfigureAwait(false);
}
testResult.Outcome = TestOutcome.Passed;
}
testResult.Duration = TimeSpan.FromSeconds(
Math.Max(
SmallestTimeRecognisedByTestRunnerInSeconds,
(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency));
}
catch (Exception exception)
{
MarkTestAsFailed(testCase, testResult, exception, logger);
}
return(testResult);
}
private async Task ExecuteSteps(
IServiceProvider serviceProvider,
TestResult testResult,
IEnumerable <IStep> steps,
DiscoveredTestData testData,
TestRunContext testRunContext)
{
var scenarioExecutionStartTime = DateTime.UtcNow;
var stepsIterator = new StepsIterator(steps);
foreach (var step in stepsIterator.Iterate())
{
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{step.Title}{Environment.NewLine}"));
var stepBinding = stepBinder.GetBindingFor(step, testData.Assembly);
var executionStrategy = StepExecutionStrategyFactory.GetFor(stepBinding);
try
{
await executionStrategy
.Execute(stepBinding, serviceProvider, testResult.Messages, testRunContext)
.ConfigureAwait(false);
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{StepLogIndent}Completed at {DateTime.UtcNow:o}{Environment.NewLine}{Environment.NewLine}"));
}
catch (Exception exception)
{
if (ScenarioFailureIsTerminal(
testData.Scenario,
scenarioExecutionStartTime,
stepsIterator))
{
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{StepLogIndent}Failed at {DateTime.UtcNow:o}{Environment.NewLine}{Environment.NewLine}"));
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardErrorCategory,
$"{exception}{Environment.NewLine}"));
throw;
}
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{StepLogIndent}Failed at {DateTime.UtcNow:o}, waiting and retrying scenario from last When step{Environment.NewLine}"));
await Task.Delay(testData.Scenario.EventuallyConsistentConfiguration.RetryInterval).ConfigureAwait(false);
}
}
}
private async Task <TestResultMessage[]> PerformEventuallyConsistentStepsTest(MethodInfo method)
{
var scenarioStep = new WhenStep("When an exception is thrown", DataTable.Empty, null);
var testFeature = new Feature(
"Feature",
null,
Background.Empty,
new[]
{
new Scenario(
"Scenario",
new[] { scenarioStep },
1,
Enumerable.Empty <Tag>())
},
Enumerable.Empty <ScenarioOutline>(),
Enumerable.Empty <Rule>(),
Enumerable.Empty <Tag>());
var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
var binding = new StepBinding(scenarioStep, method, Array.Empty <object>());
mockStepBinder
.Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
.Returns(binding);
var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);
return(testResult
.Messages
.Where(
message => message.Category == TestResultMessage.StandardOutCategory)
.ToArray());
}
public async Task SetThreadUICultureToMatchFeatureCulture()
{
var scenarioStep = new GivenStep("Scenario step", DataTable.Empty, null);
var testFeature = new Feature("Feature", null,
Background.Empty,
new[]
{
new Scenario(
"Scenario",
new[] { scenarioStep },
1,
Enumerable.Empty <Tag>())
},
Enumerable.Empty <ScenarioOutline>(),
Enumerable.Empty <Rule>(),
new[] {
new Tag("culture(nb-NO)")
});
var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
string capturedCultureName = null;
var mockScenarioStepMapping = new Mock <IStepBinding>();
mockStepBinder
.Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
.Returns(mockScenarioStepMapping.Object)
.Callback(() => capturedCultureName = CultureInfo.CurrentUICulture.Name);
var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);
Assert.AreEqual("nb-NO", capturedCultureName);
}
private async Task <TestResultMessage[]> PerformEventuallyConsistentScenarioTest(MethodInfo givenMethod, MethodInfo whenMethod, MethodInfo thenMethod)
{
var givenStep = new GivenStep("Given", DataTable.Empty, null);
var whenStep = new WhenStep("When", DataTable.Empty, null);
var thenStep = new ThenStep("Then", DataTable.Empty, null);
var testFeature = new Feature(
"Feature",
null,
Background.Empty,
new[]
{
new Scenario(
"Scenario",
new IStep[] { givenStep, whenStep, thenStep },
1,
new List <Tag> {
new Tag("eventuallyConsistent(retryInterval=00:00:01;within=00:00:05)")
})
},
Enumerable.Empty <ScenarioOutline>(),
Enumerable.Empty <Rule>(),
Enumerable.Empty <Tag>());
var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
var givenStepBinding = new StepBinding(givenStep, givenMethod, Array.Empty <object>());
var whenStepBinding = new StepBinding(whenStep, whenMethod, Array.Empty <object>());
var thenStepBinding = new StepBinding(thenStep, thenMethod, Array.Empty <object>());
mockStepBinder
.Setup(m => m.GetBindingFor(givenStep, testAssembly))
.Returns(givenStepBinding);
mockStepBinder
.Setup(m => m.GetBindingFor(whenStep, testAssembly))
.Returns(whenStepBinding);
mockStepBinder
.Setup(m => m.GetBindingFor(thenStep, testAssembly))
.Returns(thenStepBinding);
var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);
return(testResult
.Messages
.Where(
message => message.Category == TestResultMessage.StandardOutCategory)
.ToArray());
}
public async Task ExecuteFeatureBackgroundStepsThenScenarioSteps()
{
var backgroundStep = new GivenStep("Feature background step", DataTable.Empty, null);
var scenarioStep = new GivenStep("Scenario step", DataTable.Empty, null);
var testFeature = new Feature("Feature", null,
new Background(
new[] { backgroundStep },
0),
new[]
{
new Scenario(
"Scenario",
new[] { scenarioStep },
1,
Enumerable.Empty <Tag>())
},
Enumerable.Empty <ScenarioOutline>(),
Enumerable.Empty <Rule>(),
Enumerable.Empty <Tag>());
var testData = new DiscoveredTestData(testAssembly, testFeature, null, testFeature.Scenarios.First());
var invocationOrder = 0;
var mockBackgroundStepMapping = new Mock <IStepBinding>();
mockStepBinder
.Setup(m => m.GetBindingFor(backgroundStep, testAssembly))
.Returns(mockBackgroundStepMapping.Object)
.Callback(() => Assert.AreEqual(0, invocationOrder++));
var mockScenarioStepMapping = new Mock <IStepBinding>();
mockStepBinder
.Setup(m => m.GetBindingFor(scenarioStep, testAssembly))
.Returns(mockScenarioStepMapping.Object)
.Callback(() => Assert.AreEqual(1, invocationOrder++));
var testResult = await stepsExecutor.Execute(testCase, testData, testRunContext, mockLogger.Object);
mockBackgroundStepMapping.Verify(m => m.Execute(It.IsAny <IServiceProvider>(), It.IsAny <Collection <TestResultMessage> >()), Times.Once);
mockScenarioStepMapping.Verify(m => m.Execute(It.IsAny <IServiceProvider>(), It.IsAny <Collection <TestResultMessage> >()), Times.Once);
}
public async Task ExecuteSteps(IServiceProvider serviceProvider, TestResult testResult, IEnumerable <IStep> steps, DiscoveredTestData testData, TestRunContext testRunContext)
{
var startTime = DateTime.UtcNow;
for (var stepIndex = 0; stepIndex < steps.Count(); stepIndex++)
{
var step = steps.ElementAt(stepIndex);
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{step.Title}{Environment.NewLine}"));
var stepBinding = stepBinder.GetBindingFor(step, testData.Assembly);
var executionStrategy = StepExecutionStrategyFactory.GetFor(stepBinding);
try
{
await executionStrategy
.Execute(stepBinding, serviceProvider, testResult.Messages, testRunContext)
.ConfigureAwait(false);
}
catch (Exception exception)
{
if (!testData.Scenario.IsEventuallyConsistent ||
DateTime.UtcNow > startTime.Add(testData.Scenario.EventuallyConsistentConfiguration.Within))
{
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{StepsExecutor.StepLogIndent}Failed{Environment.NewLine}{Environment.NewLine}"));
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardErrorCategory,
$"{exception}{Environment.NewLine}"));
throw;
}
var previousStepsInReverse = steps.Take(stepIndex + 1).Reverse();
var lastWhenStep = previousStepsInReverse.First(s => s is WhenStep);
stepIndex = steps.ToList().IndexOf(lastWhenStep);
await Task.Delay(testData.Scenario.EventuallyConsistentConfiguration.RetryInterval).ConfigureAwait(false);
continue;
}
testResult.Messages.Add(
new TestResultMessage(
TestResultMessage.StandardOutCategory,
$"{StepsExecutor.StepLogIndent}Completed{Environment.NewLine}{Environment.NewLine}"));
}
}
