public void Running_scenarios_should_be_thread_safe()
{
Enumerable.Range(0, _elementsCount)
.ToArray()
.AsParallel()
.ForAll(Parallel_scenario);
Assert.That(_feature.GetFeatureResult().GetScenarios().Count(), Is.EqualTo(_elementsCount));
for (var i = 0; i < _elementsCount; ++i)
{
var expectedScenarioName = $"Parallel scenario \"{i}\"";
var expectedSteps = new[]
{
$"GIVEN step with parameter \"{i}\"",
$"WHEN step with parameter \"{i}\" and comments",
$"THEN step with parameter \"{i}\""
};
var expectedComments = new[] { CommentReason, $"{i}" };
var scenario = _feature.GetFeatureResult().GetScenarios().FirstOrDefault(s => s.Info.Name.ToString() == expectedScenarioName);
Assert.That(scenario, Is.Not.Null, "Missing scenario: {0}", i);
Assert.That(scenario.GetSteps().Select(s => s.Info.Name.ToString()).ToArray(), Is.EqualTo(expectedSteps), $"In scenario {i}");
Assert.That(scenario.GetSteps().ElementAt(1).Comments, Is.EqualTo(expectedComments), $"In scenario {i}");
}
}
public async Task It_should_support_parallel_asynchronous_scenario_processing()
{
var elementsCount = 2000;
var scenarios = Enumerable.Range(0, elementsCount).Select(Parallel_scenario);
await Task.WhenAll(scenarios);
Assert.That(_feature.GetFeatureResult().GetScenarios().Count(), Is.EqualTo(elementsCount));
for (var i = 0; i < elementsCount; ++i)
{
var expectedScenarioName = $"Parallel scenario \"{i}\"";
var expectedSteps = new[]
{
$"GIVEN step with parameter \"{i}\"",
$"WHEN step with parameter \"{i}\" and comments",
$"THEN step with parameter \"{i}\""
};
var expectedComments = new[] { CommentReason, $"{i}" };
var scenario = _feature.GetFeatureResult().GetScenarios().FirstOrDefault(s => s.Info.Name.ToString() == expectedScenarioName);
Assert.That(scenario, Is.Not.Null, "Missing scenario: {0}", i);
Assert.That(scenario.GetSteps().Select(s => s.Info.Name.ToString()).ToArray(), Is.EqualTo(expectedSteps), $"In scenario {i}");
Assert.That(scenario.GetSteps().ElementAt(1).Comments, Is.EqualTo(expectedComments), $"In scenario {i}");
}
}
public void It_should_capture_scenario_name_with_global_categories()
{
_runner.Test().TestScenario(Some_step);
var scenario = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(scenario.Info.Categories, Is.EqualTo(new[] { "global1", "global2" }));
}
public void It_should_capture_all_steps()
{
_runner.Test().TestScenario(
Given_step_one,
When_step_two,
Then_step_three);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step two", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "THEN step three", ExecutionStatus.Passed)
);
}
public void Execution_results_should_print_user_friendly_output()
{
try
{
_runner.Test().TestScenario(
Given_step_one,
When_step_two,
Then_step_three);
}
catch { }
var featureResult = _feature.GetFeatureResult();
Assert.That(featureResult.ToString(), Is.EqualTo("[Ticket-1][Ticket-2] FeatureFixtureRunner execution results tests"));
var scenarioResult = featureResult.GetScenarios().Single();
Assert.That(scenarioResult.ToString(), Is.EqualTo("[Label-1][Label-2] Execution results should print user friendly output: Failed (Step 3: reason)"));
Assert.That(scenarioResult.GetSteps().Select(s => s.ToString()).ToArray(), Is.EqualTo(new[]
{
"1/3 GIVEN step one: Passed",
"2/3 WHEN step two: Passed",
"3/3 THEN step three: Failed (Step 3: reason)"
}));
}
public void Runner_should_properly_capture_status_from_multiassert_steps()
{
Assert.Throws <AggregateException>(() => _runner.Test().TestGroupScenario(Multiassert_ignoring_steps, Multiassert_failing_composite, Passing_composite));
var scenario = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(scenario.Status, Is.EqualTo(ExecutionStatus.Failed));
var steps = scenario.GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Multiassert ignoring steps", ExecutionStatus.Ignored, $"Step 1.1: ignoring{Environment.NewLine}Step 1.2: ignoring"),
new StepResultExpectation(2, 3, "Multiassert failing composite", ExecutionStatus.Failed, $"Step 2.1: failing{Environment.NewLine}Step 2.2: failing"),
new StepResultExpectation(3, 3, "Passing composite", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "Ignoring step", ExecutionStatus.Ignored, "Step 1.1: ignoring"),
new StepResultExpectation("1.", 2, 3, "Ignoring step", ExecutionStatus.Ignored, "Step 1.2: ignoring"),
new StepResultExpectation("1.", 3, 3, "Passing step", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[1].GetSubSteps(),
new StepResultExpectation("2.", 1, 3, "Failing step", ExecutionStatus.Failed, "Step 2.1: failing"),
new StepResultExpectation("2.", 2, 3, "Failing step", ExecutionStatus.Failed, "Step 2.2: failing"),
new StepResultExpectation("2.", 3, 3, "Passing step", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[2].GetSubSteps(),
new StepResultExpectation("3.", 1, 2, "Passing step", ExecutionStatus.Passed),
new StepResultExpectation("3.", 2, 2, "Passing step", ExecutionStatus.Passed)
);
}
public void It_should_capture_scenario_name()
{
_runner.Test().TestScenario(Some_step);
var scenario = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(scenario.Info.Name.ToString(), Is.EqualTo("It should capture scenario name"));
Assert.That(scenario.Info.Labels, Is.Empty);
Assert.That(scenario.Info.Categories, Is.Empty);
}
public void Invalid_step_descriptors_should_make_scenario_failing_immediately_without_execution()
{
var step1 = TestStep.Create(Step_that_should_not_run);
var step2 = StepDescriptor.CreateInvalid(new Exception("reason1"));
var step3 = StepDescriptor.CreateInvalid(new Exception("reason2"));
var ex = Assert.Throws <AggregateException>(() => _runner.Test().TestScenario(step1, step2, step3));
Assert.That(ex.InnerExceptions.Select(x => x.Message).ToArray(),
Is.EqualTo(new[] { "Scenario steps initialization failed.", "reason1", "reason2" }));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Step that should not run", ExecutionStatus.NotRun),
new StepResultExpectation(2, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 2: reason1"),
new StepResultExpectation(3, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 3: reason2"));
}
public void Runner_should_instantiate_context_just_before_run_so_its_failure_would_be_included_in_results()
{
Assert.Throws <InvalidOperationException>(() => _runner.Test()
.WithContext(() => throw new InvalidOperationException("abc"), false)
.TestScenario(Given_step_one));
var scenario = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(scenario.Status, Is.EqualTo(ExecutionStatus.Failed));
Assert.That(scenario.StatusDetails, Is.EqualTo("Scenario: Scenario context initialization failed: abc"));
}
public void It_should_capture_all_steps()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, "abc"),
TestStep.CreateAsync(When_step_with_parameter, 123),
TestStep.CreateAsync(Then_step_with_parameter, 3.15));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step with parameter \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step with parameter \"123\"", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "THEN step with parameter \"3.15\"", ExecutionStatus.Passed)
);
}
public void Runner_should_capture_details_about_sub_step_initialization_failure()
{
Assert.Throws <InvalidOperationException>(() => _runner.Test().TestGroupScenario(Incorrect_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Incorrect step group", ExecutionStatus.Failed, "Step 1: Sub-steps initialization failed: abc"));
}
private void AssertScenarioExecutionTime(Action runScenario)
{
var startTime = DateTimeOffset.UtcNow;
var watch = Stopwatch.StartNew();
try { runScenario(); }
catch { }
watch.Stop();
var result = _feature.GetFeatureResult().GetScenarios().Single();
FormatTime("Measure time", new ExecutionTime(startTime, watch.Elapsed));
FormatTime("Scenario time", result.ExecutionTime);
Assert.That(result.ExecutionTime, Is.Not.Null);
Assert.That(result.ExecutionTime.Duration, Is.LessThanOrEqualTo(watch.Elapsed), "Scenario.ExecutionTime.Duration");
Assert.That(result.ExecutionTime.Start, Is
.GreaterThanOrEqualTo(startTime)
.And
.LessThan(startTime.Add(watch.Elapsed).Add(UtcNowClockPrecision)), "Scenario.ExecutionTime.Start");
AssertStepsExecutionTimesAreDoneInOrder();
}
public void Disposed_runner_should_allow_retrieving_results()
{
_feature.Dispose();
Assert.DoesNotThrow(() => _feature.GetFeatureResult());
}