public void It_should_capture_predefined_step_types()
{
_runner.Test().TestScenario(
TestStep.CreateWithTypeAsync("setup", Some_step),
TestStep.CreateWithTypeAsync("setup", Some_step),
TestStep.CreateWithTypeAsync("given", Some_step),
TestStep.CreateWithTypeAsync("given", Some_step),
TestStep.CreateWithTypeAsync("when", Some_step),
TestStep.CreateWithTypeAsync("when", Some_step),
TestStep.CreateWithTypeAsync("then", Some_step),
TestStep.CreateWithTypeAsync("then", Some_step),
TestStep.CreateWithTypeAsync("something else", Some_step),
TestStep.CreateWithTypeAsync("something else", Some_step)
);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 10, "SETUP Some step", ExecutionStatus.Passed),
new StepResultExpectation(2, 10, "AND Some step", ExecutionStatus.Passed),
new StepResultExpectation(3, 10, "GIVEN Some step", ExecutionStatus.Passed),
new StepResultExpectation(4, 10, "AND Some step", ExecutionStatus.Passed),
new StepResultExpectation(5, 10, "WHEN Some step", ExecutionStatus.Passed),
new StepResultExpectation(6, 10, "AND Some step", ExecutionStatus.Passed),
new StepResultExpectation(7, 10, "THEN Some step", ExecutionStatus.Passed),
new StepResultExpectation(8, 10, "AND Some step", ExecutionStatus.Passed),
new StepResultExpectation(9, 10, "SOMETHING ELSE Some step", ExecutionStatus.Passed),
new StepResultExpectation(10, 10, "AND Some step", ExecutionStatus.Passed)
);
}
public void Runner_should_properly_associate_steps_to_the_group()
{
Assert.DoesNotThrow(() => _runner.Test().TestGroupScenario(Passing_step_group, Composite_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 2, "Passing step group", ExecutionStatus.Passed),
new StepResultExpectation(2, 2, "Composite group", ExecutionStatus.Bypassed, $"Step 2.2.2: {BypassReason}"));
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("1.", 2, 3, "WHEN step two", ExecutionStatus.Passed),
new StepResultExpectation("1.", 3, 3, "THEN step three", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[1].GetSubSteps().ElementAt(0).GetSubSteps(),
new StepResultExpectation("2.1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("2.1.", 2, 3, "WHEN step two with comment", ExecutionStatus.Passed, null, CommentReason),
new StepResultExpectation("2.1.", 3, 3, "THEN step three", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[1].GetSubSteps().ElementAt(1).GetSubSteps(),
new StepResultExpectation("2.2.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("2.2.", 2, 3, "WHEN step two is bypassed", ExecutionStatus.Bypassed, $"Step 2.2.2: {BypassReason}"),
new StepResultExpectation("2.2.", 3, 3, "THEN step three", ExecutionStatus.Passed)
);
}
public void Runner_should_properly_capture_status_from_multiassert_steps()
{
Assert.Throws <AggregateException>(() => _runner.Test().TestGroupScenario(Multiassert_ignoring_steps, Multiassert_failing_composite, Passing_composite));
var scenario = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(scenario.Status, Is.EqualTo(ExecutionStatus.Failed));
var steps = scenario.GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Multiassert ignoring steps", ExecutionStatus.Ignored, $"Step 1.1: ignoring{Environment.NewLine}Step 1.2: ignoring"),
new StepResultExpectation(2, 3, "Multiassert failing composite", ExecutionStatus.Failed, $"Step 2.1: failing{Environment.NewLine}Step 2.2: failing"),
new StepResultExpectation(3, 3, "Passing composite", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "Ignoring step", ExecutionStatus.Ignored, "Step 1.1: ignoring"),
new StepResultExpectation("1.", 2, 3, "Ignoring step", ExecutionStatus.Ignored, "Step 1.2: ignoring"),
new StepResultExpectation("1.", 3, 3, "Passing step", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[1].GetSubSteps(),
new StepResultExpectation("2.", 1, 3, "Failing step", ExecutionStatus.Failed, "Step 2.1: failing"),
new StepResultExpectation("2.", 2, 3, "Failing step", ExecutionStatus.Failed, "Step 2.2: failing"),
new StepResultExpectation("2.", 3, 3, "Passing step", ExecutionStatus.Passed)
);
StepResultExpectation.AssertEqual(steps[2].GetSubSteps(),
new StepResultExpectation("3.", 1, 2, "Passing step", ExecutionStatus.Passed),
new StepResultExpectation("3.", 2, 2, "Passing step", ExecutionStatus.Passed)
);
}
public void It_should_infer_and_capture_default_step_types()
{
_runner.Test().TestScenario(
Setup_before_steps,
Setup_before_steps,
Given_step_one,
Given_step_one,
When_step_two,
When_step_two,
Then_step_three,
Then_step_four,
Then_step_five,
Some_step
);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 10, "SETUP before steps", ExecutionStatus.Passed),
new StepResultExpectation(2, 10, "AND before steps", ExecutionStatus.Passed),
new StepResultExpectation(3, 10, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation(4, 10, "AND step one", ExecutionStatus.Passed),
new StepResultExpectation(5, 10, "WHEN step two", ExecutionStatus.Passed),
new StepResultExpectation(6, 10, "AND step two", ExecutionStatus.Passed),
new StepResultExpectation(7, 10, "THEN step three", ExecutionStatus.Passed),
new StepResultExpectation(8, 10, "AND step four", ExecutionStatus.Passed),
new StepResultExpectation(9, 10, "AND step five", ExecutionStatus.Passed),
new StepResultExpectation(10, 10, "Some step", ExecutionStatus.Passed)
);
}
public void Runner_should_capture_details_about_sub_step_initialization_failure()
{
Assert.Throws <InvalidOperationException>(() => _runner.Test().TestGroupScenario(Incorrect_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Incorrect step group", ExecutionStatus.Failed, "Step 1: Sub-steps initialization failed: abc"));
}
public void Runner_should_execute_all_steps_within_group()
{
Assert.DoesNotThrow(() => _runner.Test().TestGroupScenario(Passing_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Passing step group", ExecutionStatus.Passed));
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("1.", 2, 3, "WHEN step two", ExecutionStatus.Passed),
new StepResultExpectation("1.", 3, 3, "THEN step three", ExecutionStatus.Passed)
);
}
public void It_should_capture_bypassed_steps()
{
_runner.Test().TestScenario(
Given_step_one,
When_step_two_is_bypassed,
Then_step_three);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step two is bypassed", ExecutionStatus.Bypassed, $"Step 2: {BypassReason}"),
new StepResultExpectation(3, 3, "THEN step three", ExecutionStatus.Passed)
);
}
public void It_should_capture_steps_with_parameters_inserted_in_proper_places()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Method_with_replaced_parameter_PARAM_in_name, "abc"),
TestStep.CreateAsync(Method_with_inserted_parameter_param_in_name, "abc"),
TestStep.CreateAsync(Method_with_appended_parameter_at_the_end_of_name, "abc"));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Method with replaced parameter \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "Method with inserted parameter param \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "Method with appended parameter at the end of name [param: \"abc\"]", ExecutionStatus.Passed)
);
}
public void It_should_capture_steps_with_parameters()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, () => "abc"),
TestStep.CreateAsync(When_step_with_parameter, () => 1),
TestStep.CreateAsync(Then_step_with_parameter, () => 3.14));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step with parameter \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step with parameter \"1\"", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "THEN step with parameter \"3.14\"", ExecutionStatus.Passed)
);
}
public void Runner_should_mark_step_ignored_if_substep_is_ignored()
{
Assert.Throws <CustomIgnoreException>(() => _runner.Test().TestGroupScenario(Ignored_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Ignored step group", ExecutionStatus.Ignored, $"Step 1.2: {IgnoreReason}"));
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("1.", 2, 3, "WHEN step two ignoring scenario", ExecutionStatus.Ignored, $"Step 1.2: {IgnoreReason}"),
new StepResultExpectation("1.", 3, 3, "THEN step three", ExecutionStatus.NotRun)
);
}
public void Runner_should_mark_step_failed_if_substep_fails()
{
Assert.Throws <InvalidOperationException>(() => _runner.Test().TestGroupScenario(Failing_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Failing step group", ExecutionStatus.Failed, $"Step 1.2: {ExceptionReason}"));
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("1.", 2, 3, "WHEN step two throwing exception", ExecutionStatus.Failed, $"Step 1.2: {ExceptionReason}"),
new StepResultExpectation("1.", 3, 3, "THEN step three", ExecutionStatus.NotRun)
);
}
public void Runner_should_mark_step_bypassed_if_substep_is_bypassed()
{
Assert.DoesNotThrow(() => _runner.Test().TestGroupScenario(Bypassed_step_group));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 1, "Bypassed step group", ExecutionStatus.Bypassed, $"Step 1.2: {BypassReason}"));
StepResultExpectation.AssertEqual(steps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation("1.", 2, 3, "WHEN step two is bypassed", ExecutionStatus.Bypassed, $"Step 1.2: {BypassReason}"),
new StepResultExpectation("1.", 3, 3, "THEN step three", ExecutionStatus.Passed)
);
}
public void It_should_collect_results_for_scenarios_causing_formatting_failures()
{
var expectedErrorMessage = "Unable to format 'param' parameter of step '1/1 Method with wrong formatter param \"<?>\"': Input string was not in a correct format.";
var ex = Assert.Throws <InvalidOperationException>(() => _runner.Test().TestScenario(TestStep.CreateAsync(Method_with_wrong_formatter_param, () => "abc")));
Assert.That(ex.Message, Is.EqualTo(expectedErrorMessage));
var result = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(result.Status, Is.EqualTo(ExecutionStatus.Failed));
Assert.That(result.StatusDetails, Is.EqualTo("Step 1: " + expectedErrorMessage));
StepResultExpectation.AssertEqual(result.GetSteps(),
new StepResultExpectation(1, 1, "Method with wrong formatter param \"<?>\"", ExecutionStatus.Failed, expectedErrorMessage));
}
public void It_should_capture_step_status_with_custom_exception_mapping()
{
var feature = TestableFeatureRunnerRepository.GetRunner(GetType());
var runner = feature.GetBddRunner(this);
Assert.Throws <CustomIgnoreException>(() => runner.Test().TestScenario(
Given_step_one,
When_step_two_ignoring_scenario,
Then_step_three));
var steps = feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step two ignoring scenario", ExecutionStatus.Ignored, $"Step 2: {IgnoreReason}"),
new StepResultExpectation(3, 3, "THEN step three", ExecutionStatus.NotRun));
}
public void Invalid_step_descriptors_should_make_scenario_failing_immediately_without_execution()
{
var step1 = TestStep.Create(Step_that_should_not_run);
var step2 = StepDescriptor.CreateInvalid(new Exception("reason1"));
var step3 = StepDescriptor.CreateInvalid(new Exception("reason2"));
var ex = Assert.Throws <AggregateException>(() => _runner.Test().TestScenario(step1, step2, step3));
Assert.That(ex.InnerExceptions.Select(x => x.Message).ToArray(),
Is.EqualTo(new[] { "Scenario steps initialization failed.", "reason1", "reason2" }));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Step that should not run", ExecutionStatus.NotRun),
new StepResultExpectation(2, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 2: reason1"),
new StepResultExpectation(3, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 3: reason2"));
}
public void It_should_capture_failed_steps()
{
try
{
_runner.Test().TestScenario(
Given_step_one,
When_step_two_throwing_exception,
Then_step_three);
}
catch { }
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step one", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step two throwing exception", ExecutionStatus.Failed, $"Step 2: {ExceptionReason}"),
new StepResultExpectation(3, 3, "THEN step three", ExecutionStatus.NotRun)
);
}
public async Task It_should_parse_composite_names()
{
var featureRunner = TestableFeatureRunnerRepository.GetRunner(GetType());
var runner = featureRunner.GetBddRunner(this);
var expectedValue = "Some\r\nmultiline\tresponse";
await runner
.AddStep("Given my scenario with value [55]", _ => { })
.AddStep(" When I send My_Request<int> for it ", _ => { })
.AddAsyncStep($"Only then I should receive \"{expectedValue}\"", _ => Task.CompletedTask)
.RunAsync();
var steps = featureRunner.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN my scenario with value [55]", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN I send My_Request<int> for it", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "Only then I should receive \"Some multiline response\"", ExecutionStatus.Passed));
}
public void It_should_capture_steps_with_parameters_and_failing_parameter_evaluation()
{
var ex = Assert.Throws <InvalidOperationException>(() =>
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, () => "def"),
TestStep.CreateAsync(When_step_with_parameter, ThrowingParameterInvocation),
TestStep.CreateAsync(Then_step_with_parameter, () => 3.27));
});
Assert.That(ex.Message, Is.EqualTo(ParameterExceptionReason));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step with parameter \"def\"", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step with parameter \"<?>\"", ExecutionStatus.Failed, ParameterExceptionReason),
new StepResultExpectation(3, 3, "THEN step with parameter \"<?>\"", ExecutionStatus.NotRun)
);
}
public void It_should_capture_steps_with_parameters_inserted_in_proper_places()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Method_with_replaced_parameter_PARAM_in_name, "abc"),
TestStep.CreateAsync(Method_with_inserted_parameter_param_in_name, "abc"),
TestStep.CreateAsync(Method_with_appended_parameter_at_the_end_of_name, "abc"),
TestStep.CreateAsync(ExtensionSteps.Extension_method_with_parameter_PARAM, "target", "abc"),
TestStep.CreateAsync(Method_with_param1_param2_param3, "abc", "def", "123"),
TestStep.CreateAsync(Method_with_appended_and_normal_param, "abc", "def", "123")
);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 6, "Method with replaced parameter \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(2, 6, "Method with inserted parameter param \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(3, 6, "Method with appended parameter at the end of name [param: \"abc\"]", ExecutionStatus.Passed),
new StepResultExpectation(4, 6, "Extension method with parameter \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(5, 6, "Method with param1 \"def\" param2 \"123\" param3 \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(6, 6, "Method with appended and normal param \"def\" [appended1: \"abc\"] [appended2: \"123\"]", ExecutionStatus.Passed)
);
}
public void Invalid_step_descriptors_should_make_composite_failing_immediately_without_execution()
{
TestCompositeStep Composite_step() => new TestCompositeStep(
TestStep.Create(Step_that_should_not_run),
StepDescriptor.CreateInvalid(new Exception("reason1")),
StepDescriptor.CreateInvalid(new Exception("reason2")));
var ex = Assert.Throws <AggregateException>(() => _runner.Test()
.TestScenario(
TestStep.CreateComposite(Composite_step),
TestStep.Create(Step_that_should_not_run)));
Assert.That(ex.InnerExceptions.Select(x => x.Message).ToArray(),
Is.EqualTo(new[] { "Sub-steps initialization failed.", "reason1", "reason2" }));
var mainSteps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
StepResultExpectation.AssertEqual(mainSteps[0].GetSubSteps(),
new StepResultExpectation("1.", 1, 3, "Step that should not run", ExecutionStatus.NotRun),
new StepResultExpectation("1.", 2, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 1.2: reason1"),
new StepResultExpectation("1.", 3, 3, "<INVALID STEP>", ExecutionStatus.Failed, "Step 1.3: reason2"));
Assert.That(mainSteps[1].Status, Is.EqualTo(ExecutionStatus.NotRun));
}