public void It_should_notify_execution_progress_for_parameterized_steps()
{
var progressNotifier = new CapturingProgressNotifier();
var feature = new TestableFeatureRunnerRepository(progressNotifier, fixture => progressNotifier).GetRunnerFor(GetType());
var runner = feature.GetBddRunner(this);
try
{
runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, () => "abc"),
TestStep.CreateAsync(When_step_with_parameter, ThrowingParameterInvocation),
TestStep.CreateAsync(Then_step_with_parameter, () => 2.22));
}
catch
{
}
feature.Dispose();
string[] expected =
{
"Feature Start: CoreBddRunner progress notification tests [label1, label2]: feature description",
"Scenario Start: It should notify execution progress for parameterized steps [lab1, lab2] <category 1, category 2>",
"Step Start: 1/3 GIVEN step with parameter \"abc\"",
"Step Finish: 1/3 GIVEN step with parameter \"abc\" | Status:Passed | ExecutionTimePresent:True | Details:",
"Scenario Finish: It should notify execution progress for parameterized steps [lab1, lab2] <category 1, category 2> | Status:Failed | ExecutionTimePresent:True | Steps:3 | Details:Step 2: parameter exception",
"Feature Finish: CoreBddRunner progress notification tests [label1, label2]: feature description | Scenarios:1"
};
Assert.That(progressNotifier.Notifications, Is.EqualTo(expected), "Expected:\r\n{0}\r\n\r\nGot:\r\n{1}\r\n\r\n", string.Join("\r\n", expected), string.Join("\r\n", progressNotifier.Notifications));
}
public void Runner_should_honor_bypassed_steps()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Bypassed_step_with_parameter,
new Complex(null, null, ParameterVerificationStatus.Failure, "msg")
));
Assert.That(GetStepResults().Single().Status, Is.EqualTo(ExecutionStatus.Bypassed));
}
public void Runner_should_honor_ignored_steps()
{
Assert.Throws <CustomIgnoreException>(() => _runner.Test().TestScenario(
TestStep.CreateAsync(Ignored_step_with_parameter,
new Complex(null, null, ParameterVerificationStatus.Failure, "msg")
)));
Assert.That(GetStepResults().Single().Status, Is.EqualTo(ExecutionStatus.Ignored));
}
public void Comment_should_ignore_empty_comments(string comment)
{
var feature = GetFeatureRunner();
var runner = feature.GetBddRunner(this);
runner.Test().TestScenario(TestStep.CreateAsync(Commented_step, comment));
Assert.That(feature.GetFeatureResult().GetScenarios().Single().GetSteps().Single().Comments.ToArray(), Is.Empty);
}
private void Parallel_scenario(int idx)
{
var steps = new[]
{
TestStep.CreateAsync(Given_step_with_parameter, idx.ToString()),
TestStep.CreateAsync(When_step_with_parameter_and_comments, idx),
TestStep.CreateAsync(Then_step_with_parameter, (double)idx)
};
_runner.Test().TestNamedScenario($"Parallel scenario \"{idx}\"", steps);
}
public void Runner_should_process_failed_steps_first()
{
Assert.Throws <Exception>(() => _runner.Test().TestScenario(
TestStep.CreateAsync(Failed_step_with_parameter,
new Complex(null, null, ParameterVerificationStatus.Failure, "msg")
)));
var result = GetStepResults().Single();
Assert.That(result.Status, Is.EqualTo(ExecutionStatus.Failed));
Assert.That(result.StatusDetails, Is.EqualTo("Step 1: exception reason"));
}
public void Runner_should_include_complex_parameters_for_failed_steps()
{
Assert.Throws <Exception>(() => _runner.Test().TestScenario(
TestStep.CreateAsync(Failed_step_with_parameter,
new Complex(null, null, ParameterVerificationStatus.Failure, "msg")
)));
var result = GetStepResults().Single();
var parameter = result.Parameters.SingleOrDefault();
Assert.That(parameter, Is.Not.Null);
Assert.That(parameter.Details.VerificationStatus, Is.EqualTo(ParameterVerificationStatus.Failure));
Assert.That(parameter.Details.VerificationMessage, Is.EqualTo("msg"));
}
public void Runner_should_capture_complex_parameter_results()
{
Assert.Throws <InvalidOperationException>(() => _runner.Test().TestScenario(
TestStep.CreateAsync(Step_with_parameters,
Complex.Failed(),
Complex.NotProvided(),
Complex.Exception())));
var result = GetStepResults().Single();
Assert.That(result.Parameters.Count, Is.EqualTo(3));
AssertParameter(result.Parameters[0], "arg1", ParameterVerificationStatus.Failure, $"{ExpectedText}/{ValueText}");
AssertParameter(result.Parameters[1], "arg2", ParameterVerificationStatus.NotProvided, $"{ExpectedText}");
AssertParameter(result.Parameters[2], "arg3", ParameterVerificationStatus.Exception, $"{ExceptionText}");
}
public void It_should_capture_steps_with_parameters_inserted_in_proper_places()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Method_with_replaced_parameter_PARAM_in_name, "abc"),
TestStep.CreateAsync(Method_with_inserted_parameter_param_in_name, "abc"),
TestStep.CreateAsync(Method_with_appended_parameter_at_the_end_of_name, "abc"));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "Method with replaced parameter \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "Method with inserted parameter param \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "Method with appended parameter at the end of name [param: \"abc\"]", ExecutionStatus.Passed)
);
}
public void It_should_capture_steps_with_parameters()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, () => "abc"),
TestStep.CreateAsync(When_step_with_parameter, () => 1),
TestStep.CreateAsync(Then_step_with_parameter, () => 3.14));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step with parameter \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step with parameter \"1\"", ExecutionStatus.Passed),
new StepResultExpectation(3, 3, "THEN step with parameter \"3.14\"", ExecutionStatus.Passed)
);
}
public void Runner_should_fail_step_with_non_successful_parameters(ParameterVerificationStatus status)
{
var ex = Assert.Throws <InvalidOperationException>(() => _runner.Test().TestScenario(
TestStep.CreateAsync(Step_with_parameters,
new Complex(null, null, status, "msg1"),
new Complex(null, null, ParameterVerificationStatus.Success, "msg2"),
new Complex(null, null, status, "msg3")
)));
Assert.That(ex.Message, Is.EqualTo($"Parameter \'arg1\' verification failed: msg1{Environment.NewLine}Parameter \'arg3\' verification failed: msg3"));
var step = GetStepResults().Single();
Assert.That(step.Status, Is.EqualTo(ExecutionStatus.Failed));
Assert.That(step.StatusDetails, Is.EqualTo($"Step 1: Parameter \'arg1\' verification failed: msg1{Environment.NewLine}\tParameter \'arg3\' verification failed: msg3"));
}
public void Runner_should_evaluate_step_parameters_just_before_step_execution()
{
var ex = Assert.Throws <Exception>(() =>
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_one, () => "def"),
TestStep.CreateAsync <int>(When_step_two, () => { throw new Exception("reason"); }),
TestStep.CreateAsync(Then_step_three, () => 3.14));
});
Assert.That(ex.Message, Is.EqualTo("reason"));
var expected = new[] { Tuple.Create("Given_step_one", (object)"def") };
Assert.That(_executedSteps, Is.EqualTo(expected));
}
public void Runner_should_call_steps_with_parameters()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_one, "abc"),
TestStep.CreateAsync(When_step_two, 123),
TestStep.CreateAsync(Then_step_three, 3.25));
var expected = new[]
{
Tuple.Create("Given_step_one", (object)"abc"),
Tuple.Create("When_step_two", (object)123),
Tuple.Create("Then_step_three", (object)3.25)
};
Assert.That(_executedSteps, Is.EqualTo(expected));
}
public void Comment_should_record_comment_in_currently_executed_step()
{
var feature = GetFeatureRunner();
var runner = feature.GetBddRunner(this);
var comment = "abc";
var otherComment = "def";
runner.Test().TestScenario(
TestStep.CreateAsync(Commented_step, comment),
TestStep.CreateAsync(Commented_step, otherComment));
var steps = feature.GetFeatureResult().GetScenarios().Single().GetSteps().ToArray();
Assert.That(steps[0].Comments.ToArray(), Is.EqualTo(new[] { comment, comment }));
Assert.That(steps[1].Comments.ToArray(), Is.EqualTo(new[] { otherComment, otherComment }));
}
public void Runner_should_evaluate_step_parameters_once()
{
int number = 0;
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_one, () => (++number).ToString()),
TestStep.CreateAsync(When_step_two, () => ++ number),
TestStep.CreateAsync(Then_step_three, () => (double)++number));
var expected = new[]
{
Tuple.Create("Given_step_one", (object)"1"),
Tuple.Create("When_step_two", (object)2),
Tuple.Create("Then_step_three", (object)3.0)
};
Assert.That(_executedSteps, Is.EqualTo(expected));
}
public void It_should_capture_steps_with_parameters_and_failing_parameter_evaluation()
{
var ex = Assert.Throws <InvalidOperationException>(() =>
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_with_parameter, () => "def"),
TestStep.CreateAsync(When_step_with_parameter, ThrowingParameterInvocation),
TestStep.CreateAsync(Then_step_with_parameter, () => 3.27));
});
Assert.That(ex.Message, Is.EqualTo(ParameterExceptionReason));
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 3, "GIVEN step with parameter \"def\"", ExecutionStatus.Passed),
new StepResultExpectation(2, 3, "WHEN step with parameter \"<?>\"", ExecutionStatus.Failed, ParameterExceptionReason),
new StepResultExpectation(3, 3, "THEN step with parameter \"<?>\"", ExecutionStatus.NotRun)
);
}
public void Runner_should_call_steps_with_parameters()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Given_step_one, "abc"),
TestStep.CreateAsync(When_step_two, 123),
TestStep.CreateAsync(Then_step_three, 3.25),
TestStep.CreateAsync(Then_another_step, (double?)3.25),
TestStep.CreateAsync(Then_another_step, (double?)null));
var expected = new[]
{
Tuple.Create(nameof(Given_step_one), (object)"abc"),
Tuple.Create(nameof(When_step_two), (object)123),
Tuple.Create(nameof(Then_step_three), (object)3.25),
Tuple.Create(nameof(Then_another_step), (object)3.25),
Tuple.Create(nameof(Then_another_step), (object)null)
};
Assert.That(_executedSteps, Is.EqualTo(expected));
}
public void It_should_capture_steps_with_parameters_inserted_in_proper_places()
{
_runner.Test().TestScenario(
TestStep.CreateAsync(Method_with_replaced_parameter_PARAM_in_name, "abc"),
TestStep.CreateAsync(Method_with_inserted_parameter_param_in_name, "abc"),
TestStep.CreateAsync(Method_with_appended_parameter_at_the_end_of_name, "abc"),
TestStep.CreateAsync(ExtensionSteps.Extension_method_with_parameter_PARAM, "target", "abc"),
TestStep.CreateAsync(Method_with_param1_param2_param3, "abc", "def", "123"),
TestStep.CreateAsync(Method_with_appended_and_normal_param, "abc", "def", "123")
);
var steps = _feature.GetFeatureResult().GetScenarios().Single().GetSteps();
StepResultExpectation.AssertEqual(steps,
new StepResultExpectation(1, 6, "Method with replaced parameter \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(2, 6, "Method with inserted parameter param \"abc\" in name", ExecutionStatus.Passed),
new StepResultExpectation(3, 6, "Method with appended parameter at the end of name [param: \"abc\"]", ExecutionStatus.Passed),
new StepResultExpectation(4, 6, "Extension method with parameter \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(5, 6, "Method with param1 \"def\" param2 \"123\" param3 \"abc\"", ExecutionStatus.Passed),
new StepResultExpectation(6, 6, "Method with appended and normal param \"def\" [appended1: \"abc\"] [appended2: \"123\"]", ExecutionStatus.Passed)
);
}
public void Comment_should_throw_exception_if_feature_is_not_enabled()
{
var runner = new TestableFeatureRunnerRepository(
TestableIntegrationContextBuilder.Default()
.WithConfiguration(cfg => cfg.ExecutionExtensionsConfiguration().EnableScenarioExecutionContext())
)
.GetRunnerFor(GetType())
.GetBddRunner(this);
var exception = Assert.Throws <InvalidOperationException>(() => runner.Test().TestScenario(TestStep.CreateAsync(Commented_step, "some comment")));
Assert.That(exception.Message, Is.EqualTo("Current task is not executing any scenario steps or commenting feature is not enabled in LightBddConfiguration. Ensure that configuration.ExecutionExtensionsConfiguration().EnableStepCommenting() is called during LightBDD initialization and commenting feature is called from task running scenario step."));
}
public void Integration_should_not_allow_running_asynchronous_tests_synchronously()
{
var ex = Assert.Throws <InvalidOperationException>(() => _runner.Test().TestScenarioPurelySync(TestStep.CreateAsync(() => { })));
Assert.That(ex.Message, Is.EqualTo("Only steps being completed upon return can be run synchronously (all steps have to return completed task). Consider using Async scenario methods for async Task or async void steps."));
}
public void It_should_collect_results_for_scenarios_causing_formatting_failures()
{
var expectedErrorMessage = "Unable to format 'param' parameter of step '1/1 Method with wrong formatter param \"<?>\"': Input string was not in a correct format.";
var ex = Assert.Throws <InvalidOperationException>(() => _runner.Test().TestScenario(TestStep.CreateAsync(Method_with_wrong_formatter_param, () => "abc")));
Assert.That(ex.Message, Is.EqualTo(expectedErrorMessage));
var result = _feature.GetFeatureResult().GetScenarios().Single();
Assert.That(result.Status, Is.EqualTo(ExecutionStatus.Failed));
Assert.That(result.StatusDetails, Is.EqualTo("Step 1: " + expectedErrorMessage));
StepResultExpectation.AssertEqual(result.GetSteps(),
new StepResultExpectation(1, 1, "Method with wrong formatter param \"<?>\"", ExecutionStatus.Failed, expectedErrorMessage));
}
