public void ShouldFailWithClearExplanationWhenParameterCountsAreMismatched()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.Parameters(Inputs(
new object[] { },
new object[] { 0 },
new object[] { 0, 1 },
new object[] { 0, 1, 2 },
new object[] { 0, 1, 2, 3 }));
convention.Execute(listener, typeof(ParameterizedTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg(0) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg(0, 1) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg(0, 1, 2) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg(0, 1, 2, 3) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(0) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(0, 1) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(0, 1, 2) failed: Expected sum of 2 but was 1.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(0, 1, 2, 3) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs(0) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs(0, 1) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs(0, 1, 2) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs(0, 1, 2, 3) failed: Parameter count mismatch.");
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build());
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"Test '" + testClass + ".SkipWithReason' skipped: Skipped due to naming convention.",
"Test '" + testClass + ".SkipWithoutReason' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"3 passed, 2 failed, 2 skipped, took 1.23 seconds (Fixie 1.2.3.4).");
}
}
public void ShouldResolveGenericTypeParameters()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.Parameters(ParametersFromAttributes);
convention.Execute(listener, typeof(GenericTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.GenericMethodWithIncorrectParameterCountProvided<System.Object>(123, 123) failed: Parameter count mismatch.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.GenericMethodWithNoInputsProvided<System.Object> failed: This parameterized test could not be executed, because no input values were available.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.MultipleGenericArgumentsMultipleParameters<System.Int32, System.Object>(123, null, 456, System.Int32, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.MultipleGenericArgumentsMultipleParameters<System.Int32, System.String>(123, \"stringArg1\", 456, System.Int32, System.String) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.MultipleGenericArgumentsMultipleParameters<System.String, System.Object>(\"stringArg\", null, null, System.String, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.MultipleGenericArgumentsMultipleParameters<System.String, System.Object>(\"stringArg1\", null, \"stringArg2\", System.String, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.MultipleGenericArgumentsMultipleParameters<System.String, System.String>(null, \"stringArg1\", \"stringArg2\", System.String, System.String) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgument<System.Int32>(123, System.Int32) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgument<System.Object>(null, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgument<System.String>(\"stringArg\", System.String) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.Int32>(123, 456, System.Int32) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.Object>(\"stringArg\", 123, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.Object>(123, \"stringArg\", System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.Object>(123, null, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.Object>(null, null, System.Object) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.String>(\"stringArg\", null, System.String) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.String>(\"stringArg1\", \"stringArg2\", System.String) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+GenericTestClass.SingleGenericArgumentMultipleParameters<System.String>(null, \"stringArg\", System.String) passed.");
}
public void ShouldNotAffectOutputByDefault()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var convention = new SelfTestConvention();
var conventionRunner = new ConventionRunner();
conventionRunner.Run(convention, listener, typeof(SampleTestClass));
console
.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.None)
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.ShouldEqual(
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.DivideByZero' failed: System.DivideByZeroException",
"Attempted to divide by zero.",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.DivideByZero() in " + PathToThisFile() + ":line #",
"",
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.FailedAssertion' failed: Fixie.Tests.SampleAssertionLibrary.AssertionException",
"Expected 1, but was 0.",
" at Fixie.Tests.SampleAssertionLibrary.SampleAssert.AreEqual(Int32 expected, Int32 actual) in " + PathToThisFile() + ":line #",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.FailedAssertion() in " + PathToThisFile() + ":line #",
"",
"");
}
}
public void ShouldReportResultsToTestDrivenDotNet()
{
var testDriven = new StubTestListener();
using (var console = new RedirectedConsole())
{
var listener = new TestDrivenListener(testDriven);
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = testDriven.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.FixtureType.ShouldEqual(null);
result.Method.ShouldEqual(null);
result.TimeSpan.ShouldEqual(TimeSpan.Zero);
result.TotalTests.ShouldEqual(0);
result.TestRunnerName.ShouldBeNull();
}
results[0].Name.ShouldEqual(testClass + ".SkipWithReason");
results[0].State.ShouldEqual(TestState.Ignored);
results[0].Message.ShouldEqual("Skipped with reason.");
results[0].StackTrace.ShouldBeNull();
results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].State.ShouldEqual(TestState.Ignored);
results[1].Message.ShouldBeNull();
results[1].StackTrace.ShouldBeNull();
results[2].Name.ShouldEqual(testClass + ".Fail");
results[2].State.ShouldEqual(TestState.Failed);
results[2].Message.ShouldEqual("Fixie.Tests.FailureException");
results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[3].Name.ShouldEqual(testClass + ".Pass(123)");
results[3].State.ShouldEqual(TestState.Passed);
results[3].Message.ShouldBeNull();
results[3].StackTrace.ShouldBeNull();
}
}
public void ShouldExecuteAllCasesInAllDiscoveredTestClasses()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.Execute(listener, typeof(SampleIrrelevantClass), typeof(PassTestClass), typeof(int), typeof(PassFailTestClass));
listener.ShouldHaveEntries("Fixie.Tests.Conventions.ConventionTests+PassFailTestClass.Pass passed.",
"Fixie.Tests.Conventions.ConventionTests+PassFailTestClass.Fail failed: 'Fail' failed!",
"Fixie.Tests.Conventions.ConventionTests+PassTestClass.PassA passed.",
"Fixie.Tests.Conventions.ConventionTests+PassTestClass.PassB passed.");
}
public void ShouldFailWithClearExplanationWhenInputParameterGenerationHasNotBeenCustomizedYetTestMethodAcceptsParameters()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.Execute(listener, typeof(ParameterizedTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg failed: This parameterized test could not be executed, because no input values were available.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes failed: This parameterized test could not be executed, because no input values were available.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs passed.");
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
using (var console = new RedirectedConsole())
using (var listener = new TeamCityListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
.ShouldEqual(
"##teamcity[testSuiteStarted name='Fixie.Tests.dll']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"##teamcity[testStarted name='" + testClass + ".FailA']",
"##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".FailB']",
"##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassA']",
"##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassB']",
"##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassC']",
"##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassC' duration='#']",
"##teamcity[testSuiteFinished name='Fixie.Tests.dll']");
}
}
public void ShouldShortCircuitTestExecutionByFailingAllCasesWhenCaseOrderingThrows()
{
var listener = new StubListener();
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>());
convention.ClassExecution
.CreateInstancePerClass()
.SortCases((caseA, caseB) => { throw new Exception("SortCases lambda expression threw!"); });
convention.Parameters
.Add <BuggyParameterSource>();
new Runner(listener).RunTypes(GetType().Assembly, convention,
typeof(SampleIrrelevantClass), typeof(PassTestClass), typeof(int),
typeof(PassFailTestClass), typeof(SkipTestClass), typeof(BuggyParameterGenerationTestClass));
//NOTE: Since the ordering of cases is deliberately failing, and since member order via reflection
// is undefined, we explicitly sort the listener Entries here to avoid making a brittle assertion.
var strings = listener.Entries.OrderBy(x => x).ToArray();
strings.ShouldEqual(
"Fixie.Tests.Internal.RunnerTests+BuggyParameterGenerationTestClass.ParameterizedA failed: Exception thrown while attempting to yield input parameters for method: ParameterizedA" + Environment.NewLine +
" Secondary Failure: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+BuggyParameterGenerationTestClass.ParameterizedB failed: Exception thrown while attempting to yield input parameters for method: ParameterizedB" + Environment.NewLine +
" Secondary Failure: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Fail failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Pass failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassA failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassB failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipA failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipB failed: Failed to compare two elements in the array." + Environment.NewLine +
" Inner Exception: SortCases lambda expression threw!");
}
public void ShouldAllowRandomShufflingOfCaseExecutionOrder()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.ClassExecution
.CreateInstancePerTestClass()
.ShuffleCases(new Random(1));
convention.Execute(listener, typeof(SampleIrrelevantClass), typeof(PassTestClass), typeof(int), typeof(PassFailTestClass), typeof(SkipTestClass));
listener.Entries.ShouldEqual("Fixie.Tests.Conventions.ConventionTests+PassTestClass.PassB passed.",
"Fixie.Tests.Conventions.ConventionTests+PassTestClass.PassA passed.",
"Fixie.Tests.Conventions.ConventionTests+PassFailTestClass.Fail failed: 'Fail' failed!",
"Fixie.Tests.Conventions.ConventionTests+PassFailTestClass.Pass passed.",
"Fixie.Tests.Conventions.ConventionTests+SkipTestClass.Skip skipped.");
}
public void ShouldExecuteAllCasesInAllDiscoveredTestClasses()
{
var listener = new StubListener();
var convention = SelfTestConvention.Build();
new Runner(listener).RunTypes(GetType().Assembly, convention,
typeof(SampleIrrelevantClass), typeof(PassTestClass), typeof(int),
typeof(PassFailTestClass), typeof(SkipTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassA passed.",
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassB passed.",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Fail failed: 'Fail' failed!",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Pass passed.",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipA skipped.",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipB skipped.");
}
public void ShouldAllowConventionToGeneratePotentiallyManySetsOfInputParametersPerMethod()
{
var listener = new StubListener();
var convention = new SelfTestConvention();
convention.Parameters(ParametersFromAttributesWithTypeDefaultFallback);
convention.Execute(listener, typeof(ParameterizedTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.IntArg(0) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(1, 1, 2) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(1, 2, 3) passed.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.MultipleCasesFromAttributes(5, 5, 11) failed: Expected sum of 11 but was 10.",
"Fixie.Tests.Cases.ParameterizedCaseTests+ParameterizedTestClass.ZeroArgs passed.");
}
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention
.Methods
.Where(method => !method.Has <SkipAttribute>());
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldProduceValidXmlDocument()
{
var listener = new StubListener();
var runner = new Runner(listener);
var executionResult = new ExecutionResult();
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
var assemblyResult = runner.RunTypes(GetType().Assembly, convention, typeof(PassFailTestClass));
executionResult.Add(assemblyResult);
var report = new NUnitXmlReport();
var actual = report.Transform(executionResult);
XsdValidate(actual);
CleanBrittleValues(actual.ToString(SaveOptions.DisableFormatting)).ShouldEqual(ExpectedReport);
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.",
"Test '" + testClass + ".SkipWithoutReason' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldAllowRandomShufflingOfCaseExecutionOrder()
{
var listener = new StubListener();
var convention = SelfTestConvention.Build();
convention.ClassExecution
.CreateInstancePerClass()
.ShuffleCases(new Random(1));
new Runner(listener).RunTypes(GetType().Assembly, convention,
typeof(SampleIrrelevantClass), typeof(PassTestClass), typeof(int),
typeof(PassFailTestClass), typeof(SkipTestClass));
listener.Entries.ShouldEqual(
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassB passed.",
"Fixie.Tests.Internal.RunnerTests+PassTestClass.PassA passed.",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Fail failed: 'Fail' failed!",
"Fixie.Tests.Internal.RunnerTests+PassFailTestClass.Pass passed.",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipB skipped.",
"Fixie.Tests.Internal.RunnerTests+SkipTestClass.SkipA skipped.");
}
public void ShouldDescribeCaseResults()
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Name == "Skip");
convention.CaseExecution.Skip(x => x.Method.Name == "SkipWithReason", x => "Skipped by naming convention.");
convention.HideExceptionDetails.For <EqualException>();
var listener = new StubCaseResultListener();
using (new RedirectedConsole())
{
typeof(SampleTestClass).Run(listener, convention);
listener.Log.Count.ShouldEqual(5);
var skip = listener.Log[0];
var skipWithReason = listener.Log[1];
var fail = listener.Log[2];
var failByAssertion = listener.Log[3];
var pass = listener.Log[4];
pass.Name.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Pass");
pass.MethodGroup.FullName.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Pass");
pass.Output.ShouldEqual("Pass" + Environment.NewLine);
pass.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
pass.Status.ShouldEqual(CaseStatus.Passed);
pass.Exceptions.ShouldBeNull();
pass.SkipReason.ShouldBeNull();
fail.Name.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Fail");
fail.MethodGroup.FullName.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Fail");
fail.Output.ShouldEqual("Fail" + Environment.NewLine);
fail.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
fail.Status.ShouldEqual(CaseStatus.Failed);
fail.Exceptions.PrimaryException.Type.ShouldEqual("Fixie.Tests.FailureException");
fail.Exceptions.CompoundStackTrace.ShouldNotBeNull();
fail.Exceptions.PrimaryException.Message.ShouldEqual("'Fail' failed!");
fail.SkipReason.ShouldBeNull();
failByAssertion.Name.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.FailByAssertion");
failByAssertion.MethodGroup.FullName.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.FailByAssertion");
failByAssertion.Output.ShouldEqual("FailByAssertion" + Environment.NewLine);
failByAssertion.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
failByAssertion.Status.ShouldEqual(CaseStatus.Failed);
failByAssertion.Exceptions.PrimaryException.Type.ShouldEqual("Should.Core.Exceptions.EqualException");
failByAssertion.Exceptions.CompoundStackTrace.ShouldNotBeNull();
failByAssertion.Exceptions.PrimaryException.Message.Lines().ShouldEqual(
"Assert.Equal() Failure",
"Expected: 2",
"Actual: 1");
failByAssertion.SkipReason.ShouldBeNull();
skip.Name.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Skip");
skip.MethodGroup.FullName.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.Skip");
skip.Output.ShouldBeNull();
skip.Duration.ShouldEqual(TimeSpan.Zero);
skip.Status.ShouldEqual(CaseStatus.Skipped);
skip.Exceptions.ShouldBeNull();
skip.SkipReason.ShouldBeNull();
skipWithReason.Name.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.SkipWithReason");
skipWithReason.MethodGroup.FullName.ShouldEqual("Fixie.Tests.Execution.CaseResultTests+SampleTestClass.SkipWithReason");
skipWithReason.Output.ShouldBeNull();
skipWithReason.Duration.ShouldEqual(TimeSpan.Zero);
skipWithReason.Status.ShouldEqual(CaseStatus.Skipped);
skipWithReason.Exceptions.ShouldBeNull();
skipWithReason.SkipReason.ShouldEqual("Skipped by naming convention.");
}
}
protected CaseTests()
{
Listener = new StubListener();
Convention = SelfTestConvention.Build();
}
public void ShouldReportResultsToAppVeyorBuildWorkerApi()
{
var results = new List <AppVeyorListener.TestResult>();
var httpClient = new HttpClient(new FakeHandler(request =>
{
request.ShouldNotBeNull();
request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests");
request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json"));
request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8");
var requestContent = request.Content.ReadAsStringAsync().Result;
results.Add(new JavaScriptSerializer().Deserialize <AppVeyorListener.TestResult>(requestContent));
return(new HttpResponseMessage {
StatusCode = HttpStatusCode.Accepted
});
}));
using (var console = new RedirectedConsole())
using (var listener = new AppVeyorListener("http://localhost:4567", httpClient))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.testFramework.ShouldEqual("Fixie");
result.fileName.ShouldEqual("Fixie.Tests.dll");
}
results[0].testName.ShouldEqual(testClass + ".SkipWithReason");
results[0].outcome.ShouldEqual("Skipped");
results[0].durationMilliseconds.ShouldEqual("0");
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].StdOut.ShouldBeNull();
results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].outcome.ShouldEqual("Skipped");
results[1].durationMilliseconds.ShouldEqual("0");
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].StdOut.ShouldBeNull();
results[2].testName.ShouldEqual(testClass + ".Fail");
results[2].outcome.ShouldEqual("Failed");
int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual("'Fail' failed!",
" at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[3].testName.ShouldEqual(testClass + ".Pass(123)");
results[3].outcome.ShouldEqual("Passed");
int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
}
}
public void ShouldReportResultsToExecutionRecorder()
{
const string assemblyPath = "assembly.path.dll";
var recorder = new StubExecutionRecorder();
using (var console = new RedirectedConsole())
using (var listener = new VisualStudioListener(recorder, assemblyPath))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = recorder.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.Traits.ShouldBeEmpty();
result.Attachments.ShouldBeEmpty();
result.ComputerName.ShouldEqual(Environment.MachineName);
result.TestCase.Traits.ShouldBeEmpty();
result.TestCase.LocalExtensionData.ShouldBeNull();
result.TestCase.Source.ShouldEqual("assembly.path.dll");
//Source locations are a discovery-time concern.
result.TestCase.CodeFilePath.ShouldBeNull();
result.TestCase.LineNumber.ShouldEqual(-1);
}
results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithReason");
results[0].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[0].Outcome.ShouldEqual(TestOutcome.Skipped);
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].Messages.ShouldBeEmpty();
results[0].Duration.ShouldEqual(TimeSpan.Zero);
results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[1].Outcome.ShouldEqual(TestOutcome.Skipped);
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].DisplayName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].Messages.ShouldBeEmpty();
results[1].Duration.ShouldEqual(TimeSpan.Zero);
results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Fail");
results[2].TestCase.DisplayName.ShouldEqual(testClass + ".Fail");
results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[2].Outcome.ShouldEqual(TestOutcome.Failed);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].DisplayName.ShouldEqual(testClass + ".Fail");
results[2].Messages.Count.ShouldEqual(1);
results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Pass");
results[3].TestCase.DisplayName.ShouldEqual(testClass + ".Pass");
results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[3].Outcome.ShouldEqual(TestOutcome.Passed);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].DisplayName.ShouldEqual(testClass + ".Pass(123)");
results[3].Messages.Count.ShouldEqual(1);
results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
}
}