public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
new SelfTestConvention().Execute(listener, typeof(PassFailTestClass));
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.ShouldEqual(
"Test '" + testClass + ".SkipA' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #");
}
}
public void Execute(Fixture fixture)
{
foreach (var caseExecution in fixture.CaseExecutions)
{
using (var console = new RedirectedConsole())
{
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
fixture.CaseExecutionBehavior.Execute(caseExecution, fixture.Instance);
}
catch (Exception exception)
{
caseExecution.Fail(exception);
}
stopwatch.Stop();
caseExecution.Duration = stopwatch.Elapsed;
caseExecution.Output = console.Output;
}
Console.Write(caseExecution.Output);
}
}
public void Execute(Fixture fixture, Action next)
{
foreach (var @case in fixture.Cases)
{
using (var console = new RedirectedConsole())
{
@case.Fixture = fixture;
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
caseBehaviors.Execute(@case);
}
catch (Exception exception)
{
@case.Fail(exception);
}
stopwatch.Stop();
@case.Fixture = null;
@case.Duration = stopwatch.Elapsed;
@case.Output = console.Output;
}
Console.Write(@case.Output);
}
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build());
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"Test '" + testClass + ".SkipWithReason' skipped: Skipped due to naming convention.",
"Test '" + testClass + ".SkipWithoutReason' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"3 passed, 2 failed, 2 skipped, took 1.23 seconds (Fixie 1.2.3.4).");
}
}
public void ShouldNotAffectOutputByDefault()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
typeof(SampleTestClass).Run(listener, SelfTestConvention.Build());
console
.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.None)
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.DivideByZero' failed: System.DivideByZeroException",
"Attempted to divide by zero.",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.DivideByZero() in " + PathToThisFile() + ":line #",
"",
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.FailedAssertion' failed: Fixie.Tests.SampleAssertionLibrary.AssertionException",
"Expected 1, but was 0.",
" at Fixie.Tests.SampleAssertionLibrary.SampleAssert.AreEqual(Int32 expected, Int32 actual) in " + PathToThisFile() + ":line #",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.FailedAssertion() in " + PathToThisFile() + ":line #",
"",
"0 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").",
"",
"");
}
}
public void ShouldUnpackResultValuesFromStronglyTypedTaskObjectsForAsyncCases()
{
using (var console = new RedirectedConsole())
{
Convention
.CaseExecution
.Wrap <TreatBoolReturnValuesAsAssertions>();
Run <SampleAsyncTestClass>();
Listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolFalse failed: Boolean test case returned false!",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolThrow failed: 'BoolThrow' failed!",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolTrue passed.",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Pass passed.",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Throw failed: 'Throw' failed!"
);
console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)
.ShouldEqual(
"BoolFalse False",
"BoolThrow null",
"BoolTrue True",
"Pass null",
"Throw null");
}
}
public void ShouldFilterAssertionLibraryImplementationDetailsWhenLibraryTypesAreSpecified()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention
.HideExceptionDetails
.For<SampleAssertionLibrary.AssertionException>()
.For(typeof(SampleAssertionLibrary.SampleAssert));
typeof(SampleTestClass).Run(listener, convention);
console
.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.None)
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.DivideByZero' failed: System.DivideByZeroException",
"Attempted to divide by zero.",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.DivideByZero() in " + PathToThisFile() + ":line #",
"",
"Test 'Fixie.Tests.AssertionLibraryFilteringTests+SampleTestClass.FailedAssertion' failed: ",
"Expected 1, but was 0.",
" at Fixie.Tests.AssertionLibraryFilteringTests.SampleTestClass.FailedAssertion() in " + PathToThisFile() + ":line #",
"",
"0 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").",
"",
"");
}
}
public void ShouldUnpackResultValuesFromStronglyTypedTaskObjectsForAsyncCases()
{
using (var console = new RedirectedConsole())
{
var discovery = new SelfTestDiscovery();
var execution = new TreatBoolReturnValuesAsAssertions();
Run <SampleAsyncTestClass>(discovery, execution)
.ShouldBe(
For <SampleAsyncTestClass>(
".BoolFalse failed: Boolean test case returned false!",
".BoolThrow failed: 'BoolThrow' failed!",
".BoolTrue passed",
".Pass passed",
".String passed",
".StringNull passed",
".Throw failed: 'Throw' failed!"));
console.Lines().ShouldBe(
"BoolFalse False",
"BoolThrow null",
"BoolTrue True",
"Pass null",
"String ABC",
"StringNull null",
"Throw null");
}
}
public void ShouldProvideCaseReturnValuesToCustomBehaviors()
{
using (var console = new RedirectedConsole())
{
var discovery = new SelfTestDiscovery();
var execution = new TreatBoolReturnValuesAsAssertions();
Run <SampleTestClass>(discovery, execution)
.ShouldBe(
For <SampleTestClass>(
".BoolFalse failed: Boolean test case returned false!",
".BoolThrow failed: 'BoolThrow' failed!",
".BoolTrue passed",
".Pass passed",
".String passed",
".StringNull passed",
".Throw failed: 'Throw' failed!"));
console.Lines().ShouldBe(
"BoolFalse False",
"BoolThrow null",
"BoolTrue True",
"Pass null",
"String ABC",
"StringNull null",
"Throw null");
}
}
public void Execute(Fixture fixture, Action next)
{
foreach (var @case in fixture.Cases)
{
using (var console = new RedirectedConsole())
{
@case.Fixture = fixture;
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
caseBehaviors.Execute(@case);
}
catch (Exception exception)
{
@case.Fail(exception);
}
stopwatch.Stop();
@case.Fixture = null;
@case.Duration += stopwatch.Elapsed;
@case.Output = console.Output;
}
Console.Write(@case.Output);
}
}
public void ShouldReportPassFailSkipCounts()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var assembly = typeof(ConsoleListener).Assembly;
var version = assembly.GetName().Version;
var assemblyResult = new AssemblyResult(assembly.Location);
var conventionResult = new ConventionResult("Fake Convention");
var classResult = new ClassResult("Fake Class");
assemblyResult.Add(conventionResult);
conventionResult.Add(classResult);
classResult.Add(CaseResult.Passed("A", TimeSpan.Zero));
classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter())));
classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter())));
classResult.Add(CaseResult.Skipped("D", "Reason"));
classResult.Add(CaseResult.Skipped("E", "Reason"));
classResult.Add(CaseResult.Skipped("F", "Reason"));
listener.AssemblyCompleted(assembly, assemblyResult);
console.Lines().ShouldEqual(string.Format("1 passed, 2 failed, 3 skipped, took {0} seconds (Fixie {1}).",
0.ToString("N2", CultureInfo.CurrentCulture),
version));
}
}
public void ShouldPublishEventsForAllListeners()
{
var listeners = new Listener[]
{
new EventHandler(),
new AnotherEventHandler(),
new CombinationEventHandler()
};
var bus = new Bus(listeners);
using (var console = new RedirectedConsole())
{
bus.Publish(new Event(1));
bus.Publish(new AnotherEvent(2));
bus.Publish(new Event(3));
console.Output.Lines()
.ShouldBe(
"EventHandler handled Event 1",
"CombinationEventHandler handled Event 1",
"AnotherEventHandler handled AnotherEvent 2",
"CombinationEventHandler handled AnotherEvent 2",
"EventHandler handled Event 3",
"CombinationEventHandler handled Event 3");
}
}
public void ShouldIgnoreCaseReturnValuesByDefault()
{
using (var console = new RedirectedConsole())
{
Run <SampleTestClass>()
.ShouldBe(
For <SampleTestClass>(
".BoolFalse passed",
".BoolThrow failed: 'BoolThrow' failed!",
".BoolTrue passed",
".Pass passed",
".String passed",
".StringNull passed",
".Throw failed: 'Throw' failed!"));
Run <SampleAsyncTestClass>()
.ShouldBe(
For <SampleAsyncTestClass>(
".BoolFalse passed",
".BoolThrow failed: 'BoolThrow' failed!",
".BoolTrue passed",
".Pass passed",
".String passed",
".StringNull passed",
".Throw failed: 'Throw' failed!"));
console.Output.ShouldBe("");
}
}
public void Execute(Fixture fixture)
{
foreach (var caseExecution in fixture.CaseExecutions)
{
using (var console = new RedirectedConsole())
{
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
fixture.CaseExecutionBehavior.Execute(caseExecution, fixture.Instance);
}
catch (Exception exception)
{
caseExecution.Fail(exception);
}
stopwatch.Stop();
caseExecution.Duration = stopwatch.Elapsed;
caseExecution.Output = console.Output;
}
Console.Write(caseExecution.Output);
}
}
public void ShouldCatchAndLogExceptionsThrowByProblematicListenersRatherThanInterruptExecution()
{
var listeners = new Listener[]
{
new EventHandler(),
new FailingEventHandler()
};
var bus = new Bus(listeners);
using (var console = new RedirectedConsole())
{
bus.Publish(new Event(1));
bus.Publish(new AnotherEvent(2));
bus.Publish(new Event(3));
console.Output.Lines()
.ShouldBe(
FullName <EventHandler>() + " handled Event 1",
FullName <FailingEventHandler>() + $" threw an exception while attempting to handle a message of type {FullName<Event>()}:",
"",
FullName <StubException>() + ": Could not handle Event 1",
"<<Stack Trace>>",
"",
FullName <EventHandler>() + " handled Event 3",
FullName <FailingEventHandler>() + $" threw an exception while attempting to handle a message of type {FullName<Event>()}:",
"",
FullName <StubException>() + ": Could not handle Event 3",
"<<Stack Trace>>");
}
}
public void ShouldProvideCaseReturnValuesToCustomBehaviors()
{
using (var console = new RedirectedConsole())
{
Convention
.CaseExecution
.Wrap <TreatBoolReturnValuesAsAssertions>();
Run <SampleTestClass>();
Listener.Entries.ShouldEqual(
"Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolFalse failed: Boolean test case returned false!",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolThrow failed: 'BoolThrow' failed!",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolTrue passed",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Pass passed",
"Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Throw failed: 'Throw' failed!"
);
console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)
.ShouldEqual(
"BoolFalse False",
"BoolThrow null",
"BoolTrue True",
"Pass null",
"Throw null");
}
}
public void ShouldReportPassFailSkipCounts()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var assembly = typeof(ConsoleListener).Assembly;
var version = assembly.GetName().Version;
var assemblyResult = new AssemblyResult(assembly.Location);
var conventionResult = new ConventionResult("Fake Convention");
var classResult = new ClassResult("Fake Class");
assemblyResult.Add(conventionResult);
conventionResult.Add(classResult);
classResult.Add(CaseResult.Passed("A", TimeSpan.Zero));
classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter())));
classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter())));
classResult.Add(CaseResult.Skipped("D", "Reason"));
classResult.Add(CaseResult.Skipped("E", "Reason"));
classResult.Add(CaseResult.Skipped("F", "Reason"));
listener.AssemblyCompleted(assembly, assemblyResult);
console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped, took 0.00 seconds (Fixie " + version + ").");
}
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
new SelfTestConvention().Execute(listener, typeof(PassFailTestClass));
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.ShouldEqual(
"Test '" + testClass + ".SkipA' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #");
}
}
public void Execute(InstanceExecution instanceExecution)
{
foreach (var caseExecution in instanceExecution.CaseExecutions)
{
using (var console = new RedirectedConsole())
{
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
var executionPlan = instanceExecution.ExecutionPlan;
var instance = instanceExecution.Instance;
executionPlan.Execute(caseExecution, instance);
}
catch (Exception exception)
{
caseExecution.Fail(exception);
}
stopwatch.Stop();
caseExecution.Duration = stopwatch.Elapsed;
caseExecution.Output = console.Output;
}
Console.Write(caseExecution.Output);
}
}
public void ShouldReportPassFailSkipCounts()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var assembly = typeof(ConsoleListener).Assembly;
var version = assembly.GetName().Version;
var assemblyResult = new AssemblyResult(assembly.Location);
var conventionResult = new ConventionResult("Fake Convention");
var classResult = new ClassResult("Fake Class");
assemblyResult.Add(conventionResult);
conventionResult.Add(classResult);
classResult.Add(CaseResult.Passed("A", TimeSpan.Zero));
classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, "Message", "Stack Trace", "Exception Type"));
classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, "Message", "Stack Trace", "Exception Type"));
classResult.Add(CaseResult.Skipped("D", "Reason"));
classResult.Add(CaseResult.Skipped("E", "Reason"));
classResult.Add(CaseResult.Skipped("F", "Reason"));
listener.AssemblyCompleted(assembly, assemblyResult);
console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped, took 0.00 seconds (Fixie " + version + ").");
}
}
public void ShouldReportResultsToTestDrivenDotNet()
{
var testDriven = new StubTestListener();
using (var console = new RedirectedConsole())
{
var listener = new TestDrivenListener(testDriven);
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = testDriven.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.FixtureType.ShouldEqual(null);
result.Method.ShouldEqual(null);
result.TimeSpan.ShouldEqual(TimeSpan.Zero);
result.TotalTests.ShouldEqual(0);
result.TestRunnerName.ShouldBeNull();
}
results[0].Name.ShouldEqual(testClass + ".SkipWithReason");
results[0].State.ShouldEqual(TestState.Ignored);
results[0].Message.ShouldEqual("Skipped with reason.");
results[0].StackTrace.ShouldBeNull();
results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].State.ShouldEqual(TestState.Ignored);
results[1].Message.ShouldBeNull();
results[1].StackTrace.ShouldBeNull();
results[2].Name.ShouldEqual(testClass + ".Fail");
results[2].State.ShouldEqual(TestState.Failed);
results[2].Message.ShouldEqual("Fixie.Tests.FailureException");
results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[3].Name.ShouldEqual(testClass + ".Pass(123)");
results[3].State.ShouldEqual(TestState.Passed);
results[3].Message.ShouldBeNull();
results[3].StackTrace.ShouldBeNull();
}
}
protected static SpecificationExecutionResult Execute <TSpecificationClass>()
{
using (var console = new RedirectedConsole())
{
var results = Execute(typeof(TSpecificationClass));
return(new SpecificationExecutionResult(results, SplitIntoLines(console.Output)));
}
}
public void Test01()
{
RedirectedConsole.Init();
RedirectedConsole.SetOut("out.cons");
Call("plain.txt format.txt abc");
Call("plain.txt format.txt 10");
RedirectedConsole.ResetAll();
}
protected Output Run()
{
using (var console = new RedirectedConsole())
{
var listener = new StubListener();
Convention.Execute(listener, typeof(SampleTestClass));
return(new Output(console.Lines.ToArray(), listener.Entries.ToArray()));
}
}
protected Output Run()
{
using (var console = new RedirectedConsole())
{
var listener = new StubListener();
Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal));
Convention.Execute(listener, typeof(SampleTestClass));
return(new Output(console.Lines().ToArray(), listener.Entries.ToArray()));
}
}
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var assembly = typeof(ConsoleListener).Assembly;
var version = assembly.GetName().Version;
listener.AssemblyCompleted(assembly, new AssemblyResult(1, 2, 0));
console.Lines().ShouldEqual("1 passed, 2 failed (Fixie " + version + ").");
}
}
public void ShouldReportPassFailSkipCounts()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
var assembly = typeof(ConsoleListener).Assembly;
var version = assembly.GetName().Version;
listener.AssemblyCompleted(assembly, new AssemblyResult(1, 2, 3));
console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped (Fixie " + version + ").");
}
}
public static RedirectedConsole Execute(string command, string args)
{
using (var process = new Process())
{
var console = new RedirectedConsole()
{
Command = $"{command} {args}"
};
WriteCommandLog(console.Command);
process.StartInfo = new ProcessStartInfo
{
WorkingDirectory = Directory.GetCurrentDirectory(),
FileName = command,
Arguments = args,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
ErrorDialog = false,
};
process.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
{
console.Output += e.Data;
console.Output += ((Process)sender).StandardError.ReadToEnd();
});
process.ErrorDataReceived += new DataReceivedEventHandler((sender, e) =>
{
console.Output += e.Data;
console.Output += ((Process)sender).StandardError.ReadToEnd();
});
try
{
process.Start();
process.BeginOutputReadLine();
process.WaitForExit();
}
catch (System.ComponentModel.Win32Exception ex)
{
console.Output += ex.Message + (ex.InnerException != null ? ex.InnerException.Message : string.Empty);
}
WriteTraceLog(console.Output);
return(console);
};
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
using (var console = new RedirectedConsole())
using (var listener = new TeamCityListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
.ShouldEqual(
"##teamcity[testSuiteStarted name='Fixie.Tests.dll']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"##teamcity[testStarted name='" + testClass + ".FailA']",
"##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".FailB']",
"##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassA']",
"##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassB']",
"##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassC']",
"##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassC' duration='#']",
"##teamcity[testSuiteFinished name='Fixie.Tests.dll']");
}
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
var listener = new TeamCityListener();
using (var console = new RedirectedConsole())
{
Run(listener);
console
.Output
.CleanStackTraceLineNumbers()
.Lines()
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'"))
.ShouldBe(
"##teamcity[testSuiteStarted name='Fixie.Tests']",
"Console.Out: Fail",
"Console.Error: Fail",
"##teamcity[testStarted name='" + TestClass + ".Fail']",
"##teamcity[testStdOut name='" + TestClass + ".Fail' out='Console.Out: Fail|r|nConsole.Error: Fail|r|n']",
"##teamcity[testFailed name='" + TestClass + ".Fail' message='|'Fail|' failed!' details='Fixie.Tests.FailureException|r|n" + At("Fail()") + "']",
"##teamcity[testFinished name='" + TestClass + ".Fail' duration='#']",
"Console.Out: FailByAssertion",
"Console.Error: FailByAssertion",
"##teamcity[testStarted name='" + TestClass + ".FailByAssertion']",
"##teamcity[testStdOut name='" + TestClass + ".FailByAssertion' out='Console.Out: FailByAssertion|r|nConsole.Error: FailByAssertion|r|n']",
"##teamcity[testFailed name='" + TestClass + ".FailByAssertion' message='Expected: 2|r|nActual: 1' details='Fixie.Tests.Assertions.AssertException|r|n" + At("FailByAssertion()") + "']",
"##teamcity[testFinished name='" + TestClass + ".FailByAssertion' duration='#']",
"Console.Out: Pass",
"Console.Error: Pass",
"##teamcity[testStarted name='" + TestClass + ".Pass']",
"##teamcity[testStdOut name='" + TestClass + ".Pass' out='Console.Out: Pass|r|nConsole.Error: Pass|r|n']",
"##teamcity[testFinished name='" + TestClass + ".Pass' duration='#']",
"##teamcity[testStarted name='" + TestClass + ".SkipWithReason']",
"##teamcity[testIgnored name='" + TestClass + ".SkipWithReason' message='|0x26a0 Skipped with reason.']",
"##teamcity[testFinished name='" + TestClass + ".SkipWithReason' duration='#']",
"##teamcity[testStarted name='" + TestClass + ".SkipWithoutReason']",
"##teamcity[testIgnored name='" + TestClass + ".SkipWithoutReason' message='']",
"##teamcity[testFinished name='" + TestClass + ".SkipWithoutReason' duration='#']",
"##teamcity[testSuiteFinished name='Fixie.Tests']");
}
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
using (var console = new RedirectedConsole())
{
var listener = new TeamCityListener();
var conventionRunner = new ConventionRunner();
conventionRunner.Run(new SelfTestConvention(), listener, typeof(PassFailTestClass));
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
.ShouldEqual(
"##teamcity[testIgnored name='" + testClass + ".SkipA']",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"##teamcity[testStarted name='"+testClass+".FailA']",
"##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".FailB']",
"##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassA']",
"##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassB']",
"##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassC']",
"##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassC' duration='#']");
}
}
private static object Execute(Assembly assembly)
{
using (var console = new RedirectedConsole())
{
var result = assembly.Execute();
var writerResult = console.Output;
if ((result == null || result == Core.Void.Value) && writerResult != "")
{
return(writerResult);
}
return(result);
}
}
public void ShouldShortCircuitSetupAndInnerBehaviorAndTearDownWhenCaseAlreadyHasExceptionsPriorToSetup()
{
builder.SetUpTearDown(SetUp, TearDown);
using (var console = new RedirectedConsole())
{
var @case = Case("Pass");
var exception = new Exception("Exception from earlier in the behavior chain.");
@case.Exceptions.Add(exception);
builder.Behavior.Execute(@case, instance);
@case.Exceptions.ToArray().Single().ShouldEqual(exception);
console.Lines.ShouldEqual();
}
}
public void ShouldShortCircuitSetupAndInnerBehaviorAndTearDownWhenCaseAlreadyHasExceptionsPriorToSetup()
{
builder.SetUpTearDown(SetUp, TearDown);
using (var console = new RedirectedConsole())
{
var @case = Case("Pass");
var exception = new Exception("Exception from earlier in the behavior chain.");
@case.Exceptions.Add(exception);
builder.Behavior.Execute(@case, instance);
@case.Exceptions.ToArray().Single().ShouldEqual(exception);
console.Lines.ShouldEqual();
}
}
public void ShouldProvideDiagnosticDescriptionWhenNoTestsWereExecuted()
{
void NoTestsFound(Discovery discovery)
=> discovery.Methods.Where(x => false);
var listener = new ConsoleListener();
using (var console = new RedirectedConsole())
{
Run(listener, NoTestsFound);
console.Output
.Lines()
.Last()
.ShouldEqual("No tests found.");
}
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
using (var console = new RedirectedConsole())
{
var listener = new TeamCityListener();
new SelfTestConvention().Execute(listener, typeof(PassFailTestClass));
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
.ShouldEqual(
"##teamcity[testIgnored name='" + testClass + ".SkipA']",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"##teamcity[testStarted name='" + testClass + ".FailA']",
"##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".FailB']",
"##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassA']",
"##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassB']",
"##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassC']",
"##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassC' duration='#']");
}
}
public void ShouldReportResults()
{
var listener = new ConsoleListener();
using (var console = new RedirectedConsole())
{
Run(listener);
console.Output
.CleanStackTraceLineNumbers()
.CleanDuration()
.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Test '" + TestClass + ".Fail' failed:",
"",
"'Fail' failed!",
"",
"Fixie.Tests.FailureException",
At("Fail()"),
"",
"Console.Out: FailByAssertion",
"Console.Error: FailByAssertion",
"Test '" + TestClass + ".FailByAssertion' failed:",
"",
"Expected: 2",
"Actual: 1",
"",
"Fixie.Assertions.AssertActualExpectedException",
At("FailByAssertion()"),
"",
"Console.Out: Pass",
"Console.Error: Pass",
"Test '" + TestClass + ".SkipWithReason' skipped:",
"Skipped with reason.",
"",
"Test '" + TestClass + ".SkipWithoutReason' skipped",
"",
"1 passed, 2 failed, 2 skipped, took 1.23 seconds");
}
}
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
{
void ZeroSkipped(Discovery discovery)
=> discovery.Methods.Where(x => !x.Name.StartsWith("Skip"));
var listener = new ConsoleListener();
using (var console = new RedirectedConsole())
{
Run(listener, ZeroSkipped);
console.Output
.Lines()
.Last()
.CleanDuration()
.ShouldEqual("1 passed, 2 failed, took 1.23 seconds");
}
}
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention
.Methods
.Where(method => !method.Has <SkipAttribute>());
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.",
"Test '" + testClass + ".SkipWithoutReason' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldProduceValidXmlDocument()
{
XDocument actual = null;
var listener = new ReportListener(report => actual = report);
using (var console = new RedirectedConsole())
{
Run(listener);
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: FailByAssertion",
"Console.Error: FailByAssertion",
"Console.Out: Pass",
"Console.Error: Pass");
}
CleanBrittleValues(actual.ToString(SaveOptions.DisableFormatting)).ShouldEqual(ExpectedReport);
}
protected Output Run()
{
using (var console = new RedirectedConsole())
{
var listener = new StubListener();
Convention.Execute(listener, typeof(SampleTestClass));
return new Output(console.Lines.ToArray(), listener.Entries.ToArray());
}
}
protected Output Run()
{
using (var console = new RedirectedConsole())
{
var listener = new StubListener();
Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal));
Convention.Execute(listener, typeof(SampleTestClass));
return new Output(console.Lines().ToArray(), listener.Entries.ToArray());
}
}
