public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
{
var listener = new ConsoleListener();
typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build());
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"Test '" + testClass + ".SkipA' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"3 passed, 2 failed, 1 skipped, took 1.23 seconds (Fixie 1.2.3.4).");
}
}
public void ShouldReportResultsToTestDrivenDotNet()
{
var testDriven = new StubTestListener();
using (var console = new RedirectedConsole())
{
var listener = new TestDrivenListener(testDriven);
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
convention.Parameters.Add<InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = testDriven.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.FixtureType.ShouldEqual(null);
result.Method.ShouldEqual(null);
result.TimeSpan.ShouldEqual(TimeSpan.Zero);
result.TotalTests.ShouldEqual(0);
result.TestRunnerName.ShouldBeNull();
}
results[0].Name.ShouldEqual(testClass + ".SkipWithReason");
results[0].State.ShouldEqual(TestState.Ignored);
results[0].Message.ShouldEqual("Skipped with reason.");
results[0].StackTrace.ShouldBeNull();
results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].State.ShouldEqual(TestState.Ignored);
results[1].Message.ShouldBeNull();
results[1].StackTrace.ShouldBeNull();
results[2].Name.ShouldEqual(testClass + ".Fail");
results[2].State.ShouldEqual(TestState.Failed);
results[2].Message.ShouldEqual("Fixie.Tests.FailureException");
results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[3].Name.ShouldEqual(testClass + ".Pass(123)");
results[3].State.ShouldEqual(TestState.Passed);
results[3].Message.ShouldBeNull();
results[3].StackTrace.ShouldBeNull();
}
}
protected Output Run()
{
using (var console = new RedirectedConsole())
{
var listener = new StubListener();
Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal));
typeof(SampleTestClass).Run(listener, Convention);
return new Output(console.Lines().ToArray(), listener.Entries.ToArray());
}
}
public void ShouldReportResultsToTheConsoleInTeamCityFormat()
{
using (var console = new RedirectedConsole())
using (var listener = new TeamCityListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
.Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
.ShouldEqual(
"##teamcity[testSuiteStarted name='Fixie.Tests.dll']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']",
"##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"##teamcity[testStarted name='"+testClass+".FailA']",
"##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".FailB']",
"##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
"##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
"##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassA']",
"##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassB']",
"##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
"##teamcity[testStarted name='" + testClass + ".PassC']",
"##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
"##teamcity[testFinished name='" + testClass + ".PassC' duration='#']",
"##teamcity[testSuiteFinished name='Fixie.Tests.dll']");
}
}
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention
.Methods
.Where(method => !method.Has<SkipAttribute>());
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldReportResultsToTheConsole()
{
using (var console = new RedirectedConsole())
using (var listener = new ConsoleListener())
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.Select(CleanBrittleValues)
.ShouldEqual(
"------ Testing Assembly Fixie.Tests.dll ------",
"",
"Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.",
"Test '" + testClass + ".SkipWithoutReason' skipped",
"Console.Out: FailA",
"Console.Error: FailA",
"Console.Out: FailB",
"Console.Error: FailB",
"Console.Out: PassA",
"Console.Error: PassA",
"Console.Out: PassB",
"Console.Error: PassB",
"Console.Out: PassC",
"Console.Error: PassC",
"Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
"'FailA' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
"",
"Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
"'FailB' failed!",
" at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
"",
"3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ").");
}
}
public void ShouldReportResultsToExecutionRecorder()
{
const string assemblyPath = "assembly.path.dll";
var recorder = new StubExecutionRecorder();
using (var console = new RedirectedConsole())
using (var listener = new VisualStudioListener(recorder, assemblyPath))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
convention.Parameters.Add<InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = recorder.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.Traits.ShouldBeEmpty();
result.Attachments.ShouldBeEmpty();
result.ComputerName.ShouldEqual(Environment.MachineName);
result.TestCase.Traits.ShouldBeEmpty();
result.TestCase.LocalExtensionData.ShouldBeNull();
result.TestCase.Source.ShouldEqual("assembly.path.dll");
//Source locations are a discovery-time concern.
result.TestCase.CodeFilePath.ShouldBeNull();
result.TestCase.LineNumber.ShouldEqual(-1);
}
results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason");
results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason");
results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[0].Outcome.ShouldEqual(TestOutcome.Skipped);
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].Messages.ShouldBeEmpty();
results[0].Duration.ShouldEqual(TimeSpan.Zero);
results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[1].Outcome.ShouldEqual(TestOutcome.Skipped);
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].Messages.ShouldBeEmpty();
results[1].Duration.ShouldEqual(TimeSpan.Zero);
results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail");
results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail");
results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[2].Outcome.ShouldEqual(TestOutcome.Failed);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].DisplayName.ShouldEqual(testClass + ".Fail");
results[2].Messages.Count.ShouldEqual(1);
results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass");
results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass");
results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[3].Outcome.ShouldEqual(TestOutcome.Passed);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].DisplayName.ShouldEqual(testClass +".Pass(123)");
results[3].Messages.Count.ShouldEqual(1);
results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
}
}
public void ShouldReportResultsToAppVeyorBuildWorkerApi()
{
var results = new List<AppVeyorListener.TestResult>();
var httpClient = new HttpClient(new FakeHandler(request =>
{
request.ShouldNotBeNull();
request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests");
request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json"));
request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8");
var requestContent = request.Content.ReadAsStringAsync().Result;
results.Add(new JavaScriptSerializer().Deserialize<AppVeyorListener.TestResult>(requestContent));
return new HttpResponseMessage { StatusCode = HttpStatusCode.Accepted };
}));
using (var console = new RedirectedConsole())
using (var listener = new AppVeyorListener("http://localhost:4567", httpClient))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
convention.Parameters.Add<InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.testFramework.ShouldEqual("Fixie");
result.fileName.ShouldEqual("Fixie.Tests.dll");
}
results[0].testName.ShouldEqual(testClass + ".SkipWithReason");
results[0].outcome.ShouldEqual("Skipped");
results[0].durationMilliseconds.ShouldEqual("0");
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].StdOut.ShouldBeNull();
results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].outcome.ShouldEqual("Skipped");
results[1].durationMilliseconds.ShouldEqual("0");
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].StdOut.ShouldBeNull();
results[2].testName.ShouldEqual(testClass + ".Fail");
results[2].outcome.ShouldEqual("Failed");
int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual("'Fail' failed!",
" at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[3].testName.ShouldEqual(testClass + ".Pass(123)");
results[3].outcome.ShouldEqual("Passed");
int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
}
}