public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build()); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "Test '" + testClass + ".SkipA' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "3 passed, 2 failed, 1 skipped, took 1.23 seconds (Fixie 1.2.3.4)."); } }
public void ShouldReportResultsToTestDrivenDotNet() { var testDriven = new StubTestListener(); using (var console = new RedirectedConsole()) { var listener = new TestDrivenListener(testDriven); var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = testDriven.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.FixtureType.ShouldEqual(null); result.Method.ShouldEqual(null); result.TimeSpan.ShouldEqual(TimeSpan.Zero); result.TotalTests.ShouldEqual(0); result.TestRunnerName.ShouldBeNull(); } results[0].Name.ShouldEqual(testClass + ".SkipWithReason"); results[0].State.ShouldEqual(TestState.Ignored); results[0].Message.ShouldEqual("Skipped with reason."); results[0].StackTrace.ShouldBeNull(); results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].State.ShouldEqual(TestState.Ignored); results[1].Message.ShouldBeNull(); results[1].StackTrace.ShouldBeNull(); results[2].Name.ShouldEqual(testClass + ".Fail"); results[2].State.ShouldEqual(TestState.Failed); results[2].Message.ShouldEqual("Fixie.Tests.FailureException"); results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual( "'Fail' failed!", " at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[3].Name.ShouldEqual(testClass + ".Pass(123)"); results[3].State.ShouldEqual(TestState.Passed); results[3].Message.ShouldBeNull(); results[3].StackTrace.ShouldBeNull(); } }
protected Output Run() { using (var console = new RedirectedConsole()) { var listener = new StubListener(); Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal)); typeof(SampleTestClass).Run(listener, Convention); return new Output(console.Lines().ToArray(), listener.Entries.ToArray()); } }
public void ShouldReportResultsToTheConsoleInTeamCityFormat() { using (var console = new RedirectedConsole()) using (var listener = new TeamCityListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations. .ShouldEqual( "##teamcity[testSuiteStarted name='Fixie.Tests.dll']", "##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']", "##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "##teamcity[testStarted name='"+testClass+".FailA']", "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']", "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']", "##teamcity[testStarted name='" + testClass + ".FailB']", "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']", "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassA']", "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']", "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassB']", "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']", "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassC']", "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']", "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']", "##teamcity[testSuiteFinished name='Fixie.Tests.dll']"); } }
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention .Methods .Where(method => !method.Has<SkipAttribute>()); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.", "Test '" + testClass + ".SkipWithoutReason' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldReportResultsToExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); using (var console = new RedirectedConsole()) using (var listener = new VisualStudioListener(recorder, assemblyPath)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = recorder.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(Environment.MachineName); result.TestCase.Traits.ShouldBeEmpty(); result.TestCase.LocalExtensionData.ShouldBeNull(); result.TestCase.Source.ShouldEqual("assembly.path.dll"); //Source locations are a discovery-time concern. result.TestCase.CodeFilePath.ShouldBeNull(); result.TestCase.LineNumber.ShouldEqual(-1); } results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[0].Outcome.ShouldEqual(TestOutcome.Skipped); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].Messages.ShouldBeEmpty(); results[0].Duration.ShouldEqual(TimeSpan.Zero); results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[1].Outcome.ShouldEqual(TestOutcome.Skipped); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].Messages.ShouldBeEmpty(); results[1].Duration.ShouldEqual(TimeSpan.Zero); results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail"); results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail"); results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[2].Outcome.ShouldEqual(TestOutcome.Failed); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual( "'Fail' failed!", " at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].DisplayName.ShouldEqual(testClass + ".Fail"); results[2].Messages.Count.ShouldEqual(1); results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass"); results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass"); results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[3].Outcome.ShouldEqual(TestOutcome.Passed); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].DisplayName.ShouldEqual(testClass +".Pass(123)"); results[3].Messages.Count.ShouldEqual(1); results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); } }
public void ShouldReportResultsToAppVeyorBuildWorkerApi() { var results = new List<AppVeyorListener.TestResult>(); var httpClient = new HttpClient(new FakeHandler(request => { request.ShouldNotBeNull(); request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests"); request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json")); request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8"); var requestContent = request.Content.ReadAsStringAsync().Result; results.Add(new JavaScriptSerializer().Deserialize<AppVeyorListener.TestResult>(requestContent)); return new HttpResponseMessage { StatusCode = HttpStatusCode.Accepted }; })); using (var console = new RedirectedConsole()) using (var listener = new AppVeyorListener("http://localhost:4567", httpClient)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); results.Count.ShouldEqual(4); foreach (var result in results) { result.testFramework.ShouldEqual("Fixie"); result.fileName.ShouldEqual("Fixie.Tests.dll"); } results[0].testName.ShouldEqual(testClass + ".SkipWithReason"); results[0].outcome.ShouldEqual("Skipped"); results[0].durationMilliseconds.ShouldEqual("0"); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].StdOut.ShouldBeNull(); results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].outcome.ShouldEqual("Skipped"); results[1].durationMilliseconds.ShouldEqual("0"); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].StdOut.ShouldBeNull(); results[2].testName.ShouldEqual(testClass + ".Fail"); results[2].outcome.ShouldEqual("Failed"); int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual("'Fail' failed!", " at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[3].testName.ShouldEqual(testClass + ".Pass(123)"); results[3].outcome.ShouldEqual("Passed"); int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); } }