public void ShouldProvideCaseReturnValuesToCustomBehaviors() { using (var console = new RedirectedConsole()) { Convention .CaseExecution .Wrap<TreatBoolReturnValuesAsAssertions>(); Run<SampleTestClass>(); Listener.Entries.ShouldEqual( "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolFalse failed: Boolean test case returned false!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolThrow failed: 'BoolThrow' failed!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolTrue passed", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Pass passed", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Throw failed: 'Throw' failed!" ); console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) .ShouldEqual( "BoolFalse False", "BoolThrow null", "BoolTrue True", "Pass null", "Throw null"); } }
public void ShouldUnpackResultValuesFromStronglyTypedTaskObjectsForAsyncCases() { using (var console = new RedirectedConsole()) { Convention .CaseExecution .Wrap<TreatBoolReturnValuesAsAssertions>(); Run<SampleAsyncTestClass>(); Listener.Entries.ShouldEqual( "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolFalse failed: Boolean test case returned false!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolThrow failed: 'BoolThrow' failed!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolTrue passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Pass passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Throw failed: 'Throw' failed!" ); console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries) .ShouldEqual( "BoolFalse False", "BoolThrow null", "BoolTrue True", "Pass null", "Throw null"); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build()); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "Test '" + testClass + ".SkipA' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "3 passed, 2 failed, 1 skipped, took 1.23 seconds (Fixie 1.2.3.4)."); } }
public void ShouldReportResultsToTestDrivenDotNet() { var testDriven = new StubTestListener(); using (var console = new RedirectedConsole()) { var listener = new TestDrivenListener(testDriven); var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = testDriven.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.FixtureType.ShouldEqual(null); result.Method.ShouldEqual(null); result.TimeSpan.ShouldEqual(TimeSpan.Zero); result.TotalTests.ShouldEqual(0); result.TestRunnerName.ShouldBeNull(); } results[0].Name.ShouldEqual(testClass + ".SkipWithReason"); results[0].State.ShouldEqual(TestState.Ignored); results[0].Message.ShouldEqual("Skipped with reason."); results[0].StackTrace.ShouldBeNull(); results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].State.ShouldEqual(TestState.Ignored); results[1].Message.ShouldBeNull(); results[1].StackTrace.ShouldBeNull(); results[2].Name.ShouldEqual(testClass + ".Fail"); results[2].State.ShouldEqual(TestState.Failed); results[2].Message.ShouldEqual("Fixie.Tests.FailureException"); results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual( "'Fail' failed!", " at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[3].Name.ShouldEqual(testClass + ".Pass(123)"); results[3].State.ShouldEqual(TestState.Passed); results[3].Message.ShouldBeNull(); results[3].StackTrace.ShouldBeNull(); } }
protected Output Run() { using (var console = new RedirectedConsole()) { var listener = new StubListener(); Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal)); typeof(SampleTestClass).Run(listener, Convention); return new Output(console.Lines().ToArray(), listener.Entries.ToArray()); } }
public void ShouldReportResultsToTheConsoleInTeamCityFormat() { using (var console = new RedirectedConsole()) using (var listener = new TeamCityListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations. .ShouldEqual( "##teamcity[testSuiteStarted name='Fixie.Tests.dll']", "##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']", "##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "##teamcity[testStarted name='"+testClass+".FailA']", "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']", "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']", "##teamcity[testStarted name='" + testClass + ".FailB']", "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']", "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassA']", "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']", "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassB']", "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']", "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassC']", "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']", "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']", "##teamcity[testSuiteFinished name='Fixie.Tests.dll']"); } }
private static object Execute(Assembly assembly) { using (var console = new RedirectedConsole()) { var result = assembly.Execute(); var writerResult = console.Output; if ((result == null || result == Core.Void.Value) && writerResult != "") return writerResult; return result; } }
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention .Methods .Where(method => !method.Has<SkipAttribute>()); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.", "Test '" + testClass + ".SkipWithoutReason' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldIgnoreCaseReturnValuesByDefault() { using (var console = new RedirectedConsole()) { Run<SampleTestClass>(); Run<SampleAsyncTestClass>(); Listener.Entries.ShouldEqual( "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolFalse passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolThrow failed: 'BoolThrow' failed!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolTrue passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Pass passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Throw failed: 'Throw' failed!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolFalse passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolThrow failed: 'BoolThrow' failed!", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolTrue passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Pass passed.", "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Throw failed: 'Throw' failed!" ); console.Output.ShouldBeEmpty(); } }
public void ShouldReportResultsToExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); using (var console = new RedirectedConsole()) using (var listener = new VisualStudioListener(recorder, assemblyPath)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = recorder.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(Environment.MachineName); result.TestCase.Traits.ShouldBeEmpty(); result.TestCase.LocalExtensionData.ShouldBeNull(); result.TestCase.Source.ShouldEqual("assembly.path.dll"); //Source locations are a discovery-time concern. result.TestCase.CodeFilePath.ShouldBeNull(); result.TestCase.LineNumber.ShouldEqual(-1); } results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[0].Outcome.ShouldEqual(TestOutcome.Skipped); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].Messages.ShouldBeEmpty(); results[0].Duration.ShouldEqual(TimeSpan.Zero); results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[1].Outcome.ShouldEqual(TestOutcome.Skipped); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].Messages.ShouldBeEmpty(); results[1].Duration.ShouldEqual(TimeSpan.Zero); results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail"); results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail"); results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[2].Outcome.ShouldEqual(TestOutcome.Failed); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual( "'Fail' failed!", " at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].DisplayName.ShouldEqual(testClass + ".Fail"); results[2].Messages.Count.ShouldEqual(1); results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass"); results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass"); results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[3].Outcome.ShouldEqual(TestOutcome.Passed); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].DisplayName.ShouldEqual(testClass +".Pass(123)"); results[3].Messages.Count.ShouldEqual(1); results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); } }
public ExecutionSummary Run(Type testClass, bool isOnlyTestClass) { var methods = methodDiscoverer.TestMethods(testClass); var summary = new ExecutionSummary(); if (!methods.Any()) { return(summary); } Start(testClass); var classStopwatch = Stopwatch.StartNew(); var orderedMethods = OrderedMethods(methods, summary); bool classLifecycleFailed = false; bool runCasesInvokedByLifecycle = false; try { Action <Action <Case> > runCases = caseLifecycle => { runCasesInvokedByLifecycle = true; foreach (var @case in YieldCases(orderedMethods, summary)) { Start(@case); Exception caseLifecycleException = null; string consoleOutput; using (var console = new RedirectedConsole()) { var caseStopwatch = Stopwatch.StartNew(); try { caseLifecycle(@case); } catch (Exception exception) { caseLifecycleException = exception; } caseStopwatch.Stop(); @case.Duration += caseStopwatch.Elapsed; consoleOutput = console.Output; @case.Output += consoleOutput; } Console.Write(consoleOutput); var caseHasNormalResult = @case.State == CaseState.Failed || @case.State == CaseState.Passed; var caseLifecycleFailed = caseLifecycleException != null; if (caseHasNormalResult) { if (@case.State == CaseState.Failed) { Fail(@case, summary); } else if (!caseLifecycleFailed) { Pass(@case, summary); } } if (caseLifecycleFailed) { Fail(new Case(@case, caseLifecycleException), summary); } else if (!caseHasNormalResult) { Skip(@case, summary); } } }; var runContext = isOnlyTestClass && methods.Count == 1 ? new TestClass(testClass, runCases, methods.Single()) : new TestClass(testClass, runCases); execution.Execute(runContext); } catch (Exception exception) { classLifecycleFailed = true; foreach (var method in orderedMethods) { Fail(method, exception, summary); } } if (!runCasesInvokedByLifecycle && !classLifecycleFailed) { //No cases ran, and we didn't already emit a general //failure for each method, so emit a general skip for //each method. foreach (var method in orderedMethods) { var @case = new Case(method); Skip(@case, summary); } } classStopwatch.Stop(); Complete(testClass, summary, classStopwatch.Elapsed); return(summary); }
public void ShouldReportResultsToAppVeyorBuildWorkerApi() { var results = new List<AppVeyorListener.TestResult>(); var httpClient = new HttpClient(new FakeHandler(request => { request.ShouldNotBeNull(); request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests"); request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json")); request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8"); var requestContent = request.Content.ReadAsStringAsync().Result; results.Add(new JavaScriptSerializer().Deserialize<AppVeyorListener.TestResult>(requestContent)); return new HttpResponseMessage { StatusCode = HttpStatusCode.Accepted }; })); using (var console = new RedirectedConsole()) using (var listener = new AppVeyorListener("http://localhost:4567", httpClient)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); results.Count.ShouldEqual(4); foreach (var result in results) { result.testFramework.ShouldEqual("Fixie"); result.fileName.ShouldEqual("Fixie.Tests.dll"); } results[0].testName.ShouldEqual(testClass + ".SkipWithReason"); results[0].outcome.ShouldEqual("Skipped"); results[0].durationMilliseconds.ShouldEqual("0"); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].StdOut.ShouldBeNull(); results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].outcome.ShouldEqual("Skipped"); results[1].durationMilliseconds.ShouldEqual("0"); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].StdOut.ShouldBeNull(); results[2].testName.ShouldEqual(testClass + ".Fail"); results[2].outcome.ShouldEqual("Failed"); int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual("'Fail' failed!", " at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[3].testName.ShouldEqual(testClass + ".Pass(123)"); results[3].outcome.ShouldEqual("Passed"); int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); } }