public void ShouldReportPassFailSkipCounts() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); var assembly = typeof(ConsoleListener).Assembly; var version = assembly.GetName().Version; var assemblyResult = new AssemblyResult(assembly.Location); var conventionResult = new ConventionResult("Fake Convention"); var classResult = new ClassResult("Fake Class"); assemblyResult.Add(conventionResult); conventionResult.Add(classResult); classResult.Add(CaseResult.Passed("A", TimeSpan.Zero)); classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter()))); classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter()))); classResult.Add(CaseResult.Skipped("D", "Reason")); classResult.Add(CaseResult.Skipped("E", "Reason")); classResult.Add(CaseResult.Skipped("F", "Reason")); listener.AssemblyCompleted(assembly, assemblyResult); console.Lines().ShouldEqual(string.Format("1 passed, 2 failed, 3 skipped, took {0} seconds (Fixie {1}).", 0.ToString("N2", CultureInfo.CurrentCulture), version)); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build()); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "Test '" + testClass + ".SkipWithReason' skipped: Skipped due to naming convention.", "Test '" + testClass + ".SkipWithoutReason' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "3 passed, 2 failed, 2 skipped, took 1.23 seconds (Fixie 1.2.3.4)."); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); new SelfTestConvention().Execute(listener, typeof(PassFailTestClass)); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .ShouldEqual( "Test '" + testClass + ".SkipA' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.Listeners.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #"); } }
public void ShouldProvideCaseReturnValuesToCustomBehaviors() { using (var console = new RedirectedConsole()) { var discovery = new SelfTestDiscovery(); var execution = new TreatBoolReturnValuesAsAssertions(); Run <SampleTestClass>(discovery, execution) .ShouldBe( For <SampleTestClass>( ".BoolFalse failed: Boolean test case returned false!", ".BoolThrow failed: 'BoolThrow' failed!", ".BoolTrue passed", ".Pass passed", ".String passed", ".StringNull passed", ".Throw failed: 'Throw' failed!")); console.Lines().ShouldBe( "BoolFalse False", "BoolThrow null", "BoolTrue True", "Pass null", "String ABC", "StringNull null", "Throw null"); } }
public void ShouldUnpackResultValuesFromStronglyTypedTaskObjectsForAsyncCases() { using (var console = new RedirectedConsole()) { var discovery = new SelfTestDiscovery(); var execution = new TreatBoolReturnValuesAsAssertions(); Run <SampleAsyncTestClass>(discovery, execution) .ShouldBe( For <SampleAsyncTestClass>( ".BoolFalse failed: Boolean test case returned false!", ".BoolThrow failed: 'BoolThrow' failed!", ".BoolTrue passed", ".Pass passed", ".String passed", ".StringNull passed", ".Throw failed: 'Throw' failed!")); console.Lines().ShouldBe( "BoolFalse False", "BoolThrow null", "BoolTrue True", "Pass null", "String ABC", "StringNull null", "Throw null"); } }
public void ShouldReportPassFailSkipCounts() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); var assembly = typeof(ConsoleListener).Assembly; var version = assembly.GetName().Version; var assemblyResult = new AssemblyResult(assembly.Location); var conventionResult = new ConventionResult("Fake Convention"); var classResult = new ClassResult("Fake Class"); assemblyResult.Add(conventionResult); conventionResult.Add(classResult); classResult.Add(CaseResult.Passed("A", TimeSpan.Zero)); classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter()))); classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, new ExceptionInfo(new Exception(), new AssertionLibraryFilter()))); classResult.Add(CaseResult.Skipped("D", "Reason")); classResult.Add(CaseResult.Skipped("E", "Reason")); classResult.Add(CaseResult.Skipped("F", "Reason")); listener.AssemblyCompleted(assembly, assemblyResult); console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped, took 0.00 seconds (Fixie " + version + ")."); } }
public void ShouldReportPassFailSkipCounts() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); var assembly = typeof(ConsoleListener).Assembly; var version = assembly.GetName().Version; var assemblyResult = new AssemblyResult(assembly.Location); var conventionResult = new ConventionResult("Fake Convention"); var classResult = new ClassResult("Fake Class"); assemblyResult.Add(conventionResult); conventionResult.Add(classResult); classResult.Add(CaseResult.Passed("A", TimeSpan.Zero)); classResult.Add(CaseResult.Failed("B", TimeSpan.Zero, "Message", "Stack Trace", "Exception Type")); classResult.Add(CaseResult.Failed("C", TimeSpan.Zero, "Message", "Stack Trace", "Exception Type")); classResult.Add(CaseResult.Skipped("D", "Reason")); classResult.Add(CaseResult.Skipped("E", "Reason")); classResult.Add(CaseResult.Skipped("F", "Reason")); listener.AssemblyCompleted(assembly, assemblyResult); console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped, took 0.00 seconds (Fixie " + version + ")."); } }
public void ShouldReportResultsToTestDrivenDotNet() { var testDriven = new StubTestListener(); using (var console = new RedirectedConsole()) { var listener = new TestDrivenListener(testDriven); var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); convention.Parameters.Add <InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = testDriven.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.FixtureType.ShouldEqual(null); result.Method.ShouldEqual(null); result.TimeSpan.ShouldEqual(TimeSpan.Zero); result.TotalTests.ShouldEqual(0); result.TestRunnerName.ShouldBeNull(); } results[0].Name.ShouldEqual(testClass + ".SkipWithReason"); results[0].State.ShouldEqual(TestState.Ignored); results[0].Message.ShouldEqual("Skipped with reason."); results[0].StackTrace.ShouldBeNull(); results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].State.ShouldEqual(TestState.Ignored); results[1].Message.ShouldBeNull(); results[1].StackTrace.ShouldBeNull(); results[2].Name.ShouldEqual(testClass + ".Fail"); results[2].State.ShouldEqual(TestState.Failed); results[2].Message.ShouldEqual("Fixie.Tests.FailureException"); results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual( "'Fail' failed!", " at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[3].Name.ShouldEqual(testClass + ".Pass(123)"); results[3].State.ShouldEqual(TestState.Passed); results[3].Message.ShouldBeNull(); results[3].StackTrace.ShouldBeNull(); } }
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); var assembly = typeof(ConsoleListener).Assembly; var version = assembly.GetName().Version; listener.AssemblyCompleted(assembly, new AssemblyResult(1, 2, 0)); console.Lines().ShouldEqual("1 passed, 2 failed (Fixie " + version + ")."); } }
public void ShouldReportPassFailSkipCounts() { using (var console = new RedirectedConsole()) { var listener = new ConsoleListener(); var assembly = typeof(ConsoleListener).Assembly; var version = assembly.GetName().Version; listener.AssemblyCompleted(assembly, new AssemblyResult(1, 2, 3)); console.Lines().ShouldEqual("1 passed, 2 failed, 3 skipped (Fixie " + version + ")."); } }
protected Output Run() { using (var console = new RedirectedConsole()) { var listener = new StubListener(); Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal)); Convention.Execute(listener, typeof(SampleTestClass)); return(new Output(console.Lines().ToArray(), listener.Entries.ToArray())); } }
public void ShouldReportResultsToTheConsoleInTeamCityFormat() { using (var console = new RedirectedConsole()) using (var listener = new TeamCityListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations. .ShouldEqual( "##teamcity[testSuiteStarted name='Fixie.Tests.dll']", "##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']", "##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "##teamcity[testStarted name='" + testClass + ".FailA']", "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']", "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']", "##teamcity[testStarted name='" + testClass + ".FailB']", "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']", "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassA']", "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']", "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassB']", "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']", "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassC']", "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']", "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']", "##teamcity[testSuiteFinished name='Fixie.Tests.dll']"); } }
public void ShouldReportResultsToTheConsoleInTeamCityFormat() { using (var console = new RedirectedConsole()) { var listener = new TeamCityListener(); var conventionRunner = new ConventionRunner(); conventionRunner.Run(new SelfTestConvention(), listener, typeof(PassFailTestClass)); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations. .ShouldEqual( "##teamcity[testIgnored name='" + testClass + ".SkipA']", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "##teamcity[testStarted name='"+testClass+".FailA']", "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']", "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']", "##teamcity[testStarted name='" + testClass + ".FailB']", "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']", "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassA']", "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']", "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassB']", "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']", "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassC']", "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']", "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']"); } }
public void ShouldReportResultsToTheConsoleInTeamCityFormat() { using (var console = new RedirectedConsole()) { var listener = new TeamCityListener(); new SelfTestConvention().Execute(listener, typeof(PassFailTestClass)); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers. .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations. .ShouldEqual( "##teamcity[testIgnored name='" + testClass + ".SkipA']", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "##teamcity[testStarted name='" + testClass + ".FailA']", "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']", "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']", "##teamcity[testStarted name='" + testClass + ".FailB']", "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']", "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n at Fixie.Tests.Listeners.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']", "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassA']", "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']", "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassB']", "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']", "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']", "##teamcity[testStarted name='" + testClass + ".PassC']", "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']", "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']"); } }
public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention .Methods .Where(method => !method.Has <SkipAttribute>()); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldReportResultsToTheConsole() { using (var console = new RedirectedConsole()) using (var listener = new ConsoleListener()) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .Select(CleanBrittleValues) .ShouldEqual( "------ Testing Assembly Fixie.Tests.dll ------", "", "Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.", "Test '" + testClass + ".SkipWithoutReason' skipped", "Console.Out: FailA", "Console.Error: FailA", "Console.Out: FailB", "Console.Error: FailB", "Console.Out: PassA", "Console.Error: PassA", "Console.Out: PassB", "Console.Error: PassB", "Console.Out: PassC", "Console.Error: PassC", "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException", "'FailA' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #", "", "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException", "'FailB' failed!", " at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #", "", "3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ")."); } }
public void ShouldProduceValidXmlDocument() { XDocument actual = null; var listener = new ReportListener(report => actual = report); using (var console = new RedirectedConsole()) { Run(listener); console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: FailByAssertion", "Console.Error: FailByAssertion", "Console.Out: Pass", "Console.Error: Pass"); } CleanBrittleValues(actual.ToString(SaveOptions.DisableFormatting)).ShouldEqual(ExpectedReport); }
public void ShouldReportResultsToAzureDevOpsApi() { AzureListener.CreateRun start = null; var results = new List <AzureListener.Result>(); var project = Guid.NewGuid().ToString(); var accessToken = Guid.NewGuid().ToString(); var buildId = Guid.NewGuid().ToString(); var runUrl = "http://localhost:4567/run/" + Guid.NewGuid(); bool first = true; var listener = new AzureListener("http://localhost:4567", project, accessToken, buildId, (client, uri, mediaType, content) => { var actualHeader = client.DefaultRequestHeaders.Accept.Single(); actualHeader.MediaType.ShouldBe("application/json"); var actualAuthorization = client.DefaultRequestHeaders.Authorization; actualAuthorization.Scheme.ShouldBe("Bearer"); actualAuthorization.Parameter.ShouldBe(accessToken); mediaType.ShouldBe("application/json"); if (first) { first = false; uri.ShouldBe($"http://localhost:4567/{project}/_apis/test/runs?api-version=5.0"); start = Deserialize <AzureListener.CreateRun>(content); return(Serialize(new AzureListener.TestRun { url = runUrl })); } uri.ShouldBe($"{runUrl}/results?api-version=5.0"); results.Add(Deserialize <AzureListener.Result[]>(content).Single()); return(null); }); using (var console = new RedirectedConsole()) { Run(listener); console.Lines() .ShouldBe( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: FailByAssertion", "Console.Error: FailByAssertion", "Console.Out: Pass", "Console.Error: Pass"); } #if NET452 start.name.ShouldBe("Fixie.Tests (.NETFramework,Version=v4.5.2)"); #else start.name.ShouldBe("Fixie.Tests (.NETCoreApp,Version=v2.0)"); #endif start.build.id.ShouldBe(buildId); start.isAutomated.ShouldBe(true); results.Count.ShouldBe(5); var fail = results[0]; var failByAssertion = results[1]; var pass = results[2]; var skipWithReason = results[3]; var skipWithoutReason = results[4]; skipWithReason.automatedTestName.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.testCaseTitle.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.outcome.ShouldBe("None"); skipWithReason.durationInMs.ShouldBeGreaterThanOrEqualTo(0); skipWithReason.errorMessage.ShouldBe("⚠ Skipped with reason."); skipWithReason.stackTrace.ShouldBe(null); skipWithoutReason.automatedTestName.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.testCaseTitle.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.outcome.ShouldBe("None"); skipWithoutReason.durationInMs.ShouldBeGreaterThanOrEqualTo(0); skipWithoutReason.errorMessage.ShouldBe(null); skipWithoutReason.stackTrace.ShouldBe(null); fail.automatedTestName.ShouldBe(TestClass + ".Fail"); fail.testCaseTitle.ShouldBe(TestClass + ".Fail"); fail.outcome.ShouldBe("Failed"); fail.durationInMs.ShouldBeGreaterThanOrEqualTo(0); fail.errorMessage.ShouldBe("'Fail' failed!"); fail.stackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.FailureException", At("Fail()")); failByAssertion.automatedTestName.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.testCaseTitle.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.outcome.ShouldBe("Failed"); failByAssertion.durationInMs.ShouldBeGreaterThanOrEqualTo(0); failByAssertion.errorMessage.Lines().ShouldBe( "Expected: 2", "Actual: 1"); failByAssertion.stackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.Assertions.AssertException", At("FailByAssertion()")); pass.automatedTestName.ShouldBe(TestClass + ".Pass"); pass.testCaseTitle.ShouldBe(TestClass + ".Pass"); pass.outcome.ShouldBe("Passed"); pass.durationInMs.ShouldBeGreaterThanOrEqualTo(0); pass.errorMessage.ShouldBe(null); pass.stackTrace.ShouldBe(null); }
public void ShouldReportResultsToAzureDevOpsApi() { var project = Guid.NewGuid().ToString(); var accessToken = Guid.NewGuid().ToString(); var buildId = Guid.NewGuid().ToString(); var runUrl = "http://localhost:4567/run/" + Guid.NewGuid(); var requests = new List <Request>(); var batchSize = 2; bool first = true; var listener = new AzureListener("http://localhost:4567", project, accessToken, buildId, (client, method, uri, mediaType, content) => { var actualHeader = client.DefaultRequestHeaders.Accept.Single(); actualHeader.MediaType.ShouldBe("application/json"); var actualAuthorization = client.DefaultRequestHeaders.Authorization; actualAuthorization.Scheme.ShouldBe("Bearer"); actualAuthorization.Parameter.ShouldBe(accessToken); mediaType.ShouldBe("application/json"); requests.Add(new Request { Method = method, Uri = uri, Content = content }); if (first) { first = false; return(Serialize(new AzureListener.TestRun { url = runUrl })); } return(null); }, batchSize); using (var console = new RedirectedConsole()) { Run(listener); console.Lines() .ShouldBe( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: FailByAssertion", "Console.Error: FailByAssertion", "Console.Out: Pass", "Console.Error: Pass"); } var firstRequest = requests.First(); firstRequest.Method.ShouldBe(HttpMethod.Post); firstRequest.Uri.ShouldBe($"http://localhost:4567/{project}/_apis/test/runs?api-version=5.0"); var createRun = Deserialize <AzureListener.CreateRun>(firstRequest.Content); #if NET452 createRun.name.ShouldBe("Fixie.Tests (.NETFramework,Version=v4.5.2)"); #elif NETCOREAPP3_1 createRun.name.ShouldBe("Fixie.Tests (.NETCoreApp,Version=v3.1)"); #else createRun.name.ShouldBe("Fixie.Tests (.NETCoreApp,Version=v2.1)"); #endif createRun.build.id.ShouldBe(buildId); createRun.isAutomated.ShouldBe(true); var resultBatches = requests.Skip(1).Take(requests.Count - 2).Select(request => { request.Method.ShouldBe(HttpMethod.Post); request.Uri.ShouldBe($"{runUrl}/results?api-version=5.0"); return(Deserialize <AzureListener.Result[]>(request.Content)); }).ToList(); resultBatches.Count.ShouldBe(3); resultBatches[0].Length.ShouldBe(2); resultBatches[1].Length.ShouldBe(2); resultBatches[2].Length.ShouldBe(1); var results = resultBatches.SelectMany(x => x).ToList(); results.Count.ShouldBe(5); var fail = results[0]; var failByAssertion = results[1]; var pass = results[2]; var skipWithReason = results[3]; var skipWithoutReason = results[4]; skipWithReason.automatedTestName.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.testCaseTitle.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.outcome.ShouldBe("Warning"); skipWithReason.durationInMs.ShouldBeGreaterThanOrEqualTo(0); skipWithReason.errorMessage.ShouldBe("⚠ Skipped with reason."); skipWithReason.stackTrace.ShouldBe(null); skipWithoutReason.automatedTestName.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.testCaseTitle.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.outcome.ShouldBe("Warning"); skipWithoutReason.durationInMs.ShouldBeGreaterThanOrEqualTo(0); skipWithoutReason.errorMessage.ShouldBe(null); skipWithoutReason.stackTrace.ShouldBe(null); fail.automatedTestName.ShouldBe(TestClass + ".Fail"); fail.testCaseTitle.ShouldBe(TestClass + ".Fail"); fail.outcome.ShouldBe("Failed"); fail.durationInMs.ShouldBeGreaterThanOrEqualTo(0); fail.errorMessage.ShouldBe("'Fail' failed!"); fail.stackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.FailureException", At("Fail()")); failByAssertion.automatedTestName.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.testCaseTitle.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.outcome.ShouldBe("Failed"); failByAssertion.durationInMs.ShouldBeGreaterThanOrEqualTo(0); failByAssertion.errorMessage.Lines().ShouldBe( "Expected: 2", "Actual: 1"); failByAssertion.stackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.Assertions.AssertException", At("FailByAssertion()")); pass.automatedTestName.ShouldBe(TestClass + ".Pass"); pass.testCaseTitle.ShouldBe(TestClass + ".Pass"); pass.outcome.ShouldBe("Passed"); pass.durationInMs.ShouldBeGreaterThanOrEqualTo(0); pass.errorMessage.ShouldBe(null); pass.stackTrace.ShouldBe(null); var lastRequest = requests.Last(); lastRequest.Method.ShouldBe(new HttpMethod("PATCH")); lastRequest.Uri.ShouldBe($"{runUrl}?api-version=5.0"); var updateRun = Deserialize <AzureListener.UpdateRun>(lastRequest.Content); updateRun.state.ShouldBe("Completed"); }
public void ShouldReportResultsToExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); using (var console = new RedirectedConsole()) using (var listener = new VisualStudioListener(recorder, assemblyPath)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); convention.Parameters.Add <InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = recorder.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(Environment.MachineName); result.TestCase.Traits.ShouldBeEmpty(); result.TestCase.LocalExtensionData.ShouldBeNull(); result.TestCase.Source.ShouldEqual("assembly.path.dll"); //Source locations are a discovery-time concern. result.TestCase.CodeFilePath.ShouldBeNull(); result.TestCase.LineNumber.ShouldEqual(-1); } results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithReason"); results[0].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[0].Outcome.ShouldEqual(TestOutcome.Skipped); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].Messages.ShouldBeEmpty(); results[0].Duration.ShouldEqual(TimeSpan.Zero); results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[1].Outcome.ShouldEqual(TestOutcome.Skipped); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].DisplayName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].Messages.ShouldBeEmpty(); results[1].Duration.ShouldEqual(TimeSpan.Zero); results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Fail"); results[2].TestCase.DisplayName.ShouldEqual(testClass + ".Fail"); results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[2].Outcome.ShouldEqual(TestOutcome.Failed); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual( "'Fail' failed!", " at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].DisplayName.ShouldEqual(testClass + ".Fail"); results[2].Messages.Count.ShouldEqual(1); results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Pass"); results[3].TestCase.DisplayName.ShouldEqual(testClass + ".Pass"); results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[3].Outcome.ShouldEqual(TestOutcome.Passed); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].DisplayName.ShouldEqual(testClass + ".Pass(123)"); results[3].Messages.Count.ShouldEqual(1); results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); } }
public void ShouldReportResultsToTestDrivenDotNet() { var testDriven = new StubTestListener(); var listener = new TestDrivenListener(testDriven); using (var console = new RedirectedConsole()) { Run(listener); console.Lines() .ShouldBe( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: FailByAssertion", "Console.Error: FailByAssertion", "Console.Out: Pass", "Console.Error: Pass"); } var results = testDriven.TestResults; results.Count.ShouldBe(5); foreach (var result in results) { result.FixtureType.ShouldBe(null); result.Method.ShouldBe(null); result.TimeSpan.ShouldBe(TimeSpan.Zero); result.TotalTests.ShouldBe(0); result.TestRunnerName.ShouldBe(null); } var fail = results[0]; var failByAssertion = results[1]; var pass = results[2]; var skipWithReason = results[3]; var skipWithoutReason = results[4]; skipWithReason.Name.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.State.ShouldBe(TestState.Ignored); skipWithReason.Message.ShouldBe("⚠ Skipped with reason."); skipWithReason.StackTrace.ShouldBe(null); skipWithoutReason.Name.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.State.ShouldBe(TestState.Ignored); skipWithoutReason.Message.ShouldBe(null); skipWithoutReason.StackTrace.ShouldBe(null); fail.Name.ShouldBe(TestClass + ".Fail"); fail.State.ShouldBe(TestState.Failed); fail.Message.ShouldBe("Fixie.Tests.FailureException"); fail.StackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe( "'Fail' failed!", "", At("Fail()")); failByAssertion.Name.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.State.ShouldBe(TestState.Failed); failByAssertion.Message.ShouldBe("Fixie.Tests.Assertions.AssertException"); failByAssertion.StackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe( "Expected: 2", "Actual: 1", "", At("FailByAssertion()")); pass.Name.ShouldBe(TestClass + ".Pass"); pass.State.ShouldBe(TestState.Passed); pass.Message.ShouldBe(null); pass.StackTrace.ShouldBe(null); }
public void ShouldReportResultsToAppVeyorBuildWorkerApi() { var results = new List <AppVeyorListener.TestResult>(); var httpClient = new HttpClient(new FakeHandler(request => { request.ShouldNotBeNull(); request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests"); request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json")); request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8"); var requestContent = request.Content.ReadAsStringAsync().Result; results.Add(new JavaScriptSerializer().Deserialize <AppVeyorListener.TestResult>(requestContent)); return(new HttpResponseMessage { StatusCode = HttpStatusCode.Accepted }); })); using (var console = new RedirectedConsole()) using (var listener = new AppVeyorListener("http://localhost:4567", httpClient)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); convention.Parameters.Add <InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); results.Count.ShouldEqual(4); foreach (var result in results) { result.testFramework.ShouldEqual("Fixie"); result.fileName.ShouldEqual("Fixie.Tests.dll"); } results[0].testName.ShouldEqual(testClass + ".SkipWithReason"); results[0].outcome.ShouldEqual("Skipped"); results[0].durationMilliseconds.ShouldEqual("0"); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].StdOut.ShouldBeNull(); results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].outcome.ShouldEqual("Skipped"); results[1].durationMilliseconds.ShouldEqual("0"); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].StdOut.ShouldBeNull(); results[2].testName.ShouldEqual(testClass + ".Fail"); results[2].outcome.ShouldEqual("Failed"); int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual("'Fail' failed!", " at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[3].testName.ShouldEqual(testClass + ".Pass(123)"); results[3].outcome.ShouldEqual("Passed"); int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); } }
public void ShouldReportResultsToAppVeyorBuildWorkerApi() { var results = new List<AppVeyorListener.TestResult>(); var listener = new AppVeyorListener("http://localhost:4567", (uri, mediaType, content) => { uri.ShouldBe("http://localhost:4567/api/tests"); mediaType.ShouldBe("application/json"); results.Add(Deserialize<AppVeyorListener.TestResult>(content)); }); using (var console = new RedirectedConsole()) { Run(listener); console.Lines() .ShouldBe( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: FailByAssertion", "Console.Error: FailByAssertion", "Console.Out: Pass", "Console.Error: Pass"); } results.Count.ShouldBe(5); foreach (var result in results) { result.TestFramework.ShouldBe("Fixie"); #if NET452 result.FileName.ShouldBe("Fixie.Tests.exe"); #else result.FileName.ShouldBe("Fixie.Tests.dll"); #endif } var fail = results[0]; var failByAssertion = results[1]; var pass = results[2]; var skipWithReason = results[3]; var skipWithoutReason = results[4]; skipWithReason.TestName.ShouldBe(TestClass + ".SkipWithReason"); skipWithReason.Outcome.ShouldBe("Skipped"); int.Parse(skipWithReason.DurationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); skipWithReason.ErrorMessage.ShouldBe("⚠ Skipped with reason."); skipWithReason.ErrorStackTrace.ShouldBe(null); skipWithReason.StdOut.ShouldBe(""); skipWithoutReason.TestName.ShouldBe(TestClass + ".SkipWithoutReason"); skipWithoutReason.Outcome.ShouldBe("Skipped"); int.Parse(skipWithoutReason.DurationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); skipWithoutReason.ErrorMessage.ShouldBe(null); skipWithoutReason.ErrorStackTrace.ShouldBe(null); skipWithoutReason.StdOut.ShouldBe(""); fail.TestName.ShouldBe(TestClass + ".Fail"); fail.Outcome.ShouldBe("Failed"); int.Parse(fail.DurationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); fail.ErrorMessage.ShouldBe("'Fail' failed!"); fail.ErrorStackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.FailureException", At("Fail()")); fail.StdOut.Lines().ShouldBe("Console.Out: Fail", "Console.Error: Fail"); failByAssertion.TestName.ShouldBe(TestClass + ".FailByAssertion"); failByAssertion.Outcome.ShouldBe("Failed"); int.Parse(failByAssertion.DurationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); failByAssertion.ErrorMessage.Lines().ShouldBe( "Expected: 2", "Actual: 1"); failByAssertion.ErrorStackTrace .CleanStackTraceLineNumbers() .Lines() .ShouldBe("Fixie.Tests.Assertions.AssertException", At("FailByAssertion()")); failByAssertion.StdOut.Lines().ShouldBe("Console.Out: FailByAssertion", "Console.Error: FailByAssertion"); pass.TestName.ShouldBe(TestClass + ".Pass"); pass.Outcome.ShouldBe("Passed"); int.Parse(pass.DurationMilliseconds).ShouldBeGreaterThanOrEqualTo(0); pass.ErrorMessage.ShouldBe(null); pass.ErrorStackTrace.ShouldBe(null); pass.StdOut.Lines().ShouldBe("Console.Out: Pass", "Console.Error: Pass"); }
protected Output Run() { using (var console = new RedirectedConsole()) { var listener = new StubListener(); Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal)); Convention.Execute(listener, typeof(SampleTestClass)); return new Output(console.Lines().ToArray(), listener.Entries.ToArray()); } }