public void ShouldReportResultsToExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); using (var console = new RedirectedConsole()) using (var listener = new VisualStudioListener(recorder, assemblyPath)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason); convention.Parameters.Add<InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = recorder.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(Environment.MachineName); result.TestCase.Traits.ShouldBeEmpty(); result.TestCase.LocalExtensionData.ShouldBeNull(); result.TestCase.Source.ShouldEqual("assembly.path.dll"); //Source locations are a discovery-time concern. result.TestCase.CodeFilePath.ShouldBeNull(); result.TestCase.LineNumber.ShouldEqual(-1); } results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason"); results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[0].Outcome.ShouldEqual(TestOutcome.Skipped); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].Messages.ShouldBeEmpty(); results[0].Duration.ShouldEqual(TimeSpan.Zero); results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[1].Outcome.ShouldEqual(TestOutcome.Skipped); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason"); results[1].Messages.ShouldBeEmpty(); results[1].Duration.ShouldEqual(TimeSpan.Zero); results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail"); results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail"); results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[2].Outcome.ShouldEqual(TestOutcome.Failed); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual( "'Fail' failed!", " at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].DisplayName.ShouldEqual(testClass + ".Fail"); results[2].Messages.Count.ShouldEqual(1); results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass"); results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass"); results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[3].Outcome.ShouldEqual(TestOutcome.Passed); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].DisplayName.ShouldEqual(testClass +".Pass(123)"); results[3].Messages.Count.ShouldEqual(1); results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); } }
public void ShouldReportResultsToExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); using (var console = new RedirectedConsole()) using (var listener = new VisualStudioListener(recorder, assemblyPath)) { var convention = SelfTestConvention.Build(); convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason); convention.Parameters.Add <InputAttributeParameterSource>(); typeof(PassFailTestClass).Run(listener, convention); var testClass = typeof(PassFailTestClass).FullName; console.Lines() .ShouldEqual( "Console.Out: Fail", "Console.Error: Fail", "Console.Out: Pass", "Console.Error: Pass"); var results = recorder.TestResults; results.Count.ShouldEqual(4); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(Environment.MachineName); result.TestCase.Traits.ShouldBeEmpty(); result.TestCase.LocalExtensionData.ShouldBeNull(); result.TestCase.Source.ShouldEqual("assembly.path.dll"); //Source locations are a discovery-time concern. result.TestCase.CodeFilePath.ShouldBeNull(); result.TestCase.LineNumber.ShouldEqual(-1); } results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithReason"); results[0].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[0].Outcome.ShouldEqual(TestOutcome.Skipped); results[0].ErrorMessage.ShouldEqual("Skipped with reason."); results[0].ErrorStackTrace.ShouldBeNull(); results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason"); results[0].Messages.ShouldBeEmpty(); results[0].Duration.ShouldEqual(TimeSpan.Zero); results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[1].Outcome.ShouldEqual(TestOutcome.Skipped); results[1].ErrorMessage.ShouldBeNull(); results[1].ErrorStackTrace.ShouldBeNull(); results[1].DisplayName.ShouldEqual(testClass + ".SkipWithoutReason"); results[1].Messages.ShouldBeEmpty(); results[1].Duration.ShouldEqual(TimeSpan.Zero); results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Fail"); results[2].TestCase.DisplayName.ShouldEqual(testClass + ".Fail"); results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[2].Outcome.ShouldEqual(TestOutcome.Failed); results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException"); results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues) .ShouldEqual( "'Fail' failed!", " at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #"); results[2].DisplayName.ShouldEqual(testClass + ".Fail"); results[2].Messages.Count.ShouldEqual(1); results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail"); results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Pass"); results[3].TestCase.DisplayName.ShouldEqual(testClass + ".Pass"); results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/"); results[3].Outcome.ShouldEqual(TestOutcome.Passed); results[3].ErrorMessage.ShouldBeNull(); results[3].ErrorStackTrace.ShouldBeNull(); results[3].DisplayName.ShouldEqual(testClass + ".Pass(123)"); results[3].Messages.Count.ShouldEqual(1); results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass"); results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); } }
public void ShouldMapResultsToVisualStudioExecutionRecorder() { const string assemblyPath = "assembly.path.dll"; var recorder = new StubExecutionRecorder(); var executionRecorder = new ExecutionRecorder(recorder, assemblyPath); executionRecorder.Record(new PipeMessage.CasePassed { Test = new PipeMessage.Test { Class = "Namespace.Class", Method = "Pass", Name = "Namespace.Class.Pass", }, Name = "Namespace.Class.Pass(1)", Duration = TimeSpan.FromSeconds(1), Output = "Output" }); executionRecorder.Record(new PipeMessage.CaseFailed { Test = new PipeMessage.Test { Class = "Namespace.Class", Method = "Fail", Name = "Namespace.Class.Fail", }, Name = "Namespace.Class.Fail", Duration = TimeSpan.FromSeconds(2), Output = "Output", Exception = new PipeMessage.Exception { Type = "Exception Type", Message = "Exception Message", StackTrace = "Exception Stack Trace" } }); executionRecorder.Record(new PipeMessage.CaseSkipped { Test = new PipeMessage.Test { Class = "Namespace.Class", Method = "Skip", Name = "Namespace.Class.Skip", }, Name = "Namespace.Class.Skip", Duration = TimeSpan.Zero, Output = null, Reason = "Skip Reason" }); var results = recorder.TestResults; results.Count.ShouldEqual(3); foreach (var result in results) { result.Traits.ShouldBeEmpty(); result.Attachments.ShouldBeEmpty(); result.ComputerName.ShouldEqual(MachineName); } var pass = results[0]; var fail = results[1]; var skip = results[2]; pass.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Pass", assemblyPath); pass.TestCase.DisplayName.ShouldEqual("Namespace.Class.Pass"); pass.Outcome.ShouldEqual(TestOutcome.Passed); pass.ErrorMessage.ShouldBeNull(); pass.ErrorStackTrace.ShouldBeNull(); pass.DisplayName.ShouldEqual("Namespace.Class.Pass(1)"); pass.Messages.Count.ShouldEqual(1); pass.Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); pass.Messages[0].Text.ShouldEqual("Output"); pass.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); fail.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Fail", assemblyPath); fail.TestCase.DisplayName.ShouldEqual("Namespace.Class.Fail"); fail.Outcome.ShouldEqual(TestOutcome.Failed); fail.ErrorMessage.ShouldEqual("Exception Message"); fail.ErrorStackTrace.ShouldEqual("Exception Type" + NewLine + "Exception Stack Trace"); fail.DisplayName.ShouldEqual("Namespace.Class.Fail"); fail.Messages.Count.ShouldEqual(1); fail.Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory); fail.Messages[0].Text.ShouldEqual("Output"); fail.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero); skip.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Skip", assemblyPath); skip.TestCase.DisplayName.ShouldEqual("Namespace.Class.Skip"); skip.Outcome.ShouldEqual(TestOutcome.Skipped); skip.ErrorMessage.ShouldEqual("Skip Reason"); skip.ErrorStackTrace.ShouldBeNull(); skip.DisplayName.ShouldEqual("Namespace.Class.Skip"); skip.Messages.ShouldBeEmpty(); skip.Duration.ShouldEqual(TimeSpan.Zero); }