public void ShouldReportResultsToExecutionRecorder()
{
const string assemblyPath = "assembly.path.dll";
var recorder = new StubExecutionRecorder();
using (var console = new RedirectedConsole())
using (var listener = new VisualStudioListener(recorder, assemblyPath))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
convention.Parameters.Add<InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = recorder.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.Traits.ShouldBeEmpty();
result.Attachments.ShouldBeEmpty();
result.ComputerName.ShouldEqual(Environment.MachineName);
result.TestCase.Traits.ShouldBeEmpty();
result.TestCase.LocalExtensionData.ShouldBeNull();
result.TestCase.Source.ShouldEqual("assembly.path.dll");
//Source locations are a discovery-time concern.
result.TestCase.CodeFilePath.ShouldBeNull();
result.TestCase.LineNumber.ShouldEqual(-1);
}
results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason");
results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason");
results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[0].Outcome.ShouldEqual(TestOutcome.Skipped);
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].Messages.ShouldBeEmpty();
results[0].Duration.ShouldEqual(TimeSpan.Zero);
results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[1].Outcome.ShouldEqual(TestOutcome.Skipped);
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
results[1].Messages.ShouldBeEmpty();
results[1].Duration.ShouldEqual(TimeSpan.Zero);
results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail");
results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail");
results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[2].Outcome.ShouldEqual(TestOutcome.Failed);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].DisplayName.ShouldEqual(testClass + ".Fail");
results[2].Messages.Count.ShouldEqual(1);
results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass");
results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass");
results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[3].Outcome.ShouldEqual(TestOutcome.Passed);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].DisplayName.ShouldEqual(testClass +".Pass(123)");
results[3].Messages.Count.ShouldEqual(1);
results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
}
}
public void ShouldReportResultsToExecutionRecorder()
{
const string assemblyPath = "assembly.path.dll";
var recorder = new StubExecutionRecorder();
using (var console = new RedirectedConsole())
using (var listener = new VisualStudioListener(recorder, assemblyPath))
{
var convention = SelfTestConvention.Build();
convention.CaseExecution.Skip(x => x.Method.Has <SkipAttribute>(), x => x.Method.GetCustomAttribute <SkipAttribute>().Reason);
convention.Parameters.Add <InputAttributeParameterSource>();
typeof(PassFailTestClass).Run(listener, convention);
var testClass = typeof(PassFailTestClass).FullName;
console.Lines()
.ShouldEqual(
"Console.Out: Fail",
"Console.Error: Fail",
"Console.Out: Pass",
"Console.Error: Pass");
var results = recorder.TestResults;
results.Count.ShouldEqual(4);
foreach (var result in results)
{
result.Traits.ShouldBeEmpty();
result.Attachments.ShouldBeEmpty();
result.ComputerName.ShouldEqual(Environment.MachineName);
result.TestCase.Traits.ShouldBeEmpty();
result.TestCase.LocalExtensionData.ShouldBeNull();
result.TestCase.Source.ShouldEqual("assembly.path.dll");
//Source locations are a discovery-time concern.
result.TestCase.CodeFilePath.ShouldBeNull();
result.TestCase.LineNumber.ShouldEqual(-1);
}
results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithReason");
results[0].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[0].Outcome.ShouldEqual(TestOutcome.Skipped);
results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
results[0].ErrorStackTrace.ShouldBeNull();
results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason");
results[0].Messages.ShouldBeEmpty();
results[0].Duration.ShouldEqual(TimeSpan.Zero);
results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].TestCase.DisplayName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[1].Outcome.ShouldEqual(TestOutcome.Skipped);
results[1].ErrorMessage.ShouldBeNull();
results[1].ErrorStackTrace.ShouldBeNull();
results[1].DisplayName.ShouldEqual(testClass + ".SkipWithoutReason");
results[1].Messages.ShouldBeEmpty();
results[1].Duration.ShouldEqual(TimeSpan.Zero);
results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Fail");
results[2].TestCase.DisplayName.ShouldEqual(testClass + ".Fail");
results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[2].Outcome.ShouldEqual(TestOutcome.Failed);
results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
.ShouldEqual(
"'Fail' failed!",
" at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
results[2].DisplayName.ShouldEqual(testClass + ".Fail");
results[2].Messages.Count.ShouldEqual(1);
results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass + ".Pass");
results[3].TestCase.DisplayName.ShouldEqual(testClass + ".Pass");
results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
results[3].Outcome.ShouldEqual(TestOutcome.Passed);
results[3].ErrorMessage.ShouldBeNull();
results[3].ErrorStackTrace.ShouldBeNull();
results[3].DisplayName.ShouldEqual(testClass + ".Pass(123)");
results[3].Messages.Count.ShouldEqual(1);
results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
}
}
public void ShouldMapResultsToVisualStudioExecutionRecorder()
{
const string assemblyPath = "assembly.path.dll";
var recorder = new StubExecutionRecorder();
var executionRecorder = new ExecutionRecorder(recorder, assemblyPath);
executionRecorder.Record(new PipeMessage.CasePassed
{
Test = new PipeMessage.Test
{
Class = "Namespace.Class",
Method = "Pass",
Name = "Namespace.Class.Pass",
},
Name = "Namespace.Class.Pass(1)",
Duration = TimeSpan.FromSeconds(1),
Output = "Output"
});
executionRecorder.Record(new PipeMessage.CaseFailed
{
Test = new PipeMessage.Test
{
Class = "Namespace.Class",
Method = "Fail",
Name = "Namespace.Class.Fail",
},
Name = "Namespace.Class.Fail",
Duration = TimeSpan.FromSeconds(2),
Output = "Output",
Exception = new PipeMessage.Exception
{
Type = "Exception Type",
Message = "Exception Message",
StackTrace = "Exception Stack Trace"
}
});
executionRecorder.Record(new PipeMessage.CaseSkipped
{
Test = new PipeMessage.Test
{
Class = "Namespace.Class",
Method = "Skip",
Name = "Namespace.Class.Skip",
},
Name = "Namespace.Class.Skip",
Duration = TimeSpan.Zero,
Output = null,
Reason = "Skip Reason"
});
var results = recorder.TestResults;
results.Count.ShouldEqual(3);
foreach (var result in results)
{
result.Traits.ShouldBeEmpty();
result.Attachments.ShouldBeEmpty();
result.ComputerName.ShouldEqual(MachineName);
}
var pass = results[0];
var fail = results[1];
var skip = results[2];
pass.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Pass", assemblyPath);
pass.TestCase.DisplayName.ShouldEqual("Namespace.Class.Pass");
pass.Outcome.ShouldEqual(TestOutcome.Passed);
pass.ErrorMessage.ShouldBeNull();
pass.ErrorStackTrace.ShouldBeNull();
pass.DisplayName.ShouldEqual("Namespace.Class.Pass(1)");
pass.Messages.Count.ShouldEqual(1);
pass.Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
pass.Messages[0].Text.ShouldEqual("Output");
pass.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
fail.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Fail", assemblyPath);
fail.TestCase.DisplayName.ShouldEqual("Namespace.Class.Fail");
fail.Outcome.ShouldEqual(TestOutcome.Failed);
fail.ErrorMessage.ShouldEqual("Exception Message");
fail.ErrorStackTrace.ShouldEqual("Exception Type" + NewLine + "Exception Stack Trace");
fail.DisplayName.ShouldEqual("Namespace.Class.Fail");
fail.Messages.Count.ShouldEqual(1);
fail.Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
fail.Messages[0].Text.ShouldEqual("Output");
fail.Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
skip.TestCase.ShouldBeExecutionTimeTest("Namespace.Class.Skip", assemblyPath);
skip.TestCase.DisplayName.ShouldEqual("Namespace.Class.Skip");
skip.Outcome.ShouldEqual(TestOutcome.Skipped);
skip.ErrorMessage.ShouldEqual("Skip Reason");
skip.ErrorStackTrace.ShouldBeNull();
skip.DisplayName.ShouldEqual("Namespace.Class.Skip");
skip.Messages.ShouldBeEmpty();
skip.Duration.ShouldEqual(TimeSpan.Zero);
}
