public static void WithTestNode_ParsesNumberWithInvariantCulture()
{
var sink = Substitute.For<IMessageSink>();
var testCollection = new Xunit1TestCollection("assembly");
var testCase = new Xunit1TestCase("assembly", "foo", "bar", "foo.bar") { TestCollection = testCollection };
var handler = new TestClassCallbackHandler(new [] { testCase }, sink);
var xml = new XmlDocument();
xml.LoadXml("<test type='foo' method='bar' name='foo.bar' time='1.234' result='Pass' />");
handler.OnXmlNode(xml.FirstChild);
var args = sink.Captured(1, x => x.OnMessage(null));
var message = args.Arg<ITestFinished>();
Assert.Equal(1.234M, message.ExecutionTime);
}
public void NestedExceptionsThrownDuringClassStart_ResultsInErrorMessage()
{
var testCollection = new Xunit1TestCollection("AssemblyName.dll");
var testCases = new[] {
new Xunit1TestCase("assembly", "failingtype", "passingmethod", "failingtype.passingmethod") { TestCollection = testCollection }
};
var exception = GetNestedExceptions();
var xunit1 = new TestableXunit1("AssemblyName.dll", "ConfigFile.config");
xunit1.Executor.TestFrameworkDisplayName.Returns("Test framework display name");
xunit1.Executor
.When(x => x.RunTests("failingtype", Arg.Any<List<string>>(),
Arg.Any<ICallbackEventHandler>()))
.Do(callInfo =>
{
var callback = callInfo.Arg<ICallbackEventHandler>();
callback.RaiseCallbackEvent("<start name='failingtype.passingmethod' type='failingtype' method='passingmethod'/>");
callback.RaiseCallbackEvent("<test name='failingtype.passingmethod' type='failingtype' method='passingmethod' result='Pass' time='1.000'/>");
callback.RaiseCallbackEvent(string.Format("<class name='failingtype' time='0.000' total='0' passed='1' failed='1' skipped='0'><failure exception-type='System.InvalidOperationException'><message>{0}</message><stack-trace><![CDATA[{1}]]></stack-trace></failure></class>", GetMessage(exception), GetStackTrace(exception)));
});
var sink = new SpyMessageSink<ITestAssemblyFinished>();
xunit1.Run(testCases, sink);
sink.Finished.WaitOne();
var errorMessage = Assert.Single(sink.Messages.OfType<IErrorMessage>());
Assert.Equal(exception.GetType().FullName, errorMessage.ExceptionTypes[0]);
Assert.Equal(exception.InnerException.GetType().FullName, errorMessage.ExceptionTypes[1]);
Assert.Equal(exception.Message, errorMessage.Messages[0]);
Assert.Equal(exception.InnerException.Message, errorMessage.Messages[1]);
Assert.Equal(exception.StackTrace, errorMessage.StackTraces[0]);
Assert.Equal(exception.InnerException.StackTrace, errorMessage.StackTraces[1]);
}
public void ExceptionThrownDuringClassFinish_ResultsInErrorMessage()
{
var testCollection = new Xunit1TestCollection("AssemblyName.dll");
var testCases = new[] {
new Xunit1TestCase("assembly", "failingtype", "passingmethod", "failingtype.passingmethod") { TestCollection = testCollection }
};
var exception = new InvalidOperationException("Cannot use a test class as its own fixture data");
var xunit1 = new TestableXunit1("AssemblyName.dll", "ConfigFile.config");
xunit1.Executor.TestFrameworkDisplayName.Returns("Test framework display name");
xunit1.Executor
.When(x => x.RunTests("failingtype", Arg.Any<List<string>>(),
Arg.Any<ICallbackEventHandler>()))
.Do(callInfo =>
{
// Ensure the exception has a callstack
try { throw exception; } catch{}
var callback = callInfo.Arg<ICallbackEventHandler>();
callback.RaiseCallbackEvent("<start name='failingtype.passingmethod' type='failingtype' method='passingmethod'/>");
callback.RaiseCallbackEvent("<test name='failingtype.passingmethod' type='failingtype' method='passingmethod' result='Pass' time='1.000'/>");
callback.RaiseCallbackEvent(string.Format("<class name='failingtype' time='0.000' total='0' passed='1' failed='1' skipped='0'><failure exception-type='System.InvalidOperationException'><message>Cannot use a test class as its own fixture data</message><stack-trace><![CDATA[{0}]]></stack-trace></failure></class>", exception.StackTrace));
});
var sink = new SpyMessageSink<ITestAssemblyFinished>();
xunit1.Run(testCases, sink);
sink.Finished.WaitOne();
var errorMessage = Assert.Single(sink.Messages.OfType<IErrorMessage>());
Assert.Equal("System.InvalidOperationException", errorMessage.ExceptionTypes.Single());
Assert.Equal("Cannot use a test class as its own fixture data", errorMessage.Messages.Single());
Assert.Equal(exception.StackTrace, errorMessage.StackTraces.Single());
}
public void NestedExceptionResultFromTests_ResultsInErrorMessage()
{
var testCollection = new Xunit1TestCollection("AssemblyName.dll");
var testCases = new[] {
new Xunit1TestCase("assembly", "type1", "failing", "type1.failing") { TestCollection = testCollection }
};
var exception = GetNestedExceptions();
var xunit1 = new TestableXunit1("AssemblyName.dll", "ConfigFile.config");
xunit1.Executor.TestFrameworkDisplayName.Returns("Test framework display name");
xunit1.Executor
.WhenForAnyArgs(x => x.RunTests(null, null, null))
.Do(callInfo =>
{
var callback = callInfo.Arg<ICallbackEventHandler>();
callback.RaiseCallbackEvent("<start name='type1.failing' type='type1' method='failing'/>");
callback.RaiseCallbackEvent(string.Format("<test name='type1.failing' type='type1' method='failing' result='Fail' time='0.234'><failure exception-type='{0}'><message>{1}</message><stack-trace><![CDATA[{2}]]></stack-trace></failure></test>", exception.GetType().FullName, GetMessage(exception), GetStackTrace(exception)));
callback.RaiseCallbackEvent("<class name='type1' time='1.234' total='1' failed='1' skipped='0'/>");
});
var sink = new SpyMessageSink<ITestAssemblyFinished>();
xunit1.Run(testCases, sink);
sink.Finished.WaitOne();
var testFailed = Assert.Single(sink.Messages.OfType<ITestFailed>());
Assert.Equal(exception.GetType().FullName, testFailed.ExceptionTypes[0]);
Assert.Equal(exception.InnerException.GetType().FullName, testFailed.ExceptionTypes[1]);
Assert.Equal(exception.Message, testFailed.Messages[0]);
Assert.Equal(exception.InnerException.Message, testFailed.Messages[1]);
Assert.Equal(exception.StackTrace, testFailed.StackTraces[0]);
Assert.Equal(exception.InnerException.StackTrace, testFailed.StackTraces[1]);
}
public void NestedExceptionsThrownDuringRunTests_ResultsInErrorMessage()
{
var testCollection = new Xunit1TestCollection("AssemblyName.dll");
var testCases = new[] {
new Xunit1TestCase("assembly", "type1", "passing", "type1.passing") { TestCollection = testCollection }
};
var exception = GetNestedExceptions();
var xunit1 = new TestableXunit1("AssemblyName.dll", "ConfigFile.config");
xunit1.Executor.TestFrameworkDisplayName.Returns("Test framework display name");
xunit1.Executor
.WhenForAnyArgs(x => x.RunTests(null, null, null))
.Do(callInfo => { throw exception; });
var sink = new SpyMessageSink<ITestAssemblyFinished>();
xunit1.Run(testCases, sink);
sink.Finished.WaitOne();
var errorMessage = Assert.Single(sink.Messages.OfType<IErrorMessage>());
Assert.Equal(exception.GetType().FullName, errorMessage.ExceptionTypes[0]);
Assert.Equal(exception.InnerException.GetType().FullName, errorMessage.ExceptionTypes[1]);
Assert.Equal(exception.Message, errorMessage.Messages[0]);
Assert.Equal(exception.InnerException.Message, errorMessage.Messages[1]);
Assert.Equal(exception.StackTrace, errorMessage.StackTraces[0]);
Assert.Equal(exception.InnerException.StackTrace, errorMessage.StackTraces[1]);
}
public void RunWithTestCases()
{
var testCollection = new Xunit1TestCollection("AssemblyName.dll");
var testCases = new[] {
new Xunit1TestCase("assembly", "type1", "passing", "type1.passing") { TestCollection = testCollection },
new Xunit1TestCase("assembly", "type1", "failing", "type1.failing") { TestCollection = testCollection },
new Xunit1TestCase("assembly", "type2", "skipping", "type2.skipping") { TestCollection = testCollection },
new Xunit1TestCase("assembly", "type2", "skipping_with_start", "type2.skipping_with_start") { TestCollection = testCollection }
};
var xunit1 = new TestableXunit1("AssemblyName.dll", "ConfigFile.config");
xunit1.Executor.TestFrameworkDisplayName.Returns("Test framework display name");
xunit1.Executor
.When(x => x.RunTests("type1", Arg.Any<List<string>>(), Arg.Any<ICallbackEventHandler>()))
.Do(callInfo =>
{
var callback = callInfo.Arg<ICallbackEventHandler>();
callback.RaiseCallbackEvent("<start name='type1.passing' type='type1' method='passing'/>");
callback.RaiseCallbackEvent("<test name='type1.passing' type='type1' method='passing' result='Pass' time='1.000'/>");
callback.RaiseCallbackEvent("<start name='type1.failing' type='type1' method='failing'/>");
callback.RaiseCallbackEvent("<test name='type1.failing' type='type1' method='failing' result='Fail' time='0.234'><failure exception-type='Xunit.MockFailureException'><message>Failure message</message><stack-trace>Stack trace</stack-trace></failure></test>");
callback.RaiseCallbackEvent("<class name='type1' time='1.234' total='2' failed='1' skipped='0'/>");
});
xunit1.Executor
.When(x => x.RunTests("type2", Arg.Any<List<string>>(), Arg.Any<ICallbackEventHandler>()))
.Do(callInfo =>
{
var callback = callInfo.Arg<ICallbackEventHandler>();
// Note. Skip does not send a start packet, unless you use a custom Fact
callback.RaiseCallbackEvent("<test name='type2.skipping' type='type2' method='skipping' result='Skip'><reason><message>Skip message</message></reason></test>");
callback.RaiseCallbackEvent("<start name='type2.skipping_with_start' type='type2' method='skipping_with_start'/>");
callback.RaiseCallbackEvent("<test name='type2.skipping_with_start' type='type2' method='skipping_with_start' result='Skip'><reason><message>Skip message</message></reason></test>");
callback.RaiseCallbackEvent("<class name='type2' time='0.000' total='1' failed='0' skipped='1'/>");
});
var sink = new SpyMessageSink<ITestAssemblyFinished>();
xunit1.Run(testCases, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.Messages,
message =>
{
var assemblyStarting = Assert.IsAssignableFrom<ITestAssemblyStarting>(message);
Assert.Equal("AssemblyName.dll", assemblyStarting.AssemblyFileName);
Assert.Equal("ConfigFile.config", assemblyStarting.ConfigFileName);
Assert.Contains("-bit .NET ", assemblyStarting.TestEnvironment);
Assert.Equal("Test framework display name", assemblyStarting.TestFrameworkDisplayName);
},
message =>
{
var testCollectionStarting = Assert.IsAssignableFrom<ITestCollectionStarting>(message);
Assert.Same(testCollection, testCollectionStarting.TestCollection);
},
message =>
{
var testClassStarting = Assert.IsAssignableFrom<ITestClassStarting>(message);
Assert.Equal("type1", testClassStarting.ClassName);
Assert.Same(testCollection, testClassStarting.TestCollection);
},
message =>
{
var testMethodStarting = Assert.IsAssignableFrom<ITestMethodStarting>(message);
Assert.Equal("type1", testMethodStarting.ClassName);
Assert.Equal("passing", testMethodStarting.MethodName);
},
message =>
{
var testCaseStarting = Assert.IsAssignableFrom<ITestCaseStarting>(message);
Assert.Equal("type1.passing", testCaseStarting.TestCase.DisplayName);
},
message =>
{
var testStarting = Assert.IsAssignableFrom<ITestStarting>(message);
Assert.Equal("type1.passing", testStarting.TestCase.DisplayName);
Assert.Equal("type1", testStarting.TestCase.Class.Name);
Assert.Equal("passing", testStarting.TestCase.Method.Name);
Assert.Same(testCollection, testStarting.TestCase.TestCollection);
},
message =>
{
var testPassed = Assert.IsAssignableFrom<ITestPassed>(message);
Assert.Equal("type1.passing", testPassed.TestCase.DisplayName);
Assert.Equal(1M, testPassed.ExecutionTime);
},
message =>
{
var testFinished = Assert.IsAssignableFrom<ITestFinished>(message);
Assert.Equal("type1.passing", testFinished.TestCase.DisplayName);
},
message =>
{
var testCaseFinished = Assert.IsAssignableFrom<ITestCaseFinished>(message);
Assert.Equal("type1.passing", testCaseFinished.TestCase.DisplayName);
Assert.Equal(1M, testCaseFinished.ExecutionTime);
Assert.Equal(0, testCaseFinished.TestsFailed);
Assert.Equal(1, testCaseFinished.TestsRun);
Assert.Equal(0, testCaseFinished.TestsSkipped);
},
message =>
{
var testMethodFinished = Assert.IsAssignableFrom<ITestMethodFinished>(message);
Assert.Equal("type1", testMethodFinished.ClassName);
Assert.Equal("passing", testMethodFinished.MethodName);
},
message =>
{
var testMethodStarting = Assert.IsAssignableFrom<ITestMethodStarting>(message);
Assert.Equal("type1", testMethodStarting.ClassName);
Assert.Equal("failing", testMethodStarting.MethodName);
},
message =>
{
var testCaseStarting = Assert.IsAssignableFrom<ITestCaseStarting>(message);
Assert.Equal("type1.failing", testCaseStarting.TestCase.DisplayName);
},
message =>
{
var testStarting = Assert.IsAssignableFrom<ITestStarting>(message);
Assert.Equal("type1.failing", testStarting.TestCase.DisplayName);
Assert.Equal("type1", testStarting.TestCase.Class.Name);
Assert.Equal("failing", testStarting.TestCase.Method.Name);
Assert.Same(testCollection, testStarting.TestCase.TestCollection);
},
message =>
{
var testFailed = Assert.IsAssignableFrom<ITestFailed>(message);
Assert.Equal("type1.failing", testFailed.TestCase.DisplayName);
Assert.Equal(0.234M, testFailed.ExecutionTime);
Assert.Equal("Xunit.MockFailureException", testFailed.ExceptionTypes.Single());
Assert.Equal("Failure message", testFailed.Messages.Single());
Assert.Equal("Stack trace", testFailed.StackTraces.Single());
},
message =>
{
var testFinished = Assert.IsAssignableFrom<ITestFinished>(message);
Assert.Equal("type1.failing", testFinished.TestCase.DisplayName);
},
message =>
{
var testCaseFinished = Assert.IsAssignableFrom<ITestCaseFinished>(message);
Assert.Equal("type1.failing", testCaseFinished.TestCase.DisplayName);
Assert.Equal(0.234M, testCaseFinished.ExecutionTime);
Assert.Equal(1, testCaseFinished.TestsFailed);
Assert.Equal(1, testCaseFinished.TestsRun);
Assert.Equal(0, testCaseFinished.TestsSkipped);
},
message =>
{
var testMethodFinished = Assert.IsAssignableFrom<ITestMethodFinished>(message);
Assert.Equal("type1", testMethodFinished.ClassName);
Assert.Equal("failing", testMethodFinished.MethodName);
},
message =>
{
var testClassFinished = Assert.IsAssignableFrom<ITestClassFinished>(message);
Assert.Equal("type1", testClassFinished.ClassName);
Assert.Equal(1.234M, testClassFinished.ExecutionTime);
Assert.Equal(1, testClassFinished.TestsFailed);
Assert.Equal(2, testClassFinished.TestsRun);
Assert.Equal(0, testClassFinished.TestsSkipped);
},
message =>
{
var testClassStarting = Assert.IsAssignableFrom<ITestClassStarting>(message);
Assert.Equal("type2", testClassStarting.ClassName);
Assert.Same(testCollection, testClassStarting.TestCollection);
},
message =>
{
var testMethodStarting = Assert.IsAssignableFrom<ITestMethodStarting>(message);
Assert.Equal("type2", testMethodStarting.ClassName);
Assert.Equal("skipping", testMethodStarting.MethodName);
},
message =>
{
var testCaseStarting = Assert.IsAssignableFrom<ITestCaseStarting>(message);
Assert.Equal("type2.skipping", testCaseStarting.TestCase.DisplayName);
},
message =>
{
var testStarting = Assert.IsAssignableFrom<ITestStarting>(message);
Assert.Equal("type2.skipping", testStarting.TestCase.DisplayName);
Assert.Equal("type2", testStarting.TestCase.Class.Name);
Assert.Equal("skipping", testStarting.TestCase.Method.Name);
Assert.Same(testCollection, testStarting.TestCase.TestCollection);
},
message =>
{
var testSkipped = Assert.IsAssignableFrom<ITestSkipped>(message);
Assert.Equal("type2.skipping", testSkipped.TestCase.DisplayName);
Assert.Equal(0M, testSkipped.ExecutionTime);
Assert.Equal("Skip message", testSkipped.Reason);
},
message =>
{
var testFinished = Assert.IsAssignableFrom<ITestFinished>(message);
Assert.Equal("type2.skipping", testFinished.TestCase.DisplayName);
},
message =>
{
var testCaseFinished = Assert.IsAssignableFrom<ITestCaseFinished>(message);
Assert.Equal("type2.skipping", testCaseFinished.TestCase.DisplayName);
Assert.Equal(0M, testCaseFinished.ExecutionTime);
Assert.Equal(0, testCaseFinished.TestsFailed);
Assert.Equal(1, testCaseFinished.TestsRun);
Assert.Equal(1, testCaseFinished.TestsSkipped);
},
message =>
{
var testMethodFinished = Assert.IsAssignableFrom<ITestMethodFinished>(message);
Assert.Equal("type2", testMethodFinished.ClassName);
Assert.Equal("skipping", testMethodFinished.MethodName);
},
message =>
{
var testMethodStarting = Assert.IsAssignableFrom<ITestMethodStarting>(message);
Assert.Equal("type2", testMethodStarting.ClassName);
Assert.Equal("skipping_with_start", testMethodStarting.MethodName);
},
message =>
{
var testCaseStarting = Assert.IsAssignableFrom<ITestCaseStarting>(message);
Assert.Equal("type2.skipping_with_start", testCaseStarting.TestCase.DisplayName);
},
message =>
{
var testStarting = Assert.IsAssignableFrom<ITestStarting>(message);
Assert.Equal("type2.skipping_with_start", testStarting.TestCase.DisplayName);
Assert.Equal("type2", testStarting.TestCase.Class.Name);
Assert.Equal("skipping_with_start", testStarting.TestCase.Method.Name);
Assert.Same(testCollection, testStarting.TestCase.TestCollection);
},
message =>
{
var testSkipped = Assert.IsAssignableFrom<ITestSkipped>(message);
Assert.Equal("type2.skipping_with_start", testSkipped.TestCase.DisplayName);
Assert.Equal(0M, testSkipped.ExecutionTime);
Assert.Equal("Skip message", testSkipped.Reason);
},
message =>
{
var testFinished = Assert.IsAssignableFrom<ITestFinished>(message);
Assert.Equal("type2.skipping_with_start", testFinished.TestCase.DisplayName);
},
message =>
{
var testCaseFinished = Assert.IsAssignableFrom<ITestCaseFinished>(message);
Assert.Equal("type2.skipping_with_start", testCaseFinished.TestCase.DisplayName);
Assert.Equal(0M, testCaseFinished.ExecutionTime);
Assert.Equal(0, testCaseFinished.TestsFailed);
Assert.Equal(1, testCaseFinished.TestsRun);
Assert.Equal(1, testCaseFinished.TestsSkipped);
},
message =>
{
var testMethodFinished = Assert.IsAssignableFrom<ITestMethodFinished>(message);
Assert.Equal("type2", testMethodFinished.ClassName);
Assert.Equal("skipping_with_start", testMethodFinished.MethodName);
},
message =>
{
var testClassFinished = Assert.IsAssignableFrom<ITestClassFinished>(message);
Assert.Equal("type2", testClassFinished.ClassName);
Assert.Equal(0M, testClassFinished.ExecutionTime);
Assert.Equal(0, testClassFinished.TestsFailed);
Assert.Equal(1, testClassFinished.TestsRun);
Assert.Equal(1, testClassFinished.TestsSkipped);
},
message =>
{
var testCollectionFinished = Assert.IsAssignableFrom<ITestCollectionFinished>(message);
Assert.Equal(1.234M, testCollectionFinished.ExecutionTime);
Assert.Equal(1, testCollectionFinished.TestsFailed);
Assert.Equal(3, testCollectionFinished.TestsRun);
Assert.Equal(1, testCollectionFinished.TestsSkipped);
Assert.Same(testCollection, testCollectionFinished.TestCollection);
},
message =>
{
var assemblyFinished = Assert.IsAssignableFrom<ITestAssemblyFinished>(message);
Assert.Equal(1.234M, assemblyFinished.ExecutionTime);
Assert.Equal(1, assemblyFinished.TestsFailed);
Assert.Equal(3, assemblyFinished.TestsRun);
Assert.Equal(1, assemblyFinished.TestsSkipped);
}
);
}