public void NullStackTraceInFailedTestResultsInEmptyStackTraceXmlElement()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting(testClass: typeof(ClassUnderTest).FullName !);
var methodStarting = TestData.TestMethodStarting(testMethod: nameof(ClassUnderTest.TestMethod));
var caseStarting = TestData.TestCaseStarting();
var testStarting = TestData.TestStarting(testDisplayName: "Test Display Name");
var testFailed = TestData.TestFailed(
exceptionParentIndices: new[] { -1 },
exceptionTypes: new[] { "Exception Type" },
executionTime: 123.4567809m,
messages: new[] { "Exception Message" },
output: "test output",
stackTraces: new[] { default(string) }
);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testFailed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
var failureElement = Assert.Single(testElement.Elements("failure"));
Assert.Empty(failureElement.Elements("stack-trace").Single().Value);
}
public void EmptyOutputStringDoesNotShowUpInResultingXml()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
testCase.SourceInformation.Returns(new SourceInformation());
var test = Mocks.Test(testCase, "Test Display Name");
var testPassed = Substitute.For <ITestPassed>();
testPassed.TestCase.Returns(testCase);
testPassed.Test.Returns(test);
testPassed.ExecutionTime.Returns(123.4567809M);
testPassed.Output.Returns(string.Empty);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name").Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type").Value);
Assert.Equal("TestMethod", testElement.Attribute("method").Value);
Assert.Equal("Pass", testElement.Attribute("result").Value);
Assert.Equal(123.4567809M.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time").Value);
Assert.Null(testElement.Attribute("output"));
Assert.Null(testElement.Attribute("source-file"));
Assert.Null(testElement.Attribute("source-line"));
Assert.Empty(testElement.Elements("traits"));
Assert.Empty(testElement.Elements("failure"));
Assert.Empty(testElement.Elements("reason"));
}
public void AddsAssemblyStartingInformationToXml()
{
var assemblyStarting = new _TestAssemblyStarting
{
AssemblyPath = "assembly",
AssemblyUniqueID = "assembly-id",
ConfigFilePath = "config",
StartTime = new DateTimeOffset(2013, 7, 6, 16, 24, 32, TimeSpan.Zero),
TargetFramework = "MentalFloss,Version=v21.12",
TestEnvironment = "256-bit MentalFloss",
TestFrameworkDisplayName = "xUnit.net v14.42"
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
Assert.Equal("assembly", assemblyElement.Attribute("name") !.Value);
Assert.Equal("MentalFloss,Version=v21.12", assemblyElement.Attribute("target-framework") !.Value);
Assert.Equal("256-bit MentalFloss", assemblyElement.Attribute("environment") !.Value);
Assert.Equal("xUnit.net v14.42", assemblyElement.Attribute("test-framework") !.Value);
Assert.Equal("config", assemblyElement.Attribute("config-file") !.Value);
Assert.Equal("2013-07-06", assemblyElement.Attribute("run-date") !.Value);
Assert.Equal("16:24:32", assemblyElement.Attribute("run-time") !.Value);
}
public void TestCollectionCleanupFailure()
{
var collectionStarting = new _TestCollectionStarting
{
AssemblyUniqueID = assemblyID,
TestCollectionDisplayName = "FooBar",
TestCollectionUniqueID = collectionID
};
var collectionCleanupFailure = new _TestCollectionCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces,
TestCollectionUniqueID = collectionID
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(collectionStarting);
sink.OnMessage(collectionCleanupFailure);
AssertFailureElement(assemblyElement, "test-collection-cleanup", "FooBar");
}
public void TestAssemblyCleanupFailure()
{
var collectionStarting = new _TestAssemblyStarting
{
AssemblyUniqueID = assemblyID,
AssemblyName = "assembly-name",
AssemblyPath = "assembly-file-path",
ConfigFilePath = "config-file-path",
StartTime = DateTimeOffset.UtcNow,
TargetFramework = "target-framework",
TestEnvironment = "test-environment",
TestFrameworkDisplayName = "test-framework"
};
var collectionCleanupFailure = new _TestAssemblyCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(collectionStarting);
sink.OnMessage(collectionCleanupFailure);
AssertFailureElement(assemblyElement, "assembly-cleanup", "assembly-file-path");
}
public void AddsSkippedTestElementToXml()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting(testClass: typeof(ClassUnderTest).FullName !);
var methodStarting = TestData.TestMethodStarting(testMethod: nameof(ClassUnderTest.TestMethod));
var caseStarting = TestData.TestCaseStarting();
var testStarting = TestData.TestStarting(testDisplayName: "Test Display Name");
var testSkipped = TestData.TestSkipped(reason: "Skip Reason");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testSkipped);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name") !.Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type") !.Value);
Assert.Equal("TestMethod", testElement.Attribute("method") !.Value);
Assert.Equal("Skip", testElement.Attribute("result") !.Value);
Assert.Equal(0m.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time") !.Value);
var reasonElement = Assert.Single(testElement.Elements("reason"));
Assert.Equal("Skip Reason", reasonElement.Value);
Assert.Empty(testElement.Elements("failure"));
}
public void AddsAssemblyFinishedInformationToXml()
{
executionSummary.Total = 2112;
executionSummary.Failed = 42;
executionSummary.Skipped = 6;
executionSummary.Time = 123.4567M;
executionSummary.Errors = 1;
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
var errorMessage = TestData.ErrorMessage(
exceptionParentIndices: new[] { -1 },
exceptionTypes: new[] { "ExceptionType" },
messages: new[] { "Message" },
stackTraces: new[] { "Stack" }
);
sink.OnMessage(errorMessage);
sink.OnMessage(assemblyFinished);
Assert.Equal("2112", assemblyElement.Attribute("total") !.Value);
Assert.Equal("2064", assemblyElement.Attribute("passed") !.Value);
Assert.Equal("42", assemblyElement.Attribute("failed") !.Value);
Assert.Equal("6", assemblyElement.Attribute("skipped") !.Value);
Assert.Equal(123.457M.ToString(CultureInfo.InvariantCulture), assemblyElement.Attribute("time") !.Value);
Assert.Equal("1", assemblyElement.Attribute("errors") !.Value);
}
public void IllegalXmlDoesNotPreventXmlFromBeingSaved(string inputName, string outputName)
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
var test = Mocks.Test(testCase, inputName);
var testSkipped = Substitute.For <ITestSkipped>();
testSkipped.TestCase.Returns(testCase);
testSkipped.Test.Returns(test);
testSkipped.Reason.Returns("Bad\0\r\nString");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testSkipped);
sink.OnMessage(assemblyFinished);
using (var writer = new StringWriter())
{
assemblyElement.Save(writer, SaveOptions.DisableFormatting);
var outputXml = writer.ToString();
Assert.Equal($@"<?xml version=""1.0"" encoding=""utf-16""?><assembly total=""0"" passed=""0"" failed=""0"" skipped=""0"" time=""0.000"" errors=""0""><errors /><collection><test name=""{outputName}"" type=""DelegatingXmlCreationSinkTests+ClassUnderTest"" method=""TestMethod"" time=""0"" result=""Skip"" source-file=""""><reason><![CDATA[Bad\0\r\nString]]></reason></test></collection></assembly>", outputXml);
}
}
public void AddsErrorMessagesToXml(IMessageSinkMessage errorMessage, string messageType, string name)
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(errorMessage);
sink.OnMessage(assemblyFinished);
var errorElement = Assert.Single(assemblyElement.Element("errors").Elements());
Assert.Equal(messageType, errorElement.Attribute("type").Value);
if (name == null)
{
Assert.Null(errorElement.Attribute("name"));
}
else
{
Assert.Equal(name, errorElement.Attribute("name").Value);
}
var failureElement = Assert.Single(errorElement.Elements("failure"));
Assert.Equal("ExceptionType", failureElement.Attribute("exception-type").Value);
Assert.Equal("ExceptionType : This is my message \\t\\r\\n", failureElement.Elements("message").Single().Value);
Assert.Equal("Line 1\r\nLine 2\r\nLine 3", failureElement.Elements("stack-trace").Single().Value);
}
public void TestElementTraisArePlacedInXmlWhenPresent()
{
var traits = new Dictionary <string, List <string> >
{
{ "name1", new List <string> {
"value1"
} },
{ "name2", new List <string> {
"value2"
} }
};
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var passingTestCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
passingTestCase.Traits.Returns(traits);
var testPassed = Substitute.For <ITestPassed>();
testPassed.TestCase.Returns(passingTestCase);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var traitsElements = assemblyElement.Elements("collection").Single().Elements("test").Single().Elements("traits").Single().Elements("trait");
var name1Element = Assert.Single(traitsElements, e => e.Attribute("name").Value == "name1");
Assert.Equal("value1", name1Element.Attribute("value").Value);
var name2Element = Assert.Single(traitsElements, e => e.Attribute("name").Value == "name2");
Assert.Equal("value2", name2Element.Attribute("value").Value);
}
public void TestMethodCleanupFailure()
{
var methodStarting = new _TestMethodStarting
{
AssemblyUniqueID = assemblyID,
TestClassUniqueID = classID,
TestCollectionUniqueID = collectionID,
TestMethod = "MyMethod",
TestMethodUniqueID = methodID,
};
var methodCleanupFailure = new _TestMethodCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces,
TestCollectionUniqueID = collectionID,
TestClassUniqueID = classID,
TestMethodUniqueID = methodID
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(methodStarting);
sink.OnMessage(methodCleanupFailure);
AssertFailureElement(assemblyElement, "test-method-cleanup", "MyMethod");
}
public void AddsSkippedTestElementToXml()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
var test = Mocks.Test(testCase, "Test Display Name");
var testSkipped = Substitute.For <ITestSkipped>();
testSkipped.TestCase.Returns(testCase);
testSkipped.Test.Returns(test);
testSkipped.ExecutionTime.Returns(0.0M);
testSkipped.Reason.Returns("Skip Reason");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testSkipped);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name").Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type").Value);
Assert.Equal("TestMethod", testElement.Attribute("method").Value);
Assert.Equal("Skip", testElement.Attribute("result").Value);
Assert.Equal(0.0M.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time").Value);
var reasonElement = Assert.Single(testElement.Elements("reason"));
Assert.Equal("Skip Reason", reasonElement.Value);
Assert.Empty(testElement.Elements("failure"));
}
public void TestElementSourceInfoIsPlacedInXmlWhenPresent()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting();
var methodStarting = TestData.TestMethodStarting();
var caseStarting = TestData.TestCaseStarting(sourceFilePath: "source file", sourceLineNumber: 42);
var testStarting = TestData.TestStarting();
var testPassed = TestData.TestPassed();
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("source file", testElement.Attribute("source-file") !.Value);
Assert.Equal("42", testElement.Attribute("source-line") !.Value);
}
public void AddsTestCollectionElementsToXml()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCollection = Substitute.For <ITestCollection>();
testCollection.DisplayName.Returns("Collection Name");
var testCollectionFinished = Substitute.For <ITestCollectionFinished>();
testCollectionFinished.TestCollection.Returns(testCollection);
testCollectionFinished.TestsRun.Returns(2112);
testCollectionFinished.TestsFailed.Returns(42);
testCollectionFinished.TestsSkipped.Returns(6);
testCollectionFinished.ExecutionTime.Returns(123.4567M);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testCollectionFinished);
sink.OnMessage(assemblyFinished);
var collectionElement = Assert.Single(assemblyElement.Elements("collection"));
Assert.Equal("Collection Name", collectionElement.Attribute("name").Value);
Assert.Equal("2112", collectionElement.Attribute("total").Value);
Assert.Equal("2064", collectionElement.Attribute("passed").Value);
Assert.Equal("42", collectionElement.Attribute("failed").Value);
Assert.Equal("6", collectionElement.Attribute("skipped").Value);
Assert.Equal(123.457M.ToString(CultureInfo.InvariantCulture), collectionElement.Attribute("time").Value);
}
private async Task <XElement> Run(Assembly assembly, string assemblyPath)
{
using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false))
{
using (var discoverySink = new TestDiscoverySink())
{
var configuration = GetConfiguration(assembly) ?? new TestAssemblyConfiguration()
{
PreEnumerateTheories = false
};
ITestFrameworkDiscoveryOptions discoveryOptions = GetFrameworkOptionsForDiscovery(configuration);
discoveryOptions.SetSynchronousMessageReporting(true);
Logger.OnDebug($"Starting test discovery in the '{assembly}' assembly");
frontController.Find(false, discoverySink, discoveryOptions);
Logger.OnDebug($"Test discovery in assembly '{assembly}' completed");
discoverySink.Finished.WaitOne();
if (discoverySink.TestCases == null || discoverySink.TestCases.Count == 0)
{
Logger.Info("No test cases discovered");
return(null);
}
TotalTests += discoverySink.TestCases.Count;
List <ITestCase> testCases;
if (_filters != null && _filters.TestCaseFilters.Any())
{
Action <string> log = LogExcludedTests ? (s) => do_log(s) : (Action <string>)null;
testCases = discoverySink.TestCases.Where(
tc => !_filters.IsExcluded(tc, log)).ToList();
FilteredTests += discoverySink.TestCases.Count - testCases.Count;
}
else
{
testCases = discoverySink.TestCases;
}
var assemblyElement = new XElement("assembly");
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(_messageSink, null, null);
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
ITestFrameworkExecutionOptions executionOptions = GetFrameworkOptionsForExecution(configuration);
executionOptions.SetDisableParallelization(!RunInParallel);
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetMaxParallelThreads(MaxParallelThreads);
// set the wait for event cb first, then execute the tests
var resultTask = WaitForEvent(resultsSink.Finished, TimeSpan.FromDays(10)).ConfigureAwait(false);
frontController.RunTests(testCases, resultsSink, executionOptions);
await resultTask;
return(assemblyElement);
}
}
}
XElement Run(Assembly assembly, string assemblyPath)
{
using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false))
{
using (var discoverySink = new TestDiscoverySink())
{
var configuration = GetConfiguration(assembly) ?? new TestAssemblyConfiguration();
ITestFrameworkDiscoveryOptions discoveryOptions = GetFrameworkOptionsForDiscovery(configuration);
discoveryOptions.SetSynchronousMessageReporting(true);
Logger.OnInfo(LogTag, $"Starting test discovery in the '{assembly}' assembly");
frontController.Find(false, discoverySink, discoveryOptions);
Logger.OnInfo(LogTag, $"Test discovery in assembly '{assembly}' completed");
discoverySink.Finished.WaitOne();
if (discoverySink.TestCases == null || discoverySink.TestCases.Count == 0)
{
Logger.OnInfo(LogTag, "No test cases discovered");
return(null);
}
TotalTests += discoverySink.TestCases.Count;
List <ITestCase> testCases;
if (filters != null && filters.Count > 0)
{
testCases = discoverySink.TestCases.Where(tc => IsIncluded(tc)).ToList();
FilteredTests += discoverySink.TestCases.Count - testCases.Count;
}
else
{
testCases = discoverySink.TestCases;
}
var assemblyElement = new XElement("assembly");
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(messageSink, null, null);
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
ITestFrameworkExecutionOptions executionOptions = GetFrameworkOptionsForExecution(configuration);
executionOptions.SetDisableParallelization(!RunInParallel);
executionOptions.SetSynchronousMessageReporting(true);
try
{
frontController.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
}
finally
{
resultsSink.Dispose();
}
return(assemblyElement);
}
}
}
public void AssemblyStartingDoesNotIncludeNullConfigFile()
{
var assemblyStarting = Substitute.For <ITestAssemblyStarting>();
assemblyStarting.TestAssembly.ConfigFileName.Returns((string)null);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
Assert.Null(assemblyElement.Attribute("config-file"));
}
public static int Main(string[] args)
{
var asm = typeof(SingleFileTestRunner).Assembly;
Console.WriteLine("Running assembly:" + asm.FullName);
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testsFinished = new TaskCompletionSource();
var testSink = new TestMessageSink();
var summarySink = new DelegatingExecutionSummarySink(testSink,
() => false,
(completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyFinishedEvent += args =>
{
Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
testsFinished.SetResult();
};
var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
var asmInfo = Reflector.Wrap(asm);
var asmName = asm.GetName();
var discoverySink = new TestDiscoverySink();
var discoverer = xunitTestFx.CreateDiscoverer(asmInfo);
discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
XunitFilters filters = new XunitFilters();
filters.ExcludedTraits.Add("category", new List <string> {
"failing"
});
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var executor = xunitTestFx.CreateExecutor(asmName);
executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution());
resultsSink.Finished.WaitOne();
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public void ErrorMessage()
{
var errorMessage = new _ErrorMessage
{
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(errorMessage);
AssertFailureElement(assemblyElement, "fatal", null);
}
public void AddsFailingTestElementToXml()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting(testClass: typeof(ClassUnderTest).FullName !);
var methodStarting = TestData.TestMethodStarting(testMethod: nameof(ClassUnderTest.TestMethod));
var caseStarting = TestData.TestCaseStarting();
var testStarting = TestData.TestStarting(testDisplayName: "Test Display Name");
var testFailed = TestData.TestFailed(
exceptionParentIndices: new[] { -1 },
exceptionTypes: new[] { "Exception Type" },
executionTime: 123.4567809m,
messages: new[] { "Exception Message" },
output: "test output",
stackTraces: new[] { "Exception Stack Trace" }
);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testFailed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name") !.Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type") !.Value);
Assert.Equal("TestMethod", testElement.Attribute("method") !.Value);
Assert.Equal("Fail", testElement.Attribute("result") !.Value);
Assert.Equal(123.4567809M.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time") !.Value);
Assert.Equal("test output", testElement.Element("output") !.Value);
var failureElement = Assert.Single(testElement.Elements("failure"));
Assert.Equal("Exception Type", failureElement.Attribute("exception-type") !.Value);
Assert.Equal("Exception Type : Exception Message", failureElement.Elements("message").Single().Value);
Assert.Equal("Exception Stack Trace", failureElement.Elements("stack-trace").Single().Value);
Assert.Empty(testElement.Elements("reason"));
}
public void IllegalXmlAcceptanceTest(
string inputName,
string outputName)
{
var classStarting = TestData.TestClassStarting(testClass: typeof(ClassUnderTest).FullName !);
var methodStarting = TestData.TestMethodStarting(testMethod: nameof(ClassUnderTest.TestMethod));
var testStarting = TestData.TestStarting(testDisplayName: inputName);
var testSkipped = TestData.TestSkipped(reason: "Bad\0\r\nString");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(TestData.TestAssemblyStarting());
sink.OnMessage(TestData.TestCollectionStarting());
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(TestData.TestCaseStarting());
sink.OnMessage(testStarting);
sink.OnMessage(testSkipped);
sink.OnMessage(TestData.TestFinished());
sink.OnMessage(TestData.TestCaseFinished());
sink.OnMessage(TestData.TestMethodFinished());
sink.OnMessage(TestData.TestClassFinished());
sink.OnMessage(TestData.TestCollectionFinished());
sink.OnMessage(TestData.TestAssemblyFinished());
using var writer = new StringWriter();
assemblyElement.Save(writer, SaveOptions.DisableFormatting);
var outputXml = writer.ToString();
Assert.Equal(
@"<?xml version=""1.0"" encoding=""utf-16""?>" +
@"<assembly name=""./test-assembly.dll"" environment=""test-environment"" test-framework=""test-framework"" run-date=""2021-01-20"" run-time=""17:00:00"" config-file=""./test-assembly.json"" target-framework="".NETMagic,Version=v98.76.54"" total=""0"" passed=""0"" failed=""0"" skipped=""0"" time=""0.000"" errors=""0"">" +
@"<errors />" +
@"<collection name=""test-collection-display-name"" total=""2112"" passed=""2064"" failed=""42"" skipped=""6"" time=""123.457"">" +
$@"<test name=""{outputName}"" type=""DelegatingXmlCreationSinkTests+ClassUnderTest"" method=""TestMethod"" time=""0"" result=""Skip"">" +
@"<reason><![CDATA[Bad\0\r\nString]]></reason>" +
@"</test>" +
@"</collection>" +
@"</assembly>",
outputXml
);
}
public void AssemblyStartingDoesNotIncludeNullValues()
{
var assemblyStarting = new _TestAssemblyStarting
{
AssemblyPath = "assembly",
AssemblyUniqueID = "assembly-id",
StartTime = new DateTimeOffset(2013, 7, 6, 16, 24, 32, TimeSpan.Zero),
TestEnvironment = "256-bit MentalFloss",
TestFrameworkDisplayName = "xUnit.net v14.42"
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
Assert.Null(assemblyElement.Attribute("config-file"));
Assert.Null(assemblyElement.Attribute("target-framework"));
}
public void TestElementTraitsArePlacedInXmlWhenPresent()
{
var traits = new Dictionary <string, List <string> >
{
{ "name1", new List <string> {
"value1"
} },
{ "name2", new List <string> {
"value2"
} }
};
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting();
var methodStarting = TestData.TestMethodStarting();
var caseStarting = TestData.TestCaseStarting(traits: traits);
var testStarting = TestData.TestStarting();
var testPassed = TestData.TestPassed();
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var traitsElements = assemblyElement.Elements("collection").Single().Elements("test").Single().Elements("traits").Single().Elements("trait");
var name1Element = Assert.Single(traitsElements, e => e.Attribute("name")?.Value == "name1");
Assert.Equal("value1", name1Element.Attribute("value") !.Value);
var name2Element = Assert.Single(traitsElements, e => e.Attribute("name")?.Value == "name2");
Assert.Equal("value2", name2Element.Attribute("value") !.Value);
}
public void AddsAssemblyStartingInformationToXml()
{
var assemblyStarting = Substitute.For <ITestAssemblyStarting>();
assemblyStarting.TestAssembly.Assembly.AssemblyPath.Returns("assembly");
assemblyStarting.TestAssembly.ConfigFileName.Returns("config");
assemblyStarting.StartTime.Returns(new DateTime(2013, 7, 6, 16, 24, 32));
assemblyStarting.TestEnvironment.Returns("256-bit MentalFloss");
assemblyStarting.TestFrameworkDisplayName.Returns("xUnit.net v14.42");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
Assert.Equal("assembly", assemblyElement.Attribute("name").Value);
Assert.Equal("256-bit MentalFloss", assemblyElement.Attribute("environment").Value);
Assert.Equal("xUnit.net v14.42", assemblyElement.Attribute("test-framework").Value);
Assert.Equal("config", assemblyElement.Attribute("config-file").Value);
Assert.Equal("2013-07-06", assemblyElement.Attribute("run-date").Value);
Assert.Equal("16:24:32", assemblyElement.Attribute("run-time").Value);
}
public void AddsTestCollectionElementsToXml()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var testCollectionStarted = TestData.TestCollectionStarting(testCollectionDisplayName: "Collection Name", testCollectionUniqueID: "abc123");
var testCollectionFinished = TestData.TestCollectionFinished(testsRun: 2112, testsFailed: 42, testsSkipped: 6, executionTime: 123.4567m, testCollectionUniqueID: "abc123");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testCollectionStarted);
sink.OnMessage(testCollectionFinished);
sink.OnMessage(assemblyFinished);
var collectionElement = Assert.Single(assemblyElement.Elements("collection"));
Assert.Equal("Collection Name", collectionElement.Attribute("name") !.Value);
Assert.Equal("2112", collectionElement.Attribute("total") !.Value);
Assert.Equal("2064", collectionElement.Attribute("passed") !.Value);
Assert.Equal("42", collectionElement.Attribute("failed") !.Value);
Assert.Equal("6", collectionElement.Attribute("skipped") !.Value);
Assert.Equal(123.457M.ToString(CultureInfo.InvariantCulture), collectionElement.Attribute("time") !.Value);
}
public void NullStackTraceInFailedTestResultsInEmptyStackTraceXmlElement()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
var testFailed = Substitute.For <ITestFailed>();
testFailed.TestCase.Returns(testCase);
testFailed.ExceptionTypes.Returns(new[] { "ExceptionType" });
testFailed.Messages.Returns(new[] { "Exception Message" });
testFailed.StackTraces.Returns(new[] { (string)null });
testFailed.ExceptionParentIndices.Returns(new[] { -1 });
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testFailed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
var failureElement = Assert.Single(testElement.Elements("failure"));
Assert.Empty(failureElement.Elements("stack-trace").Single().Value);
}
public void TestElementSourceInfoIsPlacedInXmlWhenPresent()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
testCase.SourceInformation.Returns(new SourceInformation {
FileName = "source file", LineNumber = 42
});
var testPassed = Substitute.For <ITestPassed>();
testPassed.TestCase.Returns(testCase);
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("source file", testElement.Attribute("source-file").Value);
Assert.Equal("42", testElement.Attribute("source-line").Value);
}
public void EmptyOutputStringDoesNotShowUpInResultingXml()
{
var assemblyFinished = TestData.TestAssemblyFinished();
var assemblyStarting = TestData.TestAssemblyStarting();
var collectionStarting = TestData.TestCollectionStarting();
var classStarting = TestData.TestClassStarting(testClass: typeof(ClassUnderTest).FullName !);
var methodStarting = TestData.TestMethodStarting(testMethod: nameof(ClassUnderTest.TestMethod));
var caseStarting = TestData.TestCaseStarting();
var testStarting = TestData.TestStarting(testDisplayName: "Test Display Name");
var testPassed = TestData.TestPassed(executionTime: 123.4567809m, output: "");
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(assemblyStarting);
sink.OnMessage(collectionStarting);
sink.OnMessage(classStarting);
sink.OnMessage(methodStarting);
sink.OnMessage(caseStarting);
sink.OnMessage(testStarting);
sink.OnMessage(testPassed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name") !.Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type") !.Value);
Assert.Equal("TestMethod", testElement.Attribute("method") !.Value);
Assert.Equal("Pass", testElement.Attribute("result") !.Value);
Assert.Equal(123.4567809M.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time") !.Value);
Assert.Null(testElement.Attribute("output"));
Assert.Null(testElement.Attribute("source-file"));
Assert.Null(testElement.Attribute("source-line"));
Assert.Empty(testElement.Elements("traits"));
Assert.Empty(testElement.Elements("failure"));
Assert.Empty(testElement.Elements("reason"));
}
public void AddsFailingTestElementToXml()
{
var assemblyFinished = Substitute.For <ITestAssemblyFinished>();
var testCase = Mocks.TestCase <ClassUnderTest>("TestMethod");
var test = Mocks.Test(testCase, "Test Display Name");
var testFailed = Substitute.For <ITestFailed>();
testFailed.TestCase.Returns(testCase);
testFailed.Test.Returns(test);
testFailed.ExecutionTime.Returns(123.4567809M);
testFailed.Output.Returns("test output");
testFailed.ExceptionTypes.Returns(new[] { "Exception Type" });
testFailed.Messages.Returns(new[] { "Exception Message" });
testFailed.StackTraces.Returns(new[] { "Exception Stack Trace" });
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(testFailed);
sink.OnMessage(assemblyFinished);
var testElement = Assert.Single(assemblyElement.Elements("collection").Single().Elements("test"));
Assert.Equal("Test Display Name", testElement.Attribute("name").Value);
Assert.Equal("DelegatingXmlCreationSinkTests+ClassUnderTest", testElement.Attribute("type").Value);
Assert.Equal("TestMethod", testElement.Attribute("method").Value);
Assert.Equal("Fail", testElement.Attribute("result").Value);
Assert.Equal(123.4567809M.ToString(CultureInfo.InvariantCulture), testElement.Attribute("time").Value);
Assert.Equal("test output", testElement.Element("output").Value);
var failureElement = Assert.Single(testElement.Elements("failure"));
Assert.Equal("Exception Type", failureElement.Attribute("exception-type").Value);
Assert.Equal("Exception Type : Exception Message", failureElement.Elements("message").Single().Value);
Assert.Equal("Exception Stack Trace", failureElement.Elements("stack-trace").Single().Value);
Assert.Empty(testElement.Elements("reason"));
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
return null;
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
if (longRunningSeconds > 0)
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
if (FailSkips)
resultsSink = new DelegatingFailSkipSink(resultsSink);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
if (longRunningSeconds > 0)
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
if (failSkips)
resultsSink = new DelegatingFailSkipSink(resultsSink);
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return assemblyElement;
}
