public virtual TestRunState Run(IReadOnlyList <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases == null)
{
testCases = Discover();
}
var visitor = new ResultVisitor(testListener, testCases.Count)
{
TestRunState = initialRunState
};
toDispose.Push(visitor);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
xunit.RunTests(testCases, visitor, executionOptions);
visitor.Finished.WaitOne();
return(visitor.TestRunState);
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return(TestRunState.Error);
}
}
public virtual TestRunState Run(IEnumerable <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases != null)
{
testCases = testCases.ToList();
}
var visitor = new ResultVisitor(testListener)
{
TestRunState = initialRunState
};
toDispose.Push(visitor);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
if (testCases == null)
{
xunit.RunAll(visitor, TestFrameworkOptions.ForDiscovery(configuration), executionOptions);
}
else
{
xunit.RunTests(testCases, visitor, executionOptions);
}
visitor.Finished.WaitOne();
return(visitor.TestRunState);
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return(TestRunState.Error);
}
}
public static IEnumerable <IMessageSinkMessage> Run(this Xunit2 runner, IEnumerable <ITestCase> testCases)
{
Guard.AgainstNullArgument("runner", runner);
using (var sink = new SpyMessageSink <ITestAssemblyFinished>())
{
runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution());
sink.Finished.WaitOne();
return(sink.Messages.Select(message => message));
}
}
public static IEnumerable <IMessageSinkMessage> Run(this Xunit2 runner, IEnumerable <ITestCase> testCases)
{
if (!testCases.Any())
{
return(Enumerable.Empty <IMessageSinkMessage>());
}
using (var sink = new SpyMessageSink <ITestCollectionFinished>())
{
runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution());
sink.Finished.WaitOne();
return(sink.Messages.Select(_ => _));
}
}
public static Queue <IMessageSinkMessage> Run(this Xunit2 runner, Queue <ITestCase> testCases, TestAssemblyConfiguration testAssemblyConfiguration)
{
if (!testCases.Any())
{
return(new Queue <IMessageSinkMessage>());
}
using (var sink = new SpyMessageSink <ITestCollectionFinished>())
{
runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution(testAssemblyConfiguration));
sink.Finished.Wait();
return(sink.Messages);
}
}
public List <IMessageSinkMessage> Run(Type[] types)
{
Xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), types[0].Assembly.GetLocalCodeBase(), configFileName: null, shadowCopy: true);
var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();
foreach (var type in types)
{
Xunit2.Find(type.FullName, includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
discoverySink.Finished.Reset();
}
var testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(msg => msg.TestCase).ToArray();
var runSink = new SpyMessageSink <ITestAssemblyFinished>();
Xunit2.RunTests(testCases, runSink, TestFrameworkOptions.ForExecution());
runSink.Finished.WaitOne();
return(runSink.Messages.ToList());
}
public int Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering tests for {assemblyFileName}");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static);
if (threadpoolPump != null)
{
while (!resultsSink.Finished.WaitOne(0))
{
threadpoolPump.Invoke(this, null);
}
}
else
{
resultsSink.Finished.WaitOne();
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public async Task <int> Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public static void DeserializedFactsAndTheoriesFromTheSameClassStillShareFixtures()
{
var code = @"
using System;
using System.Threading;
using Xunit;
public class TestClassFixture : IDisposable
{
public static long StaticConstructorCount = 0;
public readonly long ConstructorInstance;
public TestClassFixture() { ConstructorInstance = Interlocked.Increment(ref StaticConstructorCount); }
public void Dispose() { Assert.Equal(1, StaticConstructorCount); }
}
public class TestClass : IClassFixture<TestClassFixture>
{
readonly TestClassFixture fixture;
public TestClass(TestClassFixture fixture) { this.fixture = fixture; }
[Fact]
public void FactMethod() { Assert.Equal(1, fixture.ConstructorInstance); }
[Theory]
[InlineData(42)]
public void TheoryMethod(int x) { Assert.Equal(1, fixture.ConstructorInstance); }
}
";
using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
{
var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();
var serializedTestCases = default(List <string>);
var descriptors = default(List <TestCaseDescriptor>);
using (var xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), assembly.FileName))
{
xunit2.Find("TestClass", false, discoverySink, TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
var testCases = discoverySink.Messages
.OfType <ITestCaseDiscoveryMessage>()
.Select(x => x.TestCase)
.ToList();
serializedTestCases = testCases.Select(x => xunit2.Serialize(x)).ToList();
descriptors = xunit2.GetTestCaseDescriptors(testCases, true);
}
using (var xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), assembly.FileName))
{
var deserializations = default(List <ITestCase>);
Action <List <KeyValuePair <string, ITestCase> > > callback = r => deserializations = r.Select(x => x.Value).ToList();
new TestCaseBulkDeserializer(xunit2, xunit2, serializedTestCases, callback);
var executionSink = new SpyMessageSink <ITestAssemblyFinished>();
xunit2.RunTests(deserializations, executionSink, TestFrameworkOptions.ForExecution());
executionSink.Finished.WaitOne();
var passedTests = executionSink.Messages.OfType <ITestPassed>().ToList();
var failedTests = executionSink.Messages.OfType <ITestFailed>().ToList();
Assert.Equal(2, passedTests.Count);
Assert.Empty(failedTests);
}
}
}
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
{
WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");
Log("Starting tests...");
var filters = new XunitFilters();
foreach (var trait in excludedTraits ?? Array.Empty <string>())
{
ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
}
var configuration = new TestAssemblyConfiguration
{
ShadowCopy = false,
ParallelizeAssembly = false,
ParallelizeTestCollections = false,
MaxParallelThreads = 1,
PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(
AppDomainSupport.Denied,
new NullSourceInformationProvider(),
assemblyFileName,
configFileName: null,
shadowCopy: false,
shadowCopyFolder: null,
diagnosticMessageSink: diagnosticSink,
verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Log($"Discovering tests for {assemblyFileName}...");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Log($"Discovery finished.");
Log("");
var summarySink = new DelegatingExecutionSummarySink(
testSink,
() => false,
(completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
Console.WriteLine(resultsXml.ToString());
Log("");
Log("Test results (Base64 encoded):");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));
Log(base64, id: "results");
return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
}