public static int RunAll(Assembly assembly)
{
var logger = new ConsoleOutputVisitor();
var xassembly = new ReflectionAssemblyInfo(assembly);
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
discoveryOptions.SetSynchronousMessageReporting(true);
discoveryOptions.SetDiagnosticMessages(true);
var executionOptions = TestFrameworkOptions.ForExecution();
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDiagnosticMessages(true);
executionOptions.SetMaxParallelThreads(MaxThreadCount);
executionOptions.SetDisableParallelization(MaxThreadCount <= 1);
var controller = new XunitFrontController(
xassembly.AssemblyPath,
shadowCopy: false,
diagnosticMessageSink: logger);
controller.RunAll(logger, discoveryOptions, executionOptions);
return(0);
}
public async void CanRunTests()
{
Assert.SkipWhen(EnvironmentHelper.IsMono, "Mono does not fully support dynamic assemblies");
var assemblyInfo = new ReflectionAssemblyInfo(fixture.Assembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(assemblyInfo);
disposalTracker.Add(testFramework);
var messages = new List <_MessageSinkMessage>();
var testExecutor = testFramework.GetExecutor(assemblyInfo);
await testExecutor.RunAll(SpyMessageSink.Create(messages: messages), _TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
var assemblyStarting = Assert.Single(messages.OfType <_TestAssemblyStarting>());
Assert.Equal("DynamicAssembly, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", assemblyStarting.AssemblyName);
Assert.Null(assemblyStarting.AssemblyPath);
Assert.Null(assemblyStarting.ConfigFilePath);
Assert.Single(messages.OfType <_TestFailed>());
Assert.Single(messages.OfType <_TestPassed>());
Assert.Empty(messages.OfType <_TestSkipped>());
}
private static TestAssembly GetTestAssembly(Type testClass)
{
var assembly = testClass.GetTypeInfo().Assembly;
var reflectionAssemblyInfo = new ReflectionAssemblyInfo(assembly);
var retval = new TestAssembly(reflectionAssemblyInfo);
return(retval);
}
internal static TestMethod CreateTestMethod(Type @class, string name)
{
var assemblyInfo = new ReflectionAssemblyInfo(@class.Assembly);
var testAssembly = new TestAssembly(assemblyInfo);
var testCollection = new TestCollection(testAssembly, null, string.Empty);
var classInfo = new ReflectionTypeInfo(@class);
var testClass = new TestClass(testCollection, classInfo);
var method = testClass.Class.GetMethods(true).First(m => m.Name == name);
var testMethod = new TestMethod(testClass, method);
return(testMethod);
}
private static void TestAssembly(Assembly assembly)
{
var messageSink = new ConsoleMessageSink();
var assemblyInfo = new ReflectionAssemblyInfo(assembly);
var framework = new XunitTestFramework(messageSink);
var executor = new XunitTestFrameworkExecutor(assembly.GetName(), new NullSourceInformationProvider(), messageSink);
var executionOptions = TestFrameworkOptions.ForExecution();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
executor.RunAll(messageSink, discoveryOptions, executionOptions);
messageSink.Finished.WaitOne();
}
public async void CanDiscoverTests()
{
Assert.SkipWhen(EnvironmentHelper.IsMono, "Mono does not fully support dynamic assemblies");
var assemblyInfo = new ReflectionAssemblyInfo(fixture.Assembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(assemblyInfo);
disposalTracker.Add(testFramework);
var testCases = new List <_ITestCase>();
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
await testDiscoverer.Find(testCase => { testCases.Add(testCase); return(new(true)); }, _TestFrameworkOptions.ForDiscovery());
Assert.Collection(
testCases.OrderBy(tc => tc.TestCaseDisplayName),
testCase => Assert.Equal("UnitTests.Failing", testCase.TestCaseDisplayName),
testCase => Assert.Equal("UnitTests.Passing", testCase.TestCaseDisplayName)
);
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor);
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename);
var entryAssembly = Assembly.GetEntryAssembly();
var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly);
var discoverySink = new TestDiscoverySink(() => cancel);
using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions));
testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
}
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using var executor = testFramework.GetExecutor(entryAssembly.GetName());
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
executionSummary = resultsSink.ExecutionSummary;
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = assembly.AssemblyDisplayName;
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
disposalTracker.Add(testFramework);
var discoverySink = new TestDiscoverySink(() => cancel);
// Discover & filter the tests
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
var discoveryStarting = new TestAssemblyDiscoveryStarting
{
AppDomain = AppDomainOption.NotAvailable,
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
ShadowCopy = false
};
reporterMessageHandler.OnMessage(discoveryStarting);
testDiscoverer.Find(discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
var discoveryFinished = new TestAssemblyDiscoveryFinished
{
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
TestCasesDiscovered = testCasesDiscovered,
TestCasesToRun = testCasesToRun
};
reporterMessageHandler.OnMessage(discoveryFinished);
// Run the filtered tests
if (testCasesToRun == 0)
{
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
}
else
{
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var executor = testFramework.GetExecutor(assemblyInfo);
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
TestAssembly Deproxify(TestAssembly orig)
{
var assemblyInfo = new ReflectionAssemblyInfo(orig.Assembly.AssemblyPath);
return(new TestAssembly(assemblyInfo, orig.ConfigFileName, orig.Version));
}