async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { var assemblyFileName = Guard.ArgumentNotNull("assembly.AssemblyFilename", assembly.AssemblyFilename); // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using var _ = AssemblyHelper.SubscribeResolveForAssembly(assemblyFileName, internalDiagnosticsMessageSink); await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink); var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters); controller.FindAndRun(resultsSink, settings); resultsSink.Finished.WaitOne(); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = assembly.AssemblyDisplayName; var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var assemblyInfo = new ReflectionAssemblyInfo(testAssembly); await using var disposalTracker = new DisposalTracker(); var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo); disposalTracker.Add(testFramework); var discoverySink = new TestDiscoverySink(() => cancel); // Discover & filter the tests var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo); var discoveryStarting = new TestAssemblyDiscoveryStarting { AppDomain = AppDomainOption.NotAvailable, Assembly = assembly, DiscoveryOptions = discoveryOptions, ShadowCopy = false }; reporterMessageHandler.OnMessage(discoveryStarting); testDiscoverer.Find(discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; var discoveryFinished = new TestAssemblyDiscoveryFinished { Assembly = assembly, DiscoveryOptions = discoveryOptions, TestCasesDiscovered = testCasesDiscovered, TestCasesToRun = testCasesToRun }; reporterMessageHandler.OnMessage(discoveryFinished); // Run the filtered tests if (testCasesToRun == 0) { testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary()); } else { var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var executor = testFramework.GetExecutor(assemblyInfo); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }