protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (FailSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename); var entryAssembly = Assembly.GetEntryAssembly(); var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly); var discoverySink = new TestDiscoverySink(() => cancel); using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions)); testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); } var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink)); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using var executor = testFramework.GetExecutor(entryAssembly.GetName()); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); executionSummary = resultsSink.ExecutionSummary; reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList() }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); // Force unique names if there is more than 1 testcase with the same name foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1)) { foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames) { discoveredTestCaseWithDuplicateName.ForceUniqueName(); } } foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, RunSettings runSettings, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (runSettings.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { try { var serializations = runInfo.TestCases .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)) .ToList(); if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}", serializations.Count, Environment.NewLine, string.Join(Environment.NewLine, serializations.Select(x => $" {x}"))); } var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations); if (deserializedTestCasesByUniqueId == null) { logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize"); } else { for (int idx = 0; idx < runInfo.TestCases.Count; ++idx) { try { var kvp = deserializedTestCasesByUniqueId[idx]; var vsTestCase = runInfo.TestCases[idx]; if (kvp.Value == null) { logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key); } else { testCasesMap.Add(kvp.Key, vsTestCase); testCases.Add(kvp.Value); } } catch (Exception ex) { logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex); } } } } catch (Exception ex) { logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex); } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (runSettings.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.IfAvailable : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } if (stopOnFail.HasValue) { executionOptions.SetStopOnTestFail(stopOnFail); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) { ExitCode = 1; if (stopOnFail == true) { Log.LogMessage(MessageImportance.High, "Canceling due to test failure..."); Cancel(); } } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor, ConsoleColor.DarkGray); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; #if NETCOREAPP1_0 || NETCOREAPP2_0 using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename, internalDiagnosticsMessageSink)) #endif using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (failSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = assembly.AssemblyDisplayName; var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var assemblyInfo = new ReflectionAssemblyInfo(testAssembly); await using var disposalTracker = new DisposalTracker(); var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo); disposalTracker.Add(testFramework); var discoverySink = new TestDiscoverySink(() => cancel); // Discover & filter the tests var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo); var discoveryStarting = new TestAssemblyDiscoveryStarting { AppDomain = AppDomainOption.NotAvailable, Assembly = assembly, DiscoveryOptions = discoveryOptions, ShadowCopy = false }; reporterMessageHandler.OnMessage(discoveryStarting); testDiscoverer.Find(discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; var discoveryFinished = new TestAssemblyDiscoveryFinished { Assembly = assembly, DiscoveryOptions = discoveryOptions, TestCasesDiscovered = testCasesDiscovered, TestCasesToRun = testCasesToRun }; reporterMessageHandler.OnMessage(discoveryFinished); // Run the filtered tests if (testCasesToRun == 0) { testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary()); } else { var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var executor = testFramework.GetExecutor(assemblyInfo); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
private void ExecuteTests( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, [NotNull] XunitFilters filters, [NotNull] XunitFrontController controller, [NotNull] TestDiscoverySink discoverySink, [NotNull] ITestFrameworkExecutionOptions executionOptions, [CanBeNull] XElement assemblyElement, [NotNull] DiagnosticMessageSink diagnosticMessageSink) { var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var serialize = options.Serialize; var failSkips = options.FailSkips; var stopOnFail = options.StopOnFail; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); // Discover & filter the tests _reporterMessageHandler.OnMessage( new TestAssemblyDiscoveryStarting( assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink( _reporterMessageHandler, () => _cancel, (path, summary) => _completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0) { return; } Console.WriteLine("Canceling due to test failure..."); _cancel = true; } }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if PLATFORM_DOTNET // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) }) .Where(tc => tc.xunit != null) .ToDictionary(tc => tc.xunit, tc => tc.vs); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(xunitTestCases.Keys.ToList(), resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }