static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) resultsVisitor = new FailSkipVisitor(resultsVisitor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
public async void RunTests(string arguments, StringBuilder sbLog, Action <string> uiLogger) { log = sbLog; var reporters = new List <IRunnerReporter>(); string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); log.AppendLine($"Args: {arguments}"); var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger(uiLogger)); var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; //log.AppendLine("testCasesToRun: " + testCasesToRun); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); //log.AppendLine("Finished running tests"); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); log.AppendLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); } finally { await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement); await WriteLogs(Path.GetFileName(assembly.AssemblyFilename), log.ToString()); } } Application.Current.Exit(); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (FailSkips) resultsVisitor = new FailSkipVisitor(resultsVisitor); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); if (resultsVisitor.ExecutionSummary.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); if (resultsVisitor.ExecutionSummary.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
public FailSkipVisitorTests() { innerVisitor = Substitute.For<IExecutionVisitor>(); visitor = new FailSkipVisitor(innerVisitor); }
private async void RunTests(string arguments) { var reporters = new List<IRunnerReporter>(); string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries); log = string.Empty; log += "Args: " + args + "\n"; var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger()); var completionMessages = new ConcurrentDictionary<string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; log += "testCasesToRun: " + testCasesToRun + "\n"; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); log += "finished running tests \n"; resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); log += "logged exec errors: " + e + "\n"; } } await WriteResults(assembliesElement); await WriteLogs(log); Application.Current.Exit(); }
public FailSkipVisitorTests() { innerVisitor = Substitute.For <IExecutionVisitor>(); visitor = new FailSkipVisitor(innerVisitor); }