protected static void Go(string assemblyFileName, Stream stream, AppDomainSupport appDomainSupport, Action<XunitFrontController, TestAssemblyConfiguration, ClientWriter> action) { using (AssemblyHelper.SubscribeResolve()) using (var xunit = new XunitFrontController(appDomainSupport, assemblyFileName, shadowCopy: false)) using (var writer = new ClientWriter(stream)) { var configuration = ConfigReader.Load(assemblyFileName); action(xunit, configuration, writer); } }
private async void RunTests(string arguments) { var reporters = new List <IRunnerReporter>(); string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries); log = string.Empty; log += "Args: " + string.Join(" ", args) + "\n"; var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger()); var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; log += "testCasesToRun: " + testCasesToRun + "\n"; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); log += "finished running tests \n"; resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); log += "logged exec errors: " + e + "\n"; } } await WriteResults(assembliesElement); await WriteLogs(log); Application.Current.Exit(); }
void DiscoverTests <TVisitor>(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Func <string, ITestFrameworkDiscoverer, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, TVisitor> visitComplete = null, Stopwatch stopwatch = null) where TVisitor : IVsDiscoveryVisitor { if (stopwatch == null) { stopwatch = Stopwatch.StartNew(); } try { RemotingUtility.CleanUpRegisteredChannels(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed)); } using (AssemblyHelper.SubscribeResolve()) { foreach (string assemblyFileName in sources) { var fileName = Path.GetFileName(assemblyFileName); try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1} (no reference to xUnit.net)", stopwatch.Elapsed, fileName)); } } else { using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true)) using (var visitor = visitorFactory(assemblyFileName, framework)) { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1} (unsupported target framework '{2}')", stopwatch.Elapsed, fileName, targetFramework)); } } else { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName)); } framework.Find(includeSourceInformation: true, messageSink: visitor, options: new TestFrameworkOptions()); var totalTests = visitor.Finish(); if (visitComplete != null) { visitComplete(assemblyFileName, framework, visitor); } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, fileName, totalTests)); } } } } } catch (Exception e) { var ex = e.Unwrap(); var fileNotFound = ex as FileNotFoundException; var fileLoad = ex as FileLoadException; if (fileNotFound != null) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1} (could not find dependent assembly '{2}')", stopwatch.Elapsed, fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName))); } else if (fileLoad != null) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1} (could not find dependent assembly '{2}')", stopwatch.Elapsed, fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName))); } else { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, fileName, ex)); } } } } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests: {1}", stopwatch.Elapsed, e.Unwrap())); } stopwatch.Stop(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed)); } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("logger", logger); Guard.ArgumentNotNull("discoverySink", discoverySink); var settings = SettingsProvider.Load(); var stopwatch = Stopwatch.StartNew(); try { RemotingUtility.CleanUpRegisteredChannels(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed)); } using (AssemblyHelper.SubscribeResolve()) { foreach (string assemblyFileName in sources) { var fileName = Path.GetFileName(assemblyFileName); try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName)); } } else { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName)); } using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true)) using (var sink = new VsDiscoveryVisitor(assemblyFileName, framework, logger, discoveryContext, discoverySink, () => cancelled)) { framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions()); sink.Finished.WaitOne(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, fileName, sink.TotalTests)); } } } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, fileName, e)); } } } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests: {1}", stopwatch.Elapsed, e)); } stopwatch.Stop(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed)); } }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, RunSettings runSettings, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (runSettings.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { try { var serializations = runInfo.TestCases .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)) .ToList(); if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}", serializations.Count, Environment.NewLine, string.Join(Environment.NewLine, serializations.Select(x => $" {x}"))); } var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations); if (deserializedTestCasesByUniqueId == null) { logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize"); } else { for (int idx = 0; idx < runInfo.TestCases.Count; ++idx) { try { var kvp = deserializedTestCasesByUniqueId[idx]; var vsTestCase = runInfo.TestCases[idx]; if (kvp.Value == null) { logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key); } else { testCasesMap.Add(kvp.Key, vsTestCase); testCases.Add(kvp.Value); } } catch (Exception ex) { logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex); } } } } catch (Exception ex) { logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex); } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (runSettings.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool? parallelizeTestCollections, int? maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages ?? false) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.GetMethodDisplay(), !executionOptions.GetDisableParallelization(), executionOptions.GetMaxParallelThreads()); else Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) { Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
private static List<ITestCase> GetTestCaseList(XunitFrontController xunit, TestAssemblyConfiguration configuration, HashSet<string> testCaseNameSet) { var testCaseList = new List<ITestCase>(); using (var sink = new TestDiscoverySink(testCaseNameSet, testCaseList)) { xunit.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration)); sink.Finished.WaitOne(); } return testCaseList; }
static int Main(string[] args) { var nodeIndex = CommandLine.GetInt32("multinode.index"); var nodeRole = CommandLine.GetProperty("multinode.role"); var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly"); var typeName = CommandLine.GetProperty("multinode.test-class"); var testName = CommandLine.GetProperty("multinode.test-method"); var displayName = testName; var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address")); var listenPort = CommandLine.GetInt32("multinode.listen-port"); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex); var tcpClient = _logger = system.ActorOf <RunnerTcpClient>(); system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient); #if CORECLR // In NetCore, if the assembly file hasn't been touched, // XunitFrontController would fail loading external assemblies and its dependencies. AssemblyLoadContext.Default.Resolving += (assemblyLoadContext, assemblyName) => DefaultOnResolving(assemblyLoadContext, assemblyName, assemblyFileName); var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyFileName); DependencyContext.Load(assembly) .CompileLibraries .Where(dep => dep.Name.ToLower() .Contains(assembly.FullName.Split(new[] { ',' })[0].ToLower())) .Select(dependency => AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(dependency.Name))); #endif Thread.Sleep(TimeSpan.FromSeconds(10)); using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName)) { /* need to pass in just the assembly name to Discovery, not the full path * i.e. "Akka.Cluster.Tests.MultiNode.dll" * not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause * the Discovery class to actually not find any individual specs to run */ var assemblyName = Path.GetFileName(assemblyFileName); Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName); using (var discovery = new Discovery(assemblyName, typeName)) { using (var sink = new Sink(nodeIndex, nodeRole, tcpClient)) { try { controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution()); } catch (AggregateException ex) { var specFail = new SpecFail(nodeIndex, nodeRole, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); foreach (var innerEx in ex.Flatten().InnerExceptions) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); } _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return(1); } catch (Exception ex) { var specFail = new SpecFail(nodeIndex, nodeRole, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); var innerEx = ex.InnerException; while (innerEx != null) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); innerEx = innerEx.InnerException; } _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return(1); } var timedOut = false; if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out { var line = string.Format("Timed out while waiting for test to complete after {0} ms", MaxProcessWaitTimeout); _logger.Tell(line); Console.WriteLine(line); timedOut = true; } FlushLogMessages(); system.Terminate().Wait(); Environment.Exit(sink.Passed && !timedOut ? 0 : 1); return(sink.Passed ? 0 : 1); } } } }
void RunTestsInAssembly(List<IDisposable> toDispose, string assemblyFileName, IEnumerable<TestCaseViewModel> testCases, Stopwatch stopwatch) { if (cancelled) return; var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true); lock (toDispose) toDispose.Add(controller); var xunitTestCases = testCases.ToDictionary(tc => tc.TestCase); using (var executionVisitor = new TestExecutionVisitor(xunitTestCases, this, () => cancelled, context)) { var executionOptions = TestFrameworkOptions.ForExecution(); controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions); executionVisitor.Finished.WaitOne(); } }
public static int Main(string[] args) { string host = null; int port = 0; // First argument is the connection string if (args [0].StartsWith("tcp:")) { var parts = args [0].Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); args = args.Skip(1).ToArray(); } // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = args[0]; var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(discoverySink.TestCases, resultsSink, testOptions); resultsSink.Finished.WaitOne(); writer.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); #else MonoSdksTextUI runner; TcpWriter writer = null; string resultsXml = null; if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); resultsXml = Path.GetTempFileName(); args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray(); writer = new TcpWriter(host, port); runner = new MonoSdksTextUI(writer); } else { runner = new MonoSdksTextUI(); } runner.Execute(args); if (resultsXml != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } return(runner.Failure ? 1 : 0); #endif }
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch) { var sourceSinks = new List <SourceSink <TestDiscoveryVisitor> >(); var result = new List <IGrouping <string, TestCase> >(); try { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed)); } foreach (string assemblyFileName in sources) { try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } } else { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true); var sink = new TestDiscoveryVisitor(); sourceSinks.Add(new SourceSink <TestDiscoveryVisitor> { Framework = framework, Sink = sink, AssemblyFileName = assemblyFileName }); framework.Find(includeSourceInformation: true, messageSink: sink); } } catch (Exception e) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e)); } } var toFinish = new List <SourceSink <TestDiscoveryVisitor> >(sourceSinks); while (toFinish.Count > 0) { var finishedIdx = WaitHandle.WaitAny(sourceSinks.Select(sink => sink.Sink.Finished).ToArray()); var sourceSink = toFinish[finishedIdx]; result.Add( new Grouping <string, TestCase>( sourceSink.AssemblyFileName, sourceSink.Sink.TestCases .GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name)) .SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(sourceSink.AssemblyFileName, sourceSink.Framework, testCase, settings, forceUniqueNames: group.Count() > 1))) .ToList() ) ); if (settings.MessageDisplay != MessageDisplay.None) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(sourceSink.AssemblyFileName), sourceSink.Sink.TestCases.Count)); } toFinish.RemoveAt(finishedIdx); } } finally { foreach (var sourceSink in sourceSinks) { sourceSink.Sink.Dispose(); sourceSink.Framework.Dispose(); } } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed)); } return(result); }
public static void RunTests(Options o) { var nullMessage = new Xunit.NullMessageSink(); var discoveryOptions = TestFrameworkOptions.ForDiscovery(); using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false)) { var tv = new TestDiscoverySink(); var excludeTestCaseSet = new TestDiscoverySink(); c.Find(true, tv, discoveryOptions); tv.Finished.WaitOne(); foreach (var tc in tv.TestCases) { var method = tc.TestMethod.Method; var attributes = method.GetCustomAttributes(typeof(FactAttribute)); foreach (ReflectionAttributeInfo at in attributes) { var result = at.GetNamedArgument<string>("Skip"); if (result != null) { Console.WriteLine("SKIPPY! {0} because {1}", method, result); } if (o.TestType != TestType.All) { if (!at.ToString().EndsWith(o.TestType.ToString())) { excludeTestCaseSet.TestCases.Add(tc); } } } } foreach (var tc in excludeTestCaseSet.TestCases) { tv.TestCases.Remove(tc); } Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count); //core execution Sink int testCaseCount = tv.TestCases.Count; Stream file = new FileStream(".\\result.xml", FileMode.Create); int totalResult = 0; int totalErrors = 0; int totalFailed = 0; int totalSkipped = 0; for (int i = 0; i < testCaseCount; i++) { IExecutionSink resultsSink; ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>(); IMessageSinkWithTypes reporterMessageHandler; var reporters = GetAvailableRunnerReporters(); var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll"); IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger)); var xmlElement = new XElement("TestResult"); resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true); var message = new Xunit.NullMessageSink(); var executionOptions = TestFrameworkOptions.ForExecution(); c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions); resultsSink.Finished.WaitOne(o.TimeOut); tv.TestCases.RemoveAt(0); totalResult++; totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors; totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed; totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped; xmlElement.Save(file); file.Flush(); } file.Dispose(); Console.WriteLine("Total tests: " + totalResult); Console.ForegroundColor = ConsoleColor.DarkRed; Console.WriteLine("Error tests: " + totalErrors); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Failed tests: " + totalFailed); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Skipped tests: " + totalSkipped); Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped)); Console.ResetColor(); } }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, AppDomainSupport?appDomains, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; #if !WINDOWS_UWP using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink)) #endif using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink)); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList() }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); // Force unique names if there is more than 1 testcase with the same name foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1)) { foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames) { discoveredTestCaseWithDuplicateName.ForceUniqueName(); } } foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; if (diagnosticMessages) assembly.Configuration.DiagnosticMessages = true; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); if (!quiet) lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); else Console.WriteLine("Discovering: {0}", assemblyDisplayName); } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return assemblyElement; }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (_cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories when we're not running in Visual Studio if (!designTime) { assembly.Configuration.PreEnumerateTheories = false; } if (diagnosticMessages) { assembly.Configuration.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly); using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { var includeSourceInformation = designTime && listTestCases; // Discover & filter the tests _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestCases = null; if (designTime) { vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } if (listTestCases) { lock (consoleLock) { if (designTime) { foreach (var testcase in vsTestCases.Values) { _testDiscoverySink?.SendTestFound(testcase); Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } IExecutionVisitor resultsVisitor; if (designTime) { resultsVisitor = new DesignTimeExecutionVisitor(_testExecutionSink, vsTestCases, _reporterMessageHandler); } else { resultsVisitor = new XmlAggregateVisitor(_reporterMessageHandler, _completionMessages, assemblyElement, () => _cancel); } IList <ITestCase> filteredTestCases; var testCasesDiscovered = discoveryVisitor.TestCases.Count; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList(); } var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); if (filteredTestCases.Count == 0) { _completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { _failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent); controller.Run(discoveryVisitor.TestCases, resultsVisitor, new TestFrameworkOptions()); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Console.WriteLine(stackLine); e = e.InnerException; } } return assemblyElement; }
/// <summary> /// MultiNodeTestRunner takes the following <see cref="args"/>: /// /// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] [-Dmultinode.spec={spec name}] /// /// <list type="number"> /// <listheader> /// <term>Argument</term> /// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description> /// </listheader> /// <item> /// <term>AssemblyName</term> /// <description> /// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory. /// /// i.e. "Akka.Cluster.Tests.MultiNode.dll" /// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll" /// </description> /// </item> /// <item> /// <term>-Dmultinode.enable-filesink</term> /// <description>Having this flag set means that the contents of this test run will be saved in the /// current working directory as a .JSON file. /// </description> /// </item> /// <item> /// <term>-Dmultinode.multinode.output-directory</term> /// <description>Setting this flag means that any persistent multi-node test runner output files /// will be written to this directory instead of the default, which is the same folder /// as the test binary. /// </description> /// </item> /// <item> /// <term>-Dmultinode.listen-address={ip}</term> /// <description> /// Determines the address that this multi-node test runner will use to listen for log messages from /// individual NodeTestRunner.exe processes. /// /// Defaults to 127.0.0.1 /// </description> /// </item> /// <item> /// <term>-Dmultinode.listen-port={port}</term> /// <description> /// Determines the port number that this multi-node test runner will use to listen for log messages from /// individual NodeTestRunner.exe processes. /// /// Defaults to 6577 /// </description> /// </item> /// <item> /// <term>-Dmultinode.spec={spec name}</term> /// <description> /// Setting this flag means that only tests which contains the spec name will be executed /// otherwise all tests will be executed /// </description> /// </item> /// <item> /// <term>-Dmultinode.include={include filter}</term> /// <description> /// Setting this flag means that only tests which matches the comma separated wildcard /// filter will be executed, otherwise all tests will be executed /// </description> /// </item> /// <item> /// <term>-Dmultinode.exclude={exclude filter}</term> /// <description> /// Setting this flag means that only tests which does not match the comma separated /// wildcard filter will be executed, otherwise all tests will be executed /// </description> /// </item> /// </list> /// </summary> static void Main(string[] args) { // Force load the args CommandLine.GetPropertyOrDefault("force load", null); if (CommandLine.ShowHelp) { PrintHelp(); return; } if (CommandLine.ShowVersion) { var version = Assembly.GetExecutingAssembly().GetName().Version.ToString(); Console.WriteLine($"Version: {version}"); return; } OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty); FailedSpecsDirectory = CommandLine.GetPropertyOrDefault("multinode.failed-specs-directory", "FAILED_SPECS_LOGS"); string logLevel = CommandLine.GetPropertyOrDefault("multinode.loglevel", "WARNING"); TestRunSystem = ActorSystem.Create("TestRunnerLogging", $"akka.loglevel={logLevel}"); var suiteName = Path.GetFileNameWithoutExtension(Path.GetFullPath(args[0].Trim('"'))); var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false"); if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn)) { throw new ArgumentException("Invalid argument provided for -Dmultinode.teamcity"); } var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1")); var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var specName = CommandLine.GetPropertyOrDefault("multinode.spec", ""); var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "net"); var reporter = CommandLine.GetPropertyOrDefault("multinode.reporter", "console"); var clearOutputDirectory = CommandLine.GetInt32OrDefault("multinode.clear-output", 0); if (clearOutputDirectory > 0 && Directory.Exists(OutputDirectory)) { Directory.Delete(OutputDirectory, true); } var include = new WildcardPatterns(CommandLine.GetPropertyOrDefault("multinode.include", null), true); var exclude = new WildcardPatterns(CommandLine.GetPropertyOrDefault("multinode.exclude", null), false); Props coordinatorProps; switch (reporter.ToLowerInvariant()) { case "trx": coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TrxMessageSink(suiteName) })); break; case "teamcity": coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TeamCityMessageSink(Console.WriteLine, suiteName) })); break; case "console": coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new ConsoleMessageSink() })); break; default: throw new ArgumentException($"Given reporter name '{reporter}' is not understood, valid reporters are: trx and teamcity"); } SinkCoordinator = TestRunSystem.ActorOf(coordinatorProps, "sinkCoordinator"); #if CORECLR if (!_validNetCorePlatform.Contains(platform)) { throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore"); } #else if (platform != "net") { throw new Exception($"Target platform not supported: {platform}. Supported platforms are net"); } #endif var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger"); TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint), sender: tcpLogger); var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first EnableAllSinks(assemblyPath, platform); PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}"); #if CORECLR // In NetCore, if the assembly file hasn't been touched, // XunitFrontController would fail loading external assemblies and its dependencies. var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); var asms = assembly.GetReferencedAssemblies(); var basePath = Path.GetDirectoryName(assemblyPath); foreach (var asm in asms) { try { Assembly.Load(new AssemblyName(asm.FullName)); } catch (Exception) { var path = Path.Combine(basePath, asm.Name + ".dll"); try { AssemblyLoadContext.Default.LoadFromAssemblyPath(path); } catch (Exception e) { Console.Out.WriteLine($"Failed to load dll: {path}"); } } } #endif using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath)) { using (var discovery = new Discovery()) { controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); if (discovery.WasSuccessful) { foreach (var test in discovery.Tests.Reverse()) { var node = test.Value.First(); if (!string.IsNullOrEmpty(node.SkipReason)) { PublishRunnerMessage($"Skipping test {node.MethodName}. Reason - {node.SkipReason}"); continue; } if (!string.IsNullOrWhiteSpace(specName) && CultureInfo.InvariantCulture.CompareInfo.IndexOf(node.TestName, specName, CompareOptions.IgnoreCase) < 0) { PublishRunnerMessage($"Skipping [{node.MethodName}] (Filtering)"); continue; } // include filter if (!include.IsMatch(node.MethodName)) { PublishRunnerMessage($"Skipping [{node.MethodName}] (Include filter)"); continue; } if (exclude.IsMatch(node.MethodName)) { PublishRunnerMessage($"Skipping [{node.MethodName}] (Exclude filter)"); continue; } var processes = new List <Process>(); PublishRunnerMessage($"Starting test {node.MethodName}"); Console.Out.WriteLine($"Starting test {node.MethodName}"); StartNewSpec(test.Value); #if CORECLR var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe"); var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll"); var alternateIndex = 0; #endif var timelineCollector = TestRunSystem.ActorOf(Props.Create(() => new TimelineLogCollectorActor())); string testOutputDir = null; string runningSpecName = null; foreach (var nodeTest in test.Value) { //Loop through each test, work out number of nodes to run on and kick off process var sbArguments = new StringBuilder() //.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ") .Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ") .Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ") .Append($@"-Dmultinode.max-nodes={test.Value.Count} ") .Append($@"-Dmultinode.server-host=""{"localhost"}"" ") .Append($@"-Dmultinode.host=""{"localhost"}"" ") .Append($@"-Dmultinode.index={nodeTest.Node - 1} ") .Append($@"-Dmultinode.role=""{nodeTest.Role}"" ") .Append($@"-Dmultinode.listen-address={listenAddress} ") .Append($@"-Dmultinode.listen-port={listenPort} "); #if CORECLR string fileName = null; switch (platform) { case "net": fileName = ntrNetPath; sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" "); break; case "netcore": case "net5": fileName = "dotnet"; sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" "); sbArguments.Insert(0, ntrNetCorePath); break; } var process = new Process { StartInfo = new ProcessStartInfo { FileName = fileName, UseShellExecute = false, RedirectStandardOutput = true, Arguments = sbArguments.ToString(), WorkingDirectory = Path.GetDirectoryName(assemblyPath) } }; #else sbArguments.Insert(0, $@"-Dmultinode.test-assembly=""{assemblyPath}"" "); var process = new Process { StartInfo = new ProcessStartInfo { FileName = "Akka.NodeTestRunner.exe", UseShellExecute = false, RedirectStandardOutput = true, Arguments = sbArguments.ToString() } }; #endif processes.Add(process); var nodeIndex = nodeTest.Node; var nodeRole = nodeTest.Role; #if CORECLR if (platform == "netcore" || platform == "net5") { process.StartInfo.FileName = "dotnet"; process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments; process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath); } #endif //TODO: might need to do some validation here to avoid the 260 character max path error on Windows var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName)); testOutputDir = testOutputDir ?? folder.FullName; var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt"); runningSpecName = nodeTest.TestName; var nodeInfo = new TimelineLogCollectorActor.NodeInfo(nodeIndex, nodeRole, platform, nodeTest.TestName); var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath))); process.OutputDataReceived += (sender, eventArgs) => { if (eventArgs?.Data != null) { fileActor.Tell(eventArgs.Data); timelineCollector.Tell(new TimelineLogCollectorActor.LogMessage(nodeInfo, eventArgs.Data)); if (TeamCityFormattingOn) { // teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates } } }; var closureTest = nodeTest; process.Exited += (sender, eventArgs) => { if (process.ExitCode == 0) { ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.TestName); } }; process.Start(); process.BeginOutputReadLine(); PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}"); } var specFailed = false; foreach (var process in processes) { process.WaitForExit(); specFailed = specFailed || process.ExitCode > 0; process.Dispose(); } PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(3)); if (testOutputDir != null) { var dumpTasks = new List <Task>() { // Dump aggregated timeline to file for this test timelineCollector.Ask <Done>(new TimelineLogCollectorActor.DumpToFile(Path.Combine(testOutputDir, "aggregated.txt"))), // Print aggregated timeline into the console timelineCollector.Ask <Done>(new TimelineLogCollectorActor.PrintToConsole(logLevel)) }; if (specFailed) { var dumpFailureArtifactTask = timelineCollector.Ask <Done>( new TimelineLogCollectorActor.DumpToFile(Path.Combine(Path.GetFullPath(OutputDirectory), FailedSpecsDirectory, $"{runningSpecName}.txt"))); dumpTasks.Add(dumpFailureArtifactTask); } Task.WaitAll(dumpTasks.ToArray()); } FinishSpec(test.Value, timelineCollector); } Console.WriteLine("Complete"); PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(5)); } else { var sb = new StringBuilder(); sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted."); foreach (var err in discovery.Errors) { for (int i = 0; i < err.ExceptionTypes.Length; ++i) { sb.AppendLine(); sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}"); sb.Append(err.StackTraces[i]); } } PublishRunnerMessage(sb.ToString()); Console.Out.WriteLine(sb.ToString()); } } } AbortTcpLoggingServer(tcpLogger); CloseAllSinks(); //Block until all Sinks have been terminated. TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1)); if (Debugger.IsAttached) { Console.ReadLine(); //block when debugging } //Return the proper exit code Environment.Exit(ExitCodeContainer.ExitCode); }
/// <summary> /// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description. /// </summary> /// <param name="typeName"></param> /// <param name="testReportName"></param> private void ExecuteTests(string typeName = null, string testReportName = null) { ExecutionSummary executionSummary = null; XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = assemblyLocation, ConfigFilename = null }; XElement assembliesElement = new XElement("assemblies"); XElement assemblyElement = new XElement("assembly"); // Logger var verboserReporter = new XunitReporters.VerboseReporter(); IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true); IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger)); // Option setup ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null); ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null); executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel); executionOptions.SetDiagnosticMessages(true); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var clockTime = Stopwatch.StartNew(); bool cancel = false; using (var controller = new XunitFrontController( appDomainSupport: AppDomainSupport.Denied, assemblyFileName: assembly.AssemblyFilename, configFileName: null, shadowCopy: shadowCopy, shadowCopyFolder: null, sourceInformationProvider: null, diagnosticMessageSink: null)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting( assembly: assembly, appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy: shadowCopy, discoveryOptions: discoveryOptions)); if (typeName != null) { controller.Find(typeName, false, discoverySink, discoveryOptions); } else { controller.Find(false, discoverySink, discoveryOptions); } discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; }); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } clockTime.Stop(); assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); if (executionSummary != null) { Console.WriteLine(); KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary); reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > { kvpExecutionSummary })); if (testReportName != null) { // Create folder if it does not exist FileInfo fi = new FileInfo(testReportName); DirectoryInfo directory = fi.Directory; if (!directory.Exists) { Directory.CreateDirectory(directory.FullName); } CreateXmlTestReport(assembliesElement, fi); CreateHtmlTestReport(assembliesElement, fi); } Console.WriteLine(); Console.WriteLine(); } }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; if (DiagnosticMessages) assembly.Configuration.DiagnosticMessages = true; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, discoveryOptions, executionOptions)); controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, executionOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.UseAppDomain = appDomains.GetValueOrDefault(); } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var useAppDomain = assembly.Configuration.UseAppDomainOrDefault; using (var controller = new XunitFrontController(useAppDomain, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, useAppDomain, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
public static int Main(string[] args) { var arguments = new Stack <string> (); string host = null; int port = 0; bool closeAfterTestRun; bool failed; for (var i = args.Length - 1; i >= 0; i--) { arguments.Push(args[i]); } // First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal)) { var parts = arguments.Pop().Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); closeAfterTestRun = true; } else { closeAfterTestRun = false; } #if !ENABLE_NETCORE // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); #endif // some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = arguments.Pop(); var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments); var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(resultsXmlPath); if (host != null) { writer.WriteLine($"STARTRESULTXML"); resultsXml.Save(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; #else MonoSdksTextUI runner; TextWriter writer = null; string resultsXmlPath = Path.GetTempFileName(); string assemblyFileName = arguments.Peek(); if (File.Exists("nunit-excludes.txt")) { var excludes = File.ReadAllLines("nunit-excludes.txt"); arguments.Push("-exclude:" + String.Join(",", excludes)); } arguments.Push("-labels"); arguments.Push("-format:xunit"); arguments.Push($"-result:{resultsXmlPath}"); if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); writer = new TcpWriter(host, port); } else { writer = ConsoleWriter.Out; } Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); runner = new MonoSdksTextUI(writer); runner.Execute(arguments.ToArray()); if (host != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = runner.Failure; #endif Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}."); if (!closeAfterTestRun) { Thread.Sleep(Int32.MaxValue); } return(failed ? 1 : 0); }
void DiscoverTests <TVisitor>(IEnumerable <string> sources, LoggerHelper logger, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null) where TVisitor : IVsDiscoverySink, IDisposable { try { RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { var reporterMessageHandler = new DefaultRunnerReporterWithTypes().CreateMessageHandler(new VisualStudioRunnerLogger(logger)); foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources) { var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath; #if !PLATFORM_DOTNET assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath); #endif var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var configuration = LoadConfiguration(assemblyFileName); var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = configuration.ShadowCopyOrDefault; try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault); using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions); var totalTests = visitor.Finish(); visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } } catch (Exception e) { var ex = e.Unwrap(); var fileNotFound = ex as FileNotFoundException; #if !PLATFORM_DOTNET var fileLoad = ex as FileLoadException; #endif if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (fileNotFound != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !PLATFORM_DOTNET else if (fileLoad != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } } } } catch (Exception e) { logger.LogWarning("Exception discovering tests: {0}", e.Unwrap()); } }
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch) { var result = new List <IGrouping <string, TestCase> >(); RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed)); } foreach (string assemblyFileName in sources) { var fileName = Path.GetFileName(assemblyFileName); try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName)); } } else { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName)); } using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true)) using (var sink = new TestDiscoveryVisitor()) { framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions()); sink.Finished.WaitOne(); result.Add( new Grouping <string, TestCase>( assemblyFileName, sink.TestCases .GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name)) .SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(assemblyFileName, framework, testCase, settings, forceUniqueNames: group.Count() > 1))) .ToList() ) ); if (settings.MessageDisplay != MessageDisplay.None) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(assemblyFileName), sink.TestCases.Count)); } } } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e)); } } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed)); } return(result); } }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, IMessageSink reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if PLATFORM_DOTNET // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) }) .Where(tc => tc.xunit != null) .ToDictionary(tc => tc.xunit, tc => tc.vs); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled)) { controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions); executionSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
public TdNetRunnerHelper(Assembly assembly, ITestListener testListener) { this.testListener = testListener; frontController = new XunitFrontController(new Uri(assembly.CodeBase).LocalPath); }
private XElement ExecuteAssembly( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, bool needsXml, [NotNull] XunitFilters filters) { if (assembly == null) { throw new ArgumentNullException(nameof(assembly)); } if (options == null) { throw new ArgumentNullException(nameof(options)); } if (filters == null) { throw new ArgumentNullException(nameof(filters)); } if (_cancel) { return(null); } var internalDiagnosticMessages = options.InternalDiagnosticMessages; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(assembly.AssemblyFilename) || !ValidateFileExists(assembly.ConfigFilename)) { return(null); } ConfigureAssembly(assembly, options); // Setup discovery and execution options with command-line overrides var executionOptions = ConfiguExecutionOptions(assembly, options); var assemblyDisplayName = assembly.GetFileNameWithoutExtension().EmptyIfNull(); void LogAction(MessageHandlerArgs <IDiagnosticMessage> args, string assemblyName) => Console.WriteLine($"{assemblyDisplayName}: {args.Message.Message}"); var diagnosticMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var internalDiagnosticsMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink)) using (var controller = new XunitFrontController( appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => _cancel)) { ExecuteTests(assembly, options, filters, controller, discoverySink, executionOptions, assemblyElement, diagnosticMessageSink); } } catch (Exception ex) { _failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds); if (failSkips) { resultsSink = new FailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return(assemblyElement); }
private void ExecuteTests( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, [NotNull] XunitFilters filters, [NotNull] XunitFrontController controller, [NotNull] TestDiscoverySink discoverySink, [NotNull] ITestFrameworkExecutionOptions executionOptions, [CanBeNull] XElement assemblyElement, [NotNull] DiagnosticMessageSink diagnosticMessageSink) { var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var serialize = options.Serialize; var failSkips = options.FailSkips; var stopOnFail = options.StopOnFail; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); // Discover & filter the tests _reporterMessageHandler.OnMessage( new TestAssemblyDiscoveryStarting( assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink( _reporterMessageHandler, () => _cancel, (path, summary) => _completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0) { return; } Console.WriteLine("Canceling due to test failure..."); _cancel = true; } }
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework, LoggerHelper logger, TestPlatformContext testPlatformContext, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete, string assemblyFileName, bool shadowCopy, TestAssemblyConfiguration configuration) where TVisitor : IVsDiscoverySink, IDisposable { var reporterMessageHandler = GetRunnerReporter(new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger)); var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); try { if (cancelled) { return(false); } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions)); framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions); var totalTests = visitor.Finish(); visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } catch (Exception e) { var ex = e.Unwrap(); if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (ex is FileNotFoundException fileNotFound) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !WINDOWS_UAP else if (ex is FileLoadException fileLoad) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } return(true); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); } else if (!quiet) { Console.WriteLine("Discovering: {0}", assemblyDisplayName); } } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) { lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
static int Main(string[] args) { var nodeIndex = CommandLine.GetInt32("multinode.index"); var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly"); var typeName = CommandLine.GetProperty("multinode.test-class"); var testName = CommandLine.GetProperty("multinode.test-method"); var displayName = testName; var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address")); var listenPort = CommandLine.GetInt32("multinode.listen-port"); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex); var tcpClient = _logger = system.ActorOf <RunnerTcpClient>(); system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient); Thread.Sleep(TimeSpan.FromSeconds(10)); using (var controller = new XunitFrontController(assemblyFileName)) { /* need to pass in just the assembly name to Discovery, not the full path * i.e. "Akka.Cluster.Tests.MultiNode.dll" * not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause * the Discovery class to actually not find any indivudal specs to run */ var assemblyName = Path.GetFileName(assemblyFileName); Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName); using (var discovery = new Discovery(assemblyName, typeName)) { using (var sink = new Sink(nodeIndex, tcpClient)) { Thread.Sleep(10000); try { controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution()); } catch (AggregateException ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); foreach (var innerEx in ex.Flatten().InnerExceptions) { specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString()); specFail.FailureMessages.Add(innerEx.Message); specFail.FailureStackTraces.Add(innerEx.StackTrace); } _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return(1); } catch (Exception ex) { var specFail = new SpecFail(nodeIndex, displayName); specFail.FailureExceptionTypes.Add(ex.GetType().ToString()); specFail.FailureMessages.Add(ex.Message); specFail.FailureStackTraces.Add(ex.StackTrace); _logger.Tell(specFail.ToString()); Console.WriteLine(specFail); //make sure message is send over the wire FlushLogMessages(); Environment.Exit(1); //signal failure return(1); } var timedOut = false; if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out { var line = string.Format("Timed out while waiting for test to complete after {0} ms", MaxProcessWaitTimeout); _logger.Tell(line); Console.WriteLine(line); timedOut = true; } FlushLogMessages(); system.Shutdown(); system.AwaitTermination(); Environment.Exit(sink.Passed && !timedOut ? 0 : 1); return(sink.Passed ? 0 : 1); } } } }
public static void Main(string[] args) { OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty); TestRunSystem = ActorSystem.Create("TestRunnerLogging"); var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false"); if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn)) { throw new ArgumentException("Invalid argument provided for -Dteamcity"); } SinkCoordinator = TestRunSystem.ActorOf( // mutes ConsoleMessageSinkActor Props.Create <SinkCoordinator>(), "sinkCoordinator"); var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1")); var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577); var listenEndpoint = new IPEndPoint(listenAddress, listenPort); var specName = CommandLine.GetPropertyOrDefault("multinode.spec", ""); var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "netcore"); if (!_validNetCorePlatform.Contains(platform)) { throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore"); } var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger"); TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint)); var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first EnableAllSinks(assemblyPath, platform); PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}"); // In NetCore, if the assembly file hasn't been touched, // XunitFrontController would fail loading external assemblies and its dependencies. var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath); var asms = assembly.GetReferencedAssemblies(); var basePath = Path.GetDirectoryName(assemblyPath); foreach (var asm in asms) { try { Assembly.Load(new AssemblyName(asm.FullName)); } catch (Exception) { var path = Path.Combine(basePath, asm.Name + ".dll"); try { AssemblyLoadContext.Default.LoadFromAssemblyPath(path); } catch (Exception e) { Console.Out.WriteLine($"Failed to load dll: {path}, {e}"); } } } using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath)) { using (var discovery = new Discovery()) { controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); if (discovery.WasSuccessful) { foreach (var test in discovery.Tests.Reverse()) { if (!string.IsNullOrEmpty(test.Value.First().SkipReason)) { PublishRunnerMessage($"Skipping test {test.Value.First().MethodName}. Reason - {test.Value.First().SkipReason}"); continue; } if (!string.IsNullOrWhiteSpace(specName) && CultureInfo.InvariantCulture.CompareInfo.IndexOf(test.Value.First().TestName, specName, CompareOptions.IgnoreCase) < 0) { PublishRunnerMessage($"Skipping [{test.Value.First().MethodName}] (Filtering)"); continue; } var processes = new List <Process>(); PublishRunnerMessage($"Starting test {test.Value.First().MethodName}"); Console.Out.WriteLine($"Starting test {test.Value.First().MethodName}"); StartNewSpec(test.Value); var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe"); var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll"); foreach (var nodeTest in test.Value) { //Loop through each test, work out number of nodes to run on and kick off process var sbArguments = new StringBuilder() //.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ") .Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ") .Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ") .Append($@"-Dmultinode.max-nodes={test.Value.Count} ") .Append($@"-Dmultinode.server-host=""{"localhost"}"" ") .Append($@"-Dmultinode.host=""{"localhost"}"" ") .Append($@"-Dmultinode.index={nodeTest.Node - 1} ") .Append($@"-Dmultinode.role=""{nodeTest.Role}"" ") .Append($@"-Dmultinode.listen-address={listenAddress} ") .Append($@"-Dmultinode.listen-port={listenPort} "); string fileName = null; switch (platform) { case "net": fileName = ntrNetPath; sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" "); break; case "netcore": fileName = "dotnet"; sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" "); sbArguments.Insert(0, ntrNetCorePath); break; } var process = new Process { StartInfo = new ProcessStartInfo { FileName = fileName ?? throw new NullException(nameof(fileName)), UseShellExecute = false, RedirectStandardOutput = true, Arguments = sbArguments.ToString(), WorkingDirectory = Path.GetDirectoryName(assemblyPath) } }; processes.Add(process); var nodeIndex = nodeTest.Node; var nodeRole = nodeTest.Role; if (platform == "netcore") { process.StartInfo.FileName = "dotnet"; process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments; process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath); } //TODO: might need to do some validation here to avoid the 260 character max path error on Windows var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName)); var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt"); var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath))); process.OutputDataReceived += (sender, eventArgs) => { if (eventArgs?.Data != null) { fileActor.Tell(eventArgs.Data); if (TeamCityFormattingOn) { // teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates } } }; var closureTest = nodeTest; process.Exited += (sender, eventArgs) => { if (process.ExitCode == 0) { ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.TestName); } }; process.Start(); process.BeginOutputReadLine(); PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}"); } foreach (var process in processes) { process.WaitForExit(); process.Dispose(); } PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(3)); FinishSpec(test.Value); } Console.WriteLine("Complete"); PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(5)); } else { var sb = new StringBuilder(); sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted."); foreach (var err in discovery.Errors) { for (int i = 0; i < err.ExceptionTypes.Length; ++i) { sb.AppendLine(); sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}"); sb.Append(err.StackTraces[i]); } } PublishRunnerMessage(sb.ToString()); Console.Out.WriteLine(sb.ToString()); } } } CloseAllSinks(); //Block until all Sinks have been terminated. TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1)); if (Debugger.IsAttached) { Console.ReadLine(); //block when debugging } //Return the proper exit code Environment.Exit(ExitCodeContainer.ExitCode); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.GetMethodDisplay(), !executionOptions.GetDisableParallelization(), executionOptions.GetMaxParallelThreads()); } else { Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
/// <summary> /// MultiNodeTestRunner takes the following <see cref="args"/>: /// /// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] /// /// <list type="number"> /// <listheader> /// <term>Argument</term> /// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description> /// </listheader> /// <item> /// <term>AssemblyName</term> /// <description> /// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory. /// /// i.e. "Akka.Cluster.Tests.MultiNode.dll" /// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll" /// </description> /// </item> /// <item> /// <term>-Dmultinode.enable-filesink</term> /// <description>Having this flag set means that the contents of this test run will be saved in the /// current working directory as a .JSON file. /// </description> /// </item> /// <item> /// <term>-Dmultinode.multinode.output-directory</term> /// <description>Setting this flag means that any persistent multi-node test runner output files /// will be written to this directory instead of the default, which is the same folder /// as the test binary. /// </description> /// </item> /// </list> /// </summary> static void Main(string[] args) { TestRunSystem = ActorSystem.Create("TestRunnerLogging"); SinkCoordinator = TestRunSystem.ActorOf(Props.Create <SinkCoordinator>(), "sinkCoordinator"); var assemblyName = Path.GetFullPath(args[0]); EnableAllSinks(assemblyName); PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName)); using (var controller = new XunitFrontController(assemblyName)) { using (var discovery = new Discovery()) { controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery()); discovery.Finished.WaitOne(); foreach (var test in discovery.Tests.Reverse()) { if (!string.IsNullOrEmpty(test.Value.First().SkipReason)) { PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason)); continue; } PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName)); var processes = new List <Process>(); StartNewSpec(test.Value); foreach (var nodeTest in test.Value) { //Loop through each test, work out number of nodes to run on and kick off process var process = new Process(); processes.Add(process); process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.FileName = "Akka.NodeTestRunner.exe"; process.StartInfo.Arguments = String.Format(@"-Dmultinode.test-assembly=""{0}"" -Dmultinode.test-class=""{1}"" -Dmultinode.test-method=""{2}"" -Dmultinode.max-nodes={3} -Dmultinode.server-host=""{4}"" -Dmultinode.host=""{5}"" -Dmultinode.index={6}", assemblyName, nodeTest.TypeName, nodeTest.MethodName, test.Value.Count, "localhost", "localhost", nodeTest.Node - 1); var nodeIndex = nodeTest.Node; process.OutputDataReceived += (sender, line) => { //ignore any trailing whitespace if (string.IsNullOrEmpty(line.Data) || string.IsNullOrWhiteSpace(line.Data)) { return; } string message = line.Data; if (!message.StartsWith("[NODE", true, CultureInfo.InvariantCulture)) { message = "[NODE" + nodeIndex + "]" + message; } PublishToAllSinks(message); }; var closureTest = nodeTest; process.Exited += (sender, eventArgs) => { if (process.ExitCode == 0) { ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName); } }; process.Start(); process.BeginOutputReadLine(); PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id)); } foreach (var process in processes) { process.WaitForExit(); var exitCode = process.ExitCode; process.Close(); } PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(3)); FinishSpec(); } } } Console.WriteLine("Complete"); PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected."); Thread.Sleep(TimeSpan.FromSeconds(5)); CloseAllSinks(); //Block until all Sinks have been terminated. TestRunSystem.AwaitTermination(TimeSpan.FromMinutes(1)); //Return the proper exit code Environment.Exit(ExitCodeContainer.ExitCode); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly, AppDomainSupport?appDomains) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } if (stopOnFail.HasValue) { executionOptions.SetStopOnTestFail(stopOnFail); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) { ExitCode = 1; if (stopOnFail == true) { Log.LogMessage(MessageImportance.High, "Canceling due to test failure..."); Cancel(); } } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
IEnumerable<AssemblyRunInfo> DiscoverTestsInAssemblies() { var result = new List<AssemblyRunInfo>(); try { using (AssemblyHelper.SubscribeResolve()) { foreach (var assm in TestAssemblies) { // Xunit needs the file name var assemblyFileName = assm.GetName() .Name + ".dll"; var configuration = GetConfiguration(assm.GetName() .Name); var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); try { if (cancelled) break; using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false)) using (var sink = new TestDiscoveryVisitor()) { framework.Find(true, sink, discoveryOptions); sink.Finished.WaitOne(); result.Add(new AssemblyRunInfo { AssemblyFileName = assemblyFileName, Configuration = configuration, TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this)) .ToList() }); } } catch (Exception e) { Debug.WriteLine(e); } } } } catch (Exception e) { Debug.WriteLine(e); } return result; }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (FailSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
void RunTestsInAssembly(List<IDisposable> toDispose, AssemblyRunInfo runInfo) { if (cancelled) return; var assemblyFileName = runInfo.AssemblyFileName; var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName); lock (toDispose) toDispose.Add(controller); var xunitTestCases = runInfo.TestCases.Select(tc => new { vm = tc, tc = tc.TestCase }) .Where(tc => tc.tc.UniqueID != null) .ToDictionary(tc => tc.tc, tc => tc.vm); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); using (var executionVisitor = new TestExecutionVisitor(xunitTestCases, this, executionOptions, () => cancelled, context)) { controller.RunTests(xunitTestCases.Select(tc => tc.Value.TestCase).ToList(), executionVisitor, executionOptions); executionVisitor.Finished.WaitOne(); } }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) resultsVisitor = new FailSkipVisitor(resultsVisitor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
public async void RunTests(string arguments) { log = await Helpers.GetFileStreamWriterInLocalStorageAsync("stdout.txt"); string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); log.WriteLine(arguments); var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); // Discover & filter the tests log.WriteLine($"Discovering: {assemblyName}"); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; log.WriteLine($"Discovered: {assemblyName}"); // Run the filtered tests if (testCasesToRun == 0) { log.WriteLine($"Info: {assemblyName} has no tests to run"); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } var assemblyElement = new XElement("assembly"); StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, log, completionMessages, commandLine.ShowProgress, commandLine.FailSkips); xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); assembliesElement.Add(assemblyElement); log.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); log.WriteLine(e); } finally { await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement); log.Dispose(); } } Application.Current.Exit(); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent); controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
static int Main(string[] args) { // Handle RemoteExecutor if (args.Length > 0 && args[0] == "remote") { return(RemoteExecutor.Execute(args.Skip(1).ToArray())); } var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { break; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); // Discover & filter the tests Console.WriteLine($"Discovering: {assemblyName}"); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; Console.WriteLine($"Discovered: {assemblyName}"); // Run the filtered tests if (testCasesToRun == 0) { Console.WriteLine($"Info: {assemblyName} has no tests to run"); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } var assemblyElement = new XElement("assembly"); StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.ShowProgress, commandLine.FailSkips); xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); assembliesElement.Add(assemblyElement); Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); Console.WriteLine(e); } finally { WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult(); } } return(0); }