internal static void Go(string assemblyFileName, Stream stream) { Go(assemblyFileName, stream, AppDomainSupport.Denied, (xunit, configuration, writer) => { using (var sink = new TestDiscoverySink(writer)) { xunit.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration)); sink.Finished.WaitOne(); writer.Write(TestDataKind.EndOfData); } }); }
IReadOnlyList<ITestCase> Discover(Action<IMessageSinkWithTypes> discoveryAction) { try { var sink = new TestDiscoverySink(); toDispose.Push(sink); discoveryAction(sink); sink.Finished.WaitOne(); return sink.TestCases.ToList(); } catch (Exception ex) { testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error); return new ITestCase[0]; } }
public static int Main(string[] args) { var arguments = new Stack <string> (); string host = null; int port = 0; bool closeAfterTestRun; bool failed; for (var i = args.Length - 1; i >= 0; i--) { arguments.Push(args[i]); } // First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal)) { var parts = arguments.Pop().Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); closeAfterTestRun = true; } else { closeAfterTestRun = false; } // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); // some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = arguments.Pop(); var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments); var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(resultsXmlPath); if (host != null) { writer.WriteLine($"STARTRESULTXML"); resultsXml.Save(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; #else MonoSdksTextUI runner; TextWriter writer = null; string resultsXmlPath = Path.GetTempFileName(); string assemblyFileName = arguments.Peek(); if (File.Exists("nunit-excludes.txt")) { var excludes = File.ReadAllLines("nunit-excludes.txt"); arguments.Push("-exclude:" + String.Join(",", excludes)); } arguments.Push("-labels"); arguments.Push("-format:xunit"); arguments.Push($"-result:{resultsXmlPath}"); if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); writer = new TcpWriter(host, port); } else { writer = ConsoleWriter.Out; } Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); runner = new MonoSdksTextUI(writer); runner.Execute(arguments.ToArray()); if (host != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = runner.Failure; #endif Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}."); if (!closeAfterTestRun) { Thread.Sleep(Int32.MaxValue); } return(failed ? 1 : 0); }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename); var entryAssembly = Assembly.GetEntryAssembly(); var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly); var discoverySink = new TestDiscoverySink(() => cancel); using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions)); testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); } var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink)); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using var executor = testFramework.GetExecutor(entryAssembly.GetName()); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); executionSummary = resultsSink.ExecutionSummary; reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public static int Main(string[] args) { var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); testsFinished.SetResult(); }; var assemblyConfig = new TestAssemblyConfiguration() { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner PreEnumerateTheories = false, }; var xunitTestFx = new SingleFileTestRunner(diagnosticSink); var asmInfo = Reflector.Wrap(asm); var asmName = asm.GetName(); var discoverySink = new TestDiscoverySink(); var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig)); discoverySink.Finished.WaitOne(); string xmlResultFileName = null; XunitFilters filters = new XunitFilters(); // Quick hack wo much validation to get args that are passed (notrait, xml) Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >(); for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase)) { var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries); if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values)) { noTraits.Add(traitKeyValue[0], values = new List <string>()); } values.Add(traitKeyValue[1]); } if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase)) { xmlResultFileName = args[i + 1].Trim(); } } foreach (KeyValuePair <string, List <string> > kvp in noTraits) { filters.ExcludedTraits.Add(kvp.Key, kvp.Value); } var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var executor = xunitTestFx.CreateExecutor(asmName); executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig)); resultsSink.Finished.WaitOne(); // Helix need to see results file in the drive to detect if the test has failed or not if (xmlResultFileName != null) { resultsXmlAssembly.Save(xmlResultFileName); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public static void RunTests(Options o) { var nullMessage = new Xunit.NullMessageSink(); var discoveryOptions = TestFrameworkOptions.ForDiscovery(); using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false)) { var tv = new TestDiscoverySink(); var excludeTestCaseSet = new TestDiscoverySink(); c.Find(true, tv, discoveryOptions); tv.Finished.WaitOne(); foreach (var tc in tv.TestCases) { var method = tc.TestMethod.Method; var attributes = method.GetCustomAttributes(typeof(FactAttribute)); foreach (ReflectionAttributeInfo at in attributes) { var result = at.GetNamedArgument<string>("Skip"); if (result != null) { Console.WriteLine("SKIPPY! {0} because {1}", method, result); } if (o.TestType != TestType.All) { if (!at.ToString().EndsWith(o.TestType.ToString())) { excludeTestCaseSet.TestCases.Add(tc); } } } } foreach (var tc in excludeTestCaseSet.TestCases) { tv.TestCases.Remove(tc); } Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count); //core execution Sink int testCaseCount = tv.TestCases.Count; Stream file = new FileStream(".\\result.xml", FileMode.Create); int totalResult = 0; int totalErrors = 0; int totalFailed = 0; int totalSkipped = 0; for (int i = 0; i < testCaseCount; i++) { IExecutionSink resultsSink; ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>(); IMessageSinkWithTypes reporterMessageHandler; var reporters = GetAvailableRunnerReporters(); var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll"); IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger)); var xmlElement = new XElement("TestResult"); resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true); var message = new Xunit.NullMessageSink(); var executionOptions = TestFrameworkOptions.ForExecution(); c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions); resultsSink.Finished.WaitOne(o.TimeOut); tv.TestCases.RemoveAt(0); totalResult++; totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors; totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed; totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped; xmlElement.Save(file); file.Flush(); } file.Dispose(); Console.WriteLine("Total tests: " + totalResult); Console.ForegroundColor = ConsoleColor.DarkRed; Console.WriteLine("Error tests: " + totalErrors); Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Failed tests: " + totalFailed); Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("Skipped tests: " + totalSkipped); Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped)); Console.ResetColor(); } }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly2 assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, AppDomainSupport?appDomain, bool failSkips, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // if we had a config file use it var config = assembly.ConfigFilename != null ? assembly.Configuration : assembly.ConfigurationStream; // Turn off pre-enumeration of theories when we're not running in Visual Studio if (!designTime) { config.PreEnumerateTheories = false; } if (appDomain.HasValue) { config.AppDomain = appDomain.GetValueOrDefault(); } if (diagnosticMessages) { config.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(config); var executionOptions = TestFrameworkOptions.ForExecution(config); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, config.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = config.AppDomainOrDefault; var shadowCopy = config.ShadowCopyOrDefault; var longRunningSeconds = config.LongRunningTestSecondsOrDefault; var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly); using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink, sourceInformationProvider: sourceInformationProvider)) using (var discoverySink = new TestDiscoverySink()) { var includeSourceInformation = designTime && listTestCases; // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoverySink, discoveryOptions: discoveryOptions); discoverySink.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestCases = null; if (designTime) { vsTestCases = DesignTimeTestConverter.Convert(discoverySink.TestCases); } if (listTestCases) { lock (consoleLock) { if (designTime) { foreach (var testcase in vsTestCases.Values) { testDiscoverySink?.SendTestFound(testcase); Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoverySink.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } IExecutionSink resultsSink; if (designTime) { resultsSink = new DesignTimeExecutionSink(testExecutionSink, vsTestCases, reporterMessageHandler); } else { resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds); } if (failSkips) { resultsSink = new FailSkipSink(resultsSink); } IList <ITestCase> filteredTestCases; var testCasesDiscovered = discoverySink.TestCases.Count; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList(); } var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (FailSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
private XElement ExecuteAssembly( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, bool needsXml, [NotNull] XunitFilters filters) { if (assembly == null) { throw new ArgumentNullException(nameof(assembly)); } if (options == null) { throw new ArgumentNullException(nameof(options)); } if (filters == null) { throw new ArgumentNullException(nameof(filters)); } if (_cancel) { return(null); } var internalDiagnosticMessages = options.InternalDiagnosticMessages; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(assembly.AssemblyFilename) || !ValidateFileExists(assembly.ConfigFilename)) { return(null); } ConfigureAssembly(assembly, options); // Setup discovery and execution options with command-line overrides var executionOptions = ConfiguExecutionOptions(assembly, options); var assemblyDisplayName = assembly.GetFileNameWithoutExtension().EmptyIfNull(); void LogAction(MessageHandlerArgs <IDiagnosticMessage> args, string assemblyName) => Console.WriteLine($"{assemblyDisplayName}: {args.Message.Message}"); var diagnosticMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var internalDiagnosticsMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink)) using (var controller = new XunitFrontController( appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => _cancel)) { ExecuteTests(assembly, options, filters, controller, discoverySink, executionOptions, assemblyElement, diagnosticMessageSink); } } catch (Exception ex) { _failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits) { WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''"); Log("Starting tests..."); var filters = new XunitFilters(); foreach (var trait in excludedTraits ?? Array.Empty <string>()) { ParseEqualSeparatedArgument(filters.ExcludedTraits, trait); } var configuration = new TestAssemblyConfiguration { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2( AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Log($"Discovering tests for {assemblyFileName}..."); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Log($"Discovery finished."); Log(""); var summarySink = new DelegatingExecutionSummarySink( testSink, () => false, (completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); }; testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); }; testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); Console.WriteLine(resultsXml.ToString()); Log(""); Log("Test results (Base64 encoded):"); var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString())); Log(base64, id: "results"); return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0); }
/// <summary> /// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description. /// </summary> /// <param name="typeName"></param> /// <param name="testReportName"></param> private void ExecuteTests(string typeName = null, string testReportName = null) { ExecutionSummary executionSummary = null; XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = assemblyLocation, ConfigFilename = null }; XElement assembliesElement = new XElement("assemblies"); XElement assemblyElement = new XElement("assembly"); // Logger var verboserReporter = new XunitReporters.VerboseReporter(); IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true); IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger)); // Option setup ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null); ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null); executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel); executionOptions.SetDiagnosticMessages(true); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var clockTime = Stopwatch.StartNew(); bool cancel = false; using (var controller = new XunitFrontController( appDomainSupport: AppDomainSupport.Denied, assemblyFileName: assembly.AssemblyFilename, configFileName: null, shadowCopy: shadowCopy, shadowCopyFolder: null, sourceInformationProvider: null, diagnosticMessageSink: null)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting( assembly: assembly, appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy: shadowCopy, discoveryOptions: discoveryOptions)); if (typeName != null) { controller.Find(typeName, false, discoverySink, discoveryOptions); } else { controller.Find(false, discoverySink, discoveryOptions); } discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; }); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } clockTime.Stop(); assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); if (executionSummary != null) { Console.WriteLine(); KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary); reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > { kvpExecutionSummary })); if (testReportName != null) { // Create folder if it does not exist FileInfo fi = new FileInfo(testReportName); DirectoryInfo directory = fi.Directory; if (!directory.Exists) { Directory.CreateDirectory(directory.FullName); } CreateXmlTestReport(assembliesElement, fi); CreateHtmlTestReport(assembliesElement, fi); } Console.WriteLine(); Console.WriteLine(); } }
private static List<ITestCase> GetTestCaseList(XunitFrontController xunit, TestAssemblyConfiguration configuration, HashSet<string> testCaseNameSet) { var testCaseList = new List<ITestCase>(); using (var sink = new TestDiscoverySink(testCaseNameSet, testCaseList)) { xunit.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration)); sink.Finished.WaitOne(); } return testCaseList; }
public static int Main(string[] args) { string host = null; int port = 0; // First argument is the connection string if (args [0].StartsWith("tcp:")) { var parts = args [0].Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); args = args.Skip(1).ToArray(); } // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = args[0]; var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(discoverySink.TestCases, resultsSink, testOptions); resultsSink.Finished.WaitOne(); writer.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); #else MonoSdksTextUI runner; TcpWriter writer = null; string resultsXml = null; if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); resultsXml = Path.GetTempFileName(); args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray(); writer = new TcpWriter(host, port); runner = new MonoSdksTextUI(writer); } else { runner = new MonoSdksTextUI(); } runner.Execute(args); if (resultsXml != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } return(runner.Failure ? 1 : 0); #endif }
static int Main(string[] args) { if (args.Length < 3) { Console.WriteLine("Usage: <out-dir> <corefx dir> <test assembly filename> <xunit console options>"); return(1); } var testAssemblyName = Path.GetFileNameWithoutExtension(args [2]); var testAssemblyFull = Path.GetFullPath(args[2]); var outdir_name = Path.Combine(args [0], testAssemblyName); var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64"; args = args.Skip(2).ToArray(); // Response file support var extra_args = new List <string> (); for (int i = 0; i < args.Length; ++i) { var arg = args [i]; if (arg [0] == '@') { foreach (var line in File.ReadAllLines(arg.Substring(1))) { if (line.Length == 0 || line [0] == '#') { continue; } extra_args.AddRange(line.Split(' ')); } args [i] = ""; } } args = args.Where(s => s != String.Empty).Concat(extra_args).ToArray(); // Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir // File.Copy ($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true); var cmdline = CommandLine.Parse(args); // Ditto File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true); var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename))); var msg_sink = new MsgSink(); var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink); var sink = new TestDiscoverySink(); var config = new TestAssemblyConfiguration() { DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false }; xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config)); sink.Finished.WaitOne(); foreach (XunitTestCase tc in sink.TestCases) { ComputeTraits(assembly, tc); } // Compute testcase data var tc_data = new Dictionary <XunitTestCase, List <TcCase> > (); foreach (XunitTestCase tc in sink.TestCases) { var m = ((ReflectionMethodInfo)tc.Method).MethodInfo; var t = m.ReflectedType; var cases = new List <TcCase> (); if (m.GetParameters().Length > 0) { foreach (var cattr in m.GetCustomAttributes(true)) { if (cattr is InlineDataAttribute) { var data = ((InlineDataAttribute)cattr).GetData(null).First(); if (data == null) { data = new object [m.GetParameters().Length]; } if (data.Length != m.GetParameters().Length) { throw new Exception(); } bool unhandled = false; foreach (var val in data) { if (val is float || val is double) { unhandled = true; } if (val is Type) { var type = val as Type; if (!type.IsVisible) { unhandled = true; } } if (val is Enum) { if (!val.GetType().IsPublic) { unhandled = true; } } } if (!unhandled) { cases.Add(new TcCase() { Values = data }); } } else if (cattr is MemberDataAttribute memberData && memberData.Parameters.Length == 0) { MethodInfo testDataMethod = m.DeclaringType.GetMethod(memberData.MemberName); if (testDataMethod == null) { continue; } cases.Add(new TcCase { MemberDataMethod = testDataMethod }); } } } else { cases.Add(new TcCase()); } tc_data [tc] = cases; } var filters = cmdline.Project.Filters; // // Generate code using the roslyn syntax apis // Creating syntax nodes one-by-one is very cumbersome, so use // CSharpSyntaxTree.ParseText () to create them and ReplaceNode () to insert them into syntax trees. // var blocks = new List <BlockSyntax> (); foreach (XunitTestCase tc in sink.TestCases) { var m = ((ReflectionMethodInfo)tc.Method).MethodInfo; //Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null)); var t = m.ReflectedType; if (t.IsGenericType) { nskipped++; continue; } if (!filters.Filter(tc)) { nskipped++; continue; } var cases = tc_data [tc]; int caseindex = 0; foreach (var test in cases) { string typename = GetTypeName(t); string msg; if (cases.Count > 1) { msg = $"{typename}.{m.Name}[{caseindex}] ..."; } else { msg = $"{typename}.{m.Name} ..."; } caseindex++; var block = ParseText <BlockSyntax> (case_template .Replace("#MSG#", msg) .Replace("#NRUN#", test.MemberDataMethod == null ? "1" : ((IEnumerable)test.MemberDataMethod.Invoke(null, null)).Cast <object> ().Count().ToString())); // Obtain the node for the CALL () line var try_body_node = block.DescendantNodes().OfType <ExpressionStatementSyntax> ().Skip(2).First().Parent; // Replace with the generated call code var stmts = GenerateTcCall(t, test, m); blocks.Add(block.ReplaceNode(try_body_node, Block(stmts.ToArray()))); } } var cu = CSharpSyntaxTree.ParseText(runner_template.Replace("#NSKIPPED#", nskipped.ToString())).GetRoot(); // Replace the body of the Run () method with the generated body var run_body = cu.DescendantNodes().OfType <MethodDeclarationSyntax> ().First().DescendantNodes().OfType <BlockSyntax> ().First(); cu = cu.ReplaceNode(run_body, Block(blocks.ToArray())); // Create runner.cs Directory.CreateDirectory(outdir_name); var outfile_name = Path.Combine(outdir_name, "runner.cs"); File.WriteAllText(outfile_name, cu.NormalizeWhitespace().ToString()); // Generate csproj file var csproj_template = File.ReadAllText("gen-test.csproj.template"); csproj_template = csproj_template.Replace("#XUNIT_LOCATION#", sdkdir); csproj_template = csproj_template.Replace("#TEST_ASSEMBLY#", testAssemblyName); csproj_template = csproj_template.Replace("#TEST_ASSEMBLY_LOCATION#", testAssemblyFull); File.WriteAllText(Path.Combine(outdir_name, testAssemblyName + "-runner.csproj"), csproj_template); return(0); }
static int Main(string[] args) { if (args.Length < 3) { Console.WriteLine("Usage: <out-dir> <corefx dir> <test assembly filename> <xunit console options>"); return(1); } var testAssemblyName = Path.GetFileNameWithoutExtension(args [2]); var testAssemblyFull = Path.GetFullPath(args[2]); var outdir_name = Path.Combine(args [0], testAssemblyName); var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64"; args = args.Skip(2).ToArray(); // Response file support var extra_args = new List <string> (); for (int i = 0; i < args.Length; ++i) { var arg = args [i]; if (arg [0] == '@') { foreach (var line in File.ReadAllLines(arg.Substring(1))) { if (line.Length == 0 || line [0] == '#') { continue; } extra_args.AddRange(line.Split(' ')); } args [i] = ""; } } args = args.Where(s => s != String.Empty).Concat(extra_args).ToArray(); // Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir // File.Copy ($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true); var cmdline = CommandLine.Parse(args); // Ditto File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true); var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename))); var msg_sink = new MsgSink(); var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink); var sink = new TestDiscoverySink(); var config = new TestAssemblyConfiguration() { DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false }; xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config)); sink.Finished.WaitOne(); foreach (XunitTestCase tc in sink.TestCases) { ComputeTraits(assembly, tc); } // Compute testcase data var tc_data = new Dictionary <XunitTestCase, List <CaseData> > (); foreach (XunitTestCase tc in sink.TestCases) { var m = ((ReflectionMethodInfo)tc.Method).MethodInfo; var t = m.ReflectedType; var cases = new List <CaseData> (); if (m.GetParameters().Length > 0) { foreach (var cattr in m.GetCustomAttributes(true)) { if (cattr is InlineDataAttribute) { var data = ((InlineDataAttribute)cattr).GetData(null).First(); if (data == null) { data = new object [m.GetParameters().Length]; } if (data.Length != m.GetParameters().Length) { throw new Exception(); } bool unhandled = false; foreach (var val in data) { if (val is float || val is double) { unhandled = true; } if (val is Type) { var type = val as Type; if (!type.IsVisible) { unhandled = true; } } if (val is Enum) { if (!val.GetType().IsPublic) { unhandled = true; } } } if (!unhandled) { cases.Add(new CaseData() { Values = data }); } } } } else { cases.Add(new CaseData()); } tc_data [tc] = cases; } #if FALSE //w.WriteLine ($"\t\ttypeof({typename}).GetMethod (\"{m.Name}\", BindingFlags.Static|BindingFlags.NonPublic).Invoke(null, null);"); //w.WriteLine ($"\t\tusing (var o = new {typename} ()) {{"); //if (cmdline.StopOnFail) //w.WriteLine ("\t\tif (nfailed > 0) return 1;"); //w.WriteLine ("\t\tConsole.WriteLine (\"RUN: \" + nrun + \", FAILED: \" + nfailed);"); #endif var cu = new CodeCompileUnit(); var ns = new CodeNamespace(""); cu.Namespaces.Add(ns); ns.Imports.Add(new CodeNamespaceImport("System")); ns.Imports.Add(new CodeNamespaceImport("System.Reflection")); var code_class = new CodeTypeDeclaration("RunTests"); ns.Types.Add(code_class); var code_main = new CodeEntryPointMethod(); code_main.ReturnType = new CodeTypeReference("System.Int32"); code_class.Members.Add(code_main); var statements = code_main.Statements; statements.Add(new CodeVariableDeclarationStatement(typeof(int), "nrun", new CodePrimitiveExpression(0))); statements.Add(new CodeVariableDeclarationStatement(typeof(int), "nfailed", new CodePrimitiveExpression(0))); int nskipped = 0; var filters = cmdline.Project.Filters; foreach (XunitTestCase tc in sink.TestCases) { var m = ((ReflectionMethodInfo)tc.Method).MethodInfo; //Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null)); var t = m.ReflectedType; if (t.IsGenericType) { nskipped++; continue; } if (!filters.Filter(tc)) { nskipped++; continue; } var cases = tc_data [tc]; int caseindex = 0; foreach (var test in cases) { string typename = GetTypeName(t); string msg; if (cases.Count > 1) { msg = $"{typename}.{m.Name}[{caseindex}] ..."; } else { msg = $"{typename}.{m.Name} ..."; } caseindex++; statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodePrimitiveExpression(msg) })); statements.Add(new CodeAssignStatement(new CodeVariableReferenceExpression("nrun"), new CodeBinaryOperatorExpression(new CodeVariableReferenceExpression("nrun"), CodeBinaryOperatorType.Add, new CodePrimitiveExpression(1)))); var try1 = new CodeTryCatchFinallyStatement(); statements.Add(try1); if (!m.IsStatic) { // FIXME: Disposable try1.TryStatements.Add(new CodeVariableDeclarationStatement("var", "o", new CodeObjectCreateExpression(t, new CodeExpression[] {}))); } if (!m.IsPublic) { // FIXME: nskipped++; } else { CodeMethodInvokeExpression call; if (m.IsStatic) { call = new CodeMethodInvokeExpression(new CodeTypeReferenceExpression(t), m.Name, new CodeExpression [] {}); } else { call = new CodeMethodInvokeExpression(new CodeVariableReferenceExpression("o"), m.Name, new CodeExpression [] {}); } if (test.Values != null) { var parameters = m.GetParameters(); for (var index = 0; index < test.Values.Length; index++) { var val = test.Values [index]; call.Parameters.Add(EncodeValue(val, parameters [index].ParameterType)); } } try1.TryStatements.Add(call); } var catch1 = new CodeCatchClause("ex", new CodeTypeReference("System.Exception")); catch1.Statements.Add(new CodeAssignStatement(new CodeVariableReferenceExpression("nfailed"), new CodeBinaryOperatorExpression(new CodeVariableReferenceExpression("nfailed"), CodeBinaryOperatorType.Add, new CodePrimitiveExpression(1)))); catch1.Statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodeBinaryOperatorExpression( new CodePrimitiveExpression("FAILED: "), CodeBinaryOperatorType.Add, new CodeVariableReferenceExpression("ex")) })); try1.CatchClauses.Add(catch1); } } statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodeBinaryOperatorExpression( new CodePrimitiveExpression("RUN: "), CodeBinaryOperatorType.Add, new CodeVariableReferenceExpression("nrun")) })); statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodeBinaryOperatorExpression( new CodePrimitiveExpression("FAILURES: "), CodeBinaryOperatorType.Add, new CodeVariableReferenceExpression("nfailed")) })); statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodeBinaryOperatorExpression( new CodePrimitiveExpression("SKIPPED: "), CodeBinaryOperatorType.Add, new CodePrimitiveExpression(nskipped)) })); statements.Add(new CodeMethodReturnStatement(new CodePrimitiveExpression(0))); Directory.CreateDirectory(outdir_name); var outfile_name = Path.Combine(outdir_name, "runner.cs"); var provider = new CSharpCodeProvider(); using (var w2 = File.CreateText(outfile_name)) { provider.GenerateCodeFromCompileUnit(cu, w2, new CodeGeneratorOptions()); } var csproj_template = File.ReadAllText("gen-test.csproj.template"); csproj_template = csproj_template.Replace("#XUNIT_LOCATION#", sdkdir); csproj_template = csproj_template.Replace("#TEST_ASSEMBLY#", testAssemblyName); csproj_template = csproj_template.Replace("#TEST_ASSEMBLY_LOCATION#", testAssemblyFull); File.WriteAllText(Path.Combine(outdir_name, testAssemblyName + "-runner.csproj"), csproj_template); return(0); }
private void ExecuteTests( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, [NotNull] XunitFilters filters, [NotNull] XunitFrontController controller, [NotNull] TestDiscoverySink discoverySink, [NotNull] ITestFrameworkExecutionOptions executionOptions, [CanBeNull] XElement assemblyElement, [NotNull] DiagnosticMessageSink diagnosticMessageSink) { var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var serialize = options.Serialize; var failSkips = options.FailSkips; var stopOnFail = options.StopOnFail; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); // Discover & filter the tests _reporterMessageHandler.OnMessage( new TestAssemblyDiscoveryStarting( assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink( _reporterMessageHandler, () => _cancel, (path, summary) => _completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0) { return; } Console.WriteLine("Canceling due to test failure..."); _cancel = true; } }
static int Main(string[] args) { if (args.Length < 3) { Console.WriteLine("Usage: <outfile> <corefx dir> <test assembly filename> <xunit console options>"); return(1); } var outfile_name = args [0]; var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64"; args = args.Skip(2).ToArray(); // Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir File.Copy($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true); File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true); var cmdline = CommandLine.Parse(args); // Ditto File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true); var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename))); var msg_sink = new MsgSink(); var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink); var sink = new TestDiscoverySink(); var config = new TestAssemblyConfiguration() { DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false }; xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config)); sink.Finished.WaitOne(); foreach (XunitTestCase tc in sink.TestCases) { ComputeTraits(assembly, tc); } var w = File.CreateText(outfile_name); w.WriteLine("using System;"); w.WriteLine("using System.Reflection;"); w.WriteLine("public class RunTests {"); w.WriteLine("\tpublic static int Main () {"); w.WriteLine("\t\tint nrun = 0;"); w.WriteLine("\t\tint nfailed = 0;"); var filters = cmdline.Project.Filters; foreach (XunitTestCase tc in sink.TestCases) { var m = ((ReflectionMethodInfo)tc.Method).MethodInfo; //Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null)); if (tc.TestMethodArguments != null || m.GetParameters().Length > 0) { continue; } var t = m.ReflectedType; if (t.IsGenericType) { continue; } /* * foreach (var trait in tc.Traits) { * foreach (var s in trait.Value) * Console.WriteLine (m.Name + " " + trait.Key + " " + s); * } */ if (!filters.Filter(tc)) { //Console.WriteLine ("FILTERED: " + m); continue; } string typename = GetTypeName(t); w.WriteLine($"Console.WriteLine (\"{typename}:{m.Name}...\");"); w.WriteLine("\t\tnrun ++;"); w.WriteLine("\t\ttry {"); if (m.IsStatic) { if (!m.IsPublic) { w.WriteLine($"\t\ttypeof({typename}).GetMethod (\"{m.Name}\", BindingFlags.Static|BindingFlags.NonPublic).Invoke(null, null);"); } else { w.WriteLine($"\t\t{typename}.{m.Name} ();"); } } else { if (typeof(IDisposable).IsAssignableFrom(t)) { w.WriteLine($"\t\tusing (var o = new {typename} ()) {{"); } else { w.WriteLine("\t\t{"); w.WriteLine($"\t\t\tvar o = new {typename} ();"); } w.WriteLine($"\t\t\to.{m.Name} ();"); w.WriteLine("\t\t}"); } w.WriteLine("\t\t} catch (Exception ex) { nfailed ++; Console.WriteLine (\"FAILED: \" + ex); }"); if (cmdline.StopOnFail) { w.WriteLine("\t\tif (nfailed > 0) return 1;"); } } w.WriteLine("\t\tConsole.WriteLine (\"RUN: \" + nrun + \", FAILED: \" + nfailed);"); w.WriteLine("\t\treturn 0;"); w.WriteLine("\t}"); w.WriteLine("}"); w.Close(); return(0); }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = assembly.AssemblyDisplayName; var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var assemblyInfo = new ReflectionAssemblyInfo(testAssembly); await using var disposalTracker = new DisposalTracker(); var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo); disposalTracker.Add(testFramework); var discoverySink = new TestDiscoverySink(() => cancel); // Discover & filter the tests var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo); var discoveryStarting = new TestAssemblyDiscoveryStarting { AppDomain = AppDomainOption.NotAvailable, Assembly = assembly, DiscoveryOptions = discoveryOptions, ShadowCopy = false }; reporterMessageHandler.OnMessage(discoveryStarting); testDiscoverer.Find(discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; var discoveryFinished = new TestAssemblyDiscoveryFinished { Assembly = assembly, DiscoveryOptions = discoveryOptions, TestCasesDiscovered = testCasesDiscovered, TestCasesToRun = testCasesToRun }; reporterMessageHandler.OnMessage(discoveryFinished); // Run the filtered tests if (testCasesToRun == 0) { testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary()); } else { var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var executor = testFramework.GetExecutor(assemblyInfo); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml() { var xml = @" <assembly> <class name='Type1'> <method name='Method1 Display Name' type='Type1' method='Method1'/> </class> <class name='SpecialType'> <method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/> <method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'> <traits> <trait name='Trait1' value='Value1'/> <trait name='Trait2' value='Value2'/> </traits> </method> </class> </assembly>"; using (var xunit1 = new TestableXunit1()) { xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml)); var sink = new TestDiscoverySink(); xunit1.Find(false, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => { Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("Method1", testCase.TestMethod.Method.Name); Assert.Equal("Method1 Display Name", testCase.DisplayName); Assert.Null(testCase.SkipReason); Assert.Empty(testCase.Traits); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName); Assert.Equal("I am not run", testCase.SkipReason); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName); Assert.Collection(testCase.Traits.Keys, key => { Assert.Equal("Trait1", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value1", value) ); }, key => { Assert.Equal("Trait2", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value2", value) ); } ); } ); } }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (failSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; #if NETCOREAPP1_0 || NETCOREAPP2_0 using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename)) #endif using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public int Run(string assemblyFileName, bool printXml, XunitFilters filters) { var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering tests for {assemblyFileName}"); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Console.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static); var timerPump = Type.GetType("System.Threading.TimerQueue")?.GetMethod("PumpTimerQueue", BindingFlags.NonPublic | BindingFlags.Static); if (threadpoolPump != null && timerPump != null) { while (!resultsSink.Finished.WaitOne(0)) { threadpoolPump.Invoke(this, null); timerPump.Invoke(this, null); } } else { resultsSink.Finished.WaitOne(); } if (printXml) { Console.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(Console.OpenStandardOutput()); Console.WriteLine(); Console.WriteLine($"ENDRESULTXML"); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }