public static int Main(string[] args) { var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); testsFinished.SetResult(); }; var xunitTestFx = new SingleFileTestRunner(diagnosticSink); var asmInfo = Reflector.Wrap(asm); var asmName = asm.GetName(); var discoverySink = new TestDiscoverySink(); var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery()); discoverySink.Finished.WaitOne(); XunitFilters filters = new XunitFilters(); filters.ExcludedTraits.Add("category", new List <string> { "failing" }); var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var executor = xunitTestFx.CreateExecutor(asmName); executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution()); resultsSink.Finished.WaitOne(); var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { var assemblyFileName = Guard.ArgumentNotNull("assembly.AssemblyFilename", assembly.AssemblyFilename); // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using var _ = AssemblyHelper.SubscribeResolveForAssembly(assemblyFileName, internalDiagnosticsMessageSink); await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink); var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters); controller.FindAndRun(resultsSink, settings); resultsSink.Finished.WaitOne(); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public async ValueTask <int> EntryPoint() { var globalInternalDiagnosticMessages = false; var noColor = false; try { var reporters = GetAvailableRunnerReporters(); if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help") { PrintHeader(); PrintUsage(reporters); return(2); } if (commandLine.ParseFault != null) { ExceptionDispatchInfo.Capture(commandLine.ParseFault).Throw(); } if (commandLine.Project.Assemblies.Count == 0) { throw new ArgumentException("must specify at least one assembly"); } AppDomain.CurrentDomain.UnhandledException += OnUnhandledException; Console.CancelKeyPress += (sender, e) => { if (!cancel) { Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)"); cancel = true; e.Cancel = true; } }; await using var reporter = commandLine.ChooseReporter(reporters); if (commandLine.Project.Configuration.PauseOrDefault) { Console.Write("Press any key to start execution..."); Console.ReadKey(true); Console.WriteLine(); } if (commandLine.Project.Configuration.DebugOrDefault) { Debugger.Launch(); } // We will enable "global" internal diagnostic messages if any test assembly wanted them globalInternalDiagnosticMessages = commandLine.Project.Assemblies.Any(a => a.Configuration.InternalDiagnosticMessagesOrDefault); noColor = commandLine.Project.Configuration.NoColorOrDefault; logger = new ConsoleRunnerLogger(!noColor, consoleLock); var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, globalInternalDiagnosticMessages, noColor); var reporterMessageHandler = await reporter.CreateMessageHandler(logger, diagnosticMessageSink); if (!reporter.ForceNoLogo && !commandLine.Project.Configuration.NoLogoOrDefault) { PrintHeader(); } var failCount = await RunProject( commandLine.Project, reporterMessageHandler ); if (cancel) { return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C } if (commandLine.Project.Configuration.WaitOrDefault) { Console.WriteLine(); Console.Write("Press any key to continue..."); Console.ReadKey(); Console.WriteLine(); } return(failCount > 0 ? 1 : 0); } catch (Exception ex) { if (!noColor) { ConsoleHelper.SetForegroundColor(ConsoleColor.Red); } Console.WriteLine($"error: {ex.Message}"); if (globalInternalDiagnosticMessages) { if (!noColor) { ConsoleHelper.SetForegroundColor(ConsoleColor.DarkGray); } Console.WriteLine(ex.StackTrace); } return(ex is ArgumentException ? 3 : 4); } finally { if (!noColor) { ConsoleHelper.ResetColor(); } } }
public static int Main(string[] args) { var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); testsFinished.SetResult(); }; var assemblyConfig = new TestAssemblyConfiguration() { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner PreEnumerateTheories = false, }; var xunitTestFx = new SingleFileTestRunner(diagnosticSink); var asmInfo = Reflector.Wrap(asm); var asmName = asm.GetName(); var discoverySink = new TestDiscoverySink(); var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig)); discoverySink.Finished.WaitOne(); string xmlResultFileName = null; XunitFilters filters = new XunitFilters(); // Quick hack wo much validation to get args that are passed (notrait, xml) Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >(); for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase)) { var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries); if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values)) { noTraits.Add(traitKeyValue[0], values = new List <string>()); } values.Add(traitKeyValue[1]); } if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase)) { xmlResultFileName = args[i + 1].Trim(); } } foreach (KeyValuePair <string, List <string> > kvp in noTraits) { filters.ExcludedTraits.Add(kvp.Key, kvp.Value); } var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var executor = xunitTestFx.CreateExecutor(asmName); executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig)); resultsSink.Finished.WaitOne(); // Helix need to see results file in the drive to detect if the test has failed or not if (xmlResultFileName != null) { resultsXmlAssembly.Save(xmlResultFileName); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = assembly.AssemblyDisplayName; var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var assemblyInfo = new ReflectionAssemblyInfo(testAssembly); await using var disposalTracker = new DisposalTracker(); var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo); disposalTracker.Add(testFramework); var discoverySink = new TestDiscoverySink(() => cancel); // Discover & filter the tests var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo); var discoveryStarting = new TestAssemblyDiscoveryStarting { AppDomain = AppDomainOption.NotAvailable, Assembly = assembly, DiscoveryOptions = discoveryOptions, ShadowCopy = false }; reporterMessageHandler.OnMessage(discoveryStarting); testDiscoverer.Find(discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; var discoveryFinished = new TestAssemblyDiscoveryFinished { Assembly = assembly, DiscoveryOptions = discoveryOptions, TestCasesDiscovered = testCasesDiscovered, TestCasesToRun = testCasesToRun }; reporterMessageHandler.OnMessage(discoveryFinished); // Run the filtered tests if (testCasesToRun == 0) { testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary()); } else { var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var executor = testFramework.GetExecutor(assemblyInfo); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits) { WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''"); Log("Starting tests..."); var filters = new XunitFilters(); foreach (var trait in excludedTraits ?? Array.Empty <string>()) { ParseEqualSeparatedArgument(filters.ExcludedTraits, trait); } var configuration = new TestAssemblyConfiguration { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2( AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Log($"Discovering tests for {assemblyFileName}..."); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Log($"Discovery finished."); Log(""); var summarySink = new DelegatingExecutionSummarySink( testSink, () => false, (completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); }; testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); }; testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); Console.WriteLine(resultsXml.ToString()); Log(""); Log("Test results (Base64 encoded):"); var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString())); Log(base64, id: "results"); return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0); }