public static int Main(string[] args) { var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); testsFinished.SetResult(); }; var xunitTestFx = new SingleFileTestRunner(diagnosticSink); var asmInfo = Reflector.Wrap(asm); var asmName = asm.GetName(); var discoverySink = new TestDiscoverySink(); var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery()); discoverySink.Finished.WaitOne(); XunitFilters filters = new XunitFilters(); filters.ExcludedTraits.Add("category", new List <string> { "failing" }); var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var executor = xunitTestFx.CreateExecutor(asmName); executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution()); resultsSink.Finished.WaitOne(); var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public XUnitTestRunner(LogWriter logger) : base(logger) { messageSink = new TestMessageSink(); messageSink.Diagnostics.DiagnosticMessageEvent += HandleDiagnosticMessage; messageSink.Diagnostics.ErrorMessageEvent += HandleDiagnosticErrorMessage; messageSink.Discovery.DiscoveryCompleteMessageEvent += HandleDiscoveryCompleteMessage; messageSink.Discovery.TestCaseDiscoveryMessageEvent += HandleDiscoveryTestCaseMessage; messageSink.Runner.TestAssemblyDiscoveryFinishedEvent += HandleTestAssemblyDiscoveryFinished; messageSink.Runner.TestAssemblyDiscoveryStartingEvent += HandleTestAssemblyDiscoveryStarting; messageSink.Runner.TestAssemblyExecutionFinishedEvent += HandleTestAssemblyExecutionFinished; messageSink.Runner.TestAssemblyExecutionStartingEvent += HandleTestAssemblyExecutionStarting; messageSink.Runner.TestExecutionSummaryEvent += HandleTestExecutionSummary; messageSink.Execution.AfterTestFinishedEvent += (MessageHandlerArgs <IAfterTestFinished> args) => HandleEvent("AfterTestFinishedEvent", args, HandleAfterTestFinished); messageSink.Execution.AfterTestStartingEvent += (MessageHandlerArgs <IAfterTestStarting> args) => HandleEvent("AfterTestStartingEvent", args, HandleAfterTestStarting); messageSink.Execution.BeforeTestFinishedEvent += (MessageHandlerArgs <IBeforeTestFinished> args) => HandleEvent("BeforeTestFinishedEvent", args, HandleBeforeTestFinished); messageSink.Execution.BeforeTestStartingEvent += (MessageHandlerArgs <IBeforeTestStarting> args) => HandleEvent("BeforeTestStartingEvent", args, HandleBeforeTestStarting); messageSink.Execution.TestAssemblyCleanupFailureEvent += (MessageHandlerArgs <ITestAssemblyCleanupFailure> args) => HandleEvent("TestAssemblyCleanupFailureEvent", args, HandleTestAssemblyCleanupFailure); messageSink.Execution.TestAssemblyFinishedEvent += (MessageHandlerArgs <ITestAssemblyFinished> args) => HandleEvent("TestAssemblyFinishedEvent", args, HandleTestAssemblyFinished); messageSink.Execution.TestAssemblyStartingEvent += (MessageHandlerArgs <ITestAssemblyStarting> args) => HandleEvent("TestAssemblyStartingEvent", args, HandleTestAssemblyStarting); messageSink.Execution.TestCaseCleanupFailureEvent += (MessageHandlerArgs <ITestCaseCleanupFailure> args) => HandleEvent("TestCaseCleanupFailureEvent", args, HandleTestCaseCleanupFailure); messageSink.Execution.TestCaseFinishedEvent += (MessageHandlerArgs <ITestCaseFinished> args) => HandleEvent("TestCaseFinishedEvent", args, HandleTestCaseFinished); messageSink.Execution.TestCaseStartingEvent += (MessageHandlerArgs <ITestCaseStarting> args) => HandleEvent("TestStartingEvent", args, HandleTestCaseStarting); messageSink.Execution.TestClassCleanupFailureEvent += (MessageHandlerArgs <ITestClassCleanupFailure> args) => HandleEvent("TestClassCleanupFailureEvent", args, HandleTestClassCleanupFailure); messageSink.Execution.TestClassConstructionFinishedEvent += (MessageHandlerArgs <ITestClassConstructionFinished> args) => HandleEvent("TestClassConstructionFinishedEvent", args, HandleTestClassConstructionFinished); messageSink.Execution.TestClassConstructionStartingEvent += (MessageHandlerArgs <ITestClassConstructionStarting> args) => HandleEvent("TestClassConstructionStartingEvent", args, HandleTestClassConstructionStarting); messageSink.Execution.TestClassDisposeFinishedEvent += (MessageHandlerArgs <ITestClassDisposeFinished> args) => HandleEvent("TestClassDisposeFinishedEvent", args, HandleTestClassDisposeFinished); messageSink.Execution.TestClassDisposeStartingEvent += (MessageHandlerArgs <ITestClassDisposeStarting> args) => HandleEvent("TestClassDisposeStartingEvent", args, HandleTestClassDisposeStarting); messageSink.Execution.TestClassFinishedEvent += (MessageHandlerArgs <ITestClassFinished> args) => HandleEvent("TestClassFinishedEvent", args, HandleTestClassFinished); messageSink.Execution.TestClassStartingEvent += (MessageHandlerArgs <ITestClassStarting> args) => HandleEvent("TestClassStartingEvent", args, HandleTestClassStarting); messageSink.Execution.TestCleanupFailureEvent += (MessageHandlerArgs <ITestCleanupFailure> args) => HandleEvent("TestCleanupFailureEvent", args, HandleTestCleanupFailure); messageSink.Execution.TestCollectionCleanupFailureEvent += (MessageHandlerArgs <ITestCollectionCleanupFailure> args) => HandleEvent("TestCollectionCleanupFailureEvent", args, HandleTestCollectionCleanupFailure); messageSink.Execution.TestCollectionFinishedEvent += (MessageHandlerArgs <ITestCollectionFinished> args) => HandleEvent("TestCollectionFinishedEvent", args, HandleTestCollectionFinished); messageSink.Execution.TestCollectionStartingEvent += (MessageHandlerArgs <ITestCollectionStarting> args) => HandleEvent("TestCollectionStartingEvent", args, HandleTestCollectionStarting); messageSink.Execution.TestFailedEvent += (MessageHandlerArgs <ITestFailed> args) => HandleEvent("TestFailedEvent", args, HandleTestFailed); messageSink.Execution.TestFinishedEvent += (MessageHandlerArgs <ITestFinished> args) => HandleEvent("TestFinishedEvent", args, HandleTestFinished); messageSink.Execution.TestMethodCleanupFailureEvent += (MessageHandlerArgs <ITestMethodCleanupFailure> args) => HandleEvent("TestMethodCleanupFailureEvent", args, HandleTestMethodCleanupFailure); messageSink.Execution.TestMethodFinishedEvent += (MessageHandlerArgs <ITestMethodFinished> args) => HandleEvent("TestMethodFinishedEvent", args, HandleTestMethodFinished); messageSink.Execution.TestMethodStartingEvent += (MessageHandlerArgs <ITestMethodStarting> args) => HandleEvent("TestMethodStartingEvent", args, HandleTestMethodStarting); messageSink.Execution.TestOutputEvent += (MessageHandlerArgs <ITestOutput> args) => HandleEvent("TestOutputEvent", args, HandleTestOutput); messageSink.Execution.TestPassedEvent += (MessageHandlerArgs <ITestPassed> args) => HandleEvent("TestPassedEvent", args, HandleTestPassed); messageSink.Execution.TestSkippedEvent += (MessageHandlerArgs <ITestSkipped> args) => HandleEvent("TestSkippedEvent", args, HandleTestSkipped); messageSink.Execution.TestStartingEvent += (MessageHandlerArgs <ITestStarting> args) => HandleEvent("TestStartingEvent", args, HandleTestStarting); }
static async Task Main(string[] args) { var dll = "FakeTests.dll"; var xunit = new XunitFrontController(AppDomainSupport.Denied, dll); var discoverySink = new TestDiscoverySink(); xunit.Find(true, discoverySink, TestFrameworkOptions.ForDiscovery()); discoverySink.Finished.WaitOne(); var cases = discoverySink.TestCases; if (cases.Count == 0) { throw new Exception("No test discovered in " + dll); } var messagesSink = new TestMessageSink(); // vvv THIS IS NEVER EXECUTED vvv messagesSink.Runner.TestExecutionSummaryEvent += Runner_TestExecutionSummaryEvent; // vvv THIS IS NEVER EXECUTED vvv messagesSink.Runner.TestAssemblyExecutionFinishedEvent += Runner_TestAssemblyExecutionFinishedEvent; messagesSink.Execution.TestFinishedEvent += Execution_TestFinishedEvent; //run xunit.RunAll(messagesSink, TestFrameworkOptions.ForDiscovery(), TestFrameworkOptions.ForExecution()); finished.WaitOne(); finished.Dispose(); }
public int Run(string assemblyFileName, bool printXml, XunitFilters filters) { var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering tests for {assemblyFileName}"); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Console.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static); if (threadpoolPump != null) { while (!resultsSink.Finished.WaitOne(0)) { threadpoolPump.Invoke(this, null); } } else { resultsSink.Finished.WaitOne(); } if (printXml) { Console.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(Console.OpenStandardOutput()); Console.WriteLine(); Console.WriteLine($"ENDRESULTXML"); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public async Task <int> Run(string assemblyFileName, bool printXml, XunitFilters filters) { var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})"); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)"); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) { testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); }; } testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); while (!resultsSink.Finished.WaitOne(0)) { await Task.Delay(1); } if (printXml) { Console.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(Console.OpenStandardOutput()); Console.WriteLine(); Console.WriteLine($"ENDRESULTXML"); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public static int Main(string[] args) { var arguments = new Stack <string> (); string host = null; int port = 0; bool closeAfterTestRun; bool failed; for (var i = args.Length - 1; i >= 0; i--) { arguments.Push(args[i]); } // First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal)) { var parts = arguments.Pop().Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); closeAfterTestRun = true; } else { closeAfterTestRun = false; } // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); // some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one SynchronizationContext.SetSynchronizationContext(new SynchronizationContext()); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = arguments.Pop(); var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments); var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(resultsXmlPath); if (host != null) { writer.WriteLine($"STARTRESULTXML"); resultsXml.Save(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; #else MonoSdksTextUI runner; TextWriter writer = null; string resultsXmlPath = Path.GetTempFileName(); string assemblyFileName = arguments.Peek(); if (File.Exists("nunit-excludes.txt")) { var excludes = File.ReadAllLines("nunit-excludes.txt"); arguments.Push("-exclude:" + String.Join(",", excludes)); } arguments.Push("-labels"); arguments.Push("-format:xunit"); arguments.Push($"-result:{resultsXmlPath}"); if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); writer = new TcpWriter(host, port); } else { writer = ConsoleWriter.Out; } Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}..."); runner = new MonoSdksTextUI(writer); runner.Execute(arguments.ToArray()); if (host != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } failed = runner.Failure; #endif Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}."); if (!closeAfterTestRun) { Thread.Sleep(Int32.MaxValue); } return(failed ? 1 : 0); }
public static int Main(string[] args) { var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s")); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); testsFinished.SetResult(); }; var assemblyConfig = new TestAssemblyConfiguration() { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner PreEnumerateTheories = false, }; var xunitTestFx = new SingleFileTestRunner(diagnosticSink); var asmInfo = Reflector.Wrap(asm); var asmName = asm.GetName(); var discoverySink = new TestDiscoverySink(); var discoverer = xunitTestFx.CreateDiscoverer(asmInfo); discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig)); discoverySink.Finished.WaitOne(); string xmlResultFileName = null; XunitFilters filters = new XunitFilters(); // Quick hack wo much validation to get args that are passed (notrait, xml) Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >(); for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase)) { var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries); if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values)) { noTraits.Add(traitKeyValue[0], values = new List <string>()); } values.Add(traitKeyValue[1]); } if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase)) { xmlResultFileName = args[i + 1].Trim(); } } foreach (KeyValuePair <string, List <string> > kvp in noTraits) { filters.ExcludedTraits.Add(kvp.Key, kvp.Value); } var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var executor = xunitTestFx.CreateExecutor(asmName); executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig)); resultsSink.Finished.WaitOne(); // Helix need to see results file in the drive to detect if the test has failed or not if (xmlResultFileName != null) { resultsXmlAssembly.Save(xmlResultFileName); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits) { WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''"); Log("Starting tests..."); var filters = new XunitFilters(); foreach (var trait in excludedTraits ?? Array.Empty <string>()) { ParseEqualSeparatedArgument(filters.ExcludedTraits, trait); } var configuration = new TestAssemblyConfiguration { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2( AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Log($"Discovering tests for {assemblyFileName}..."); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Log($"Discovery finished."); Log(""); var summarySink = new DelegatingExecutionSummarySink( testSink, () => false, (completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); }; testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); }; testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); Console.WriteLine(resultsXml.ToString()); Log(""); Log("Test results (Base64 encoded):"); var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString())); Log(base64, id: "results"); return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0); }
public static int Main(string[] args) { string host = null; int port = 0; // First argument is the connection string if (args [0].StartsWith("tcp:")) { var parts = args [0].Split(':'); if (parts.Length != 3) { throw new Exception(); } host = parts [1]; port = Int32.Parse(parts [2]); args = args.Skip(1).ToArray(); } // Make sure the TLS subsystem including the DependencyInjector is initialized. // This would normally happen on system startup in // `xamarin-macios/src/ObjcRuntime/Runtime.cs`. MonoTlsProviderFactory.Initialize(); #if XUNIT_RUNNER var writer = new TcpWriter(host, port); var assemblyFileName = args[0]; var configuration = new TestAssemblyConfiguration() { ShadowCopy = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new DiagnosticTextWriterMessageSink(writer); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink); writer.WriteLine($"Discovering tests for {assemblyFileName}"); controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); writer.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); }; testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); }; testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); }; testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(discoverySink.TestCases, resultsSink, testOptions); resultsSink.Finished.WaitOne(); writer.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); #else MonoSdksTextUI runner; TcpWriter writer = null; string resultsXml = null; if (host != null) { Console.WriteLine($"Connecting to harness at {host}:{port}."); resultsXml = Path.GetTempFileName(); args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray(); writer = new TcpWriter(host, port); runner = new MonoSdksTextUI(writer); } else { runner = new MonoSdksTextUI(); } runner.Execute(args); if (resultsXml != null) { writer.WriteLine($"STARTRESULTXML"); using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream); writer.WriteLine(); writer.WriteLine($"ENDRESULTXML"); } return(runner.Failure ? 1 : 0); #endif }