public virtual TestRunState Run(IReadOnlyList <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests) { try { if (testCases == null) { testCases = Discover(); } var visitor = new ResultVisitor(testListener, testCases.Count) { TestRunState = initialRunState }; toDispose.Push(visitor); var executionOptions = TestFrameworkOptions.ForExecution(configuration); xunit.RunTests(testCases, visitor, executionOptions); visitor.Finished.WaitOne(); return(visitor.TestRunState); } catch (Exception ex) { testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error); return(TestRunState.Error); } }
public virtual TestRunState Run(IEnumerable <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests) { try { if (testCases != null) { testCases = testCases.ToList(); } var visitor = new ResultVisitor(testListener) { TestRunState = initialRunState }; toDispose.Push(visitor); var executionOptions = TestFrameworkOptions.ForExecution(configuration); if (testCases == null) { xunit.RunAll(visitor, TestFrameworkOptions.ForDiscovery(configuration), executionOptions); } else { xunit.RunTests(testCases, visitor, executionOptions); } visitor.Finished.WaitOne(); return(visitor.TestRunState); } catch (Exception ex) { testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error); return(TestRunState.Error); } }
public static IEnumerable <IMessageSinkMessage> Run(this Xunit2 runner, IEnumerable <ITestCase> testCases) { Guard.AgainstNullArgument("runner", runner); using (var sink = new SpyMessageSink <ITestAssemblyFinished>()) { runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution()); sink.Finished.WaitOne(); return(sink.Messages.Select(message => message)); } }
public static IEnumerable <IMessageSinkMessage> Run(this Xunit2 runner, IEnumerable <ITestCase> testCases) { if (!testCases.Any()) { return(Enumerable.Empty <IMessageSinkMessage>()); } using (var sink = new SpyMessageSink <ITestCollectionFinished>()) { runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution()); sink.Finished.WaitOne(); return(sink.Messages.Select(_ => _)); } }
public static Queue <IMessageSinkMessage> Run(this Xunit2 runner, Queue <ITestCase> testCases, TestAssemblyConfiguration testAssemblyConfiguration) { if (!testCases.Any()) { return(new Queue <IMessageSinkMessage>()); } using (var sink = new SpyMessageSink <ITestCollectionFinished>()) { runner.RunTests(testCases, sink, TestFrameworkOptions.ForExecution(testAssemblyConfiguration)); sink.Finished.Wait(); return(sink.Messages); } }
public List <IMessageSinkMessage> Run(Type[] types) { Xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), types[0].Assembly.GetLocalCodeBase(), configFileName: null, shadowCopy: true); var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>(); foreach (var type in types) { Xunit2.Find(type.FullName, includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery()); discoverySink.Finished.WaitOne(); discoverySink.Finished.Reset(); } var testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(msg => msg.TestCase).ToArray(); var runSink = new SpyMessageSink <ITestAssemblyFinished>(); Xunit2.RunTests(testCases, runSink, TestFrameworkOptions.ForExecution()); runSink.Finished.WaitOne(); return(runSink.Messages.ToList()); }
public int Run(string assemblyFileName, bool printXml, XunitFilters filters) { var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering tests for {assemblyFileName}"); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Console.WriteLine($"Discovery finished."); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static); if (threadpoolPump != null) { while (!resultsSink.Finished.WaitOne(0)) { threadpoolPump.Invoke(this, null); } } else { resultsSink.Finished.WaitOne(); } if (printXml) { Console.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(Console.OpenStandardOutput()); Console.WriteLine(); Console.WriteLine($"ENDRESULTXML"); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public async Task <int> Run(string assemblyFileName, bool printXml, XunitFilters filters) { var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})"); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)"); var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) { testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); }; } testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); }; testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); while (!resultsSink.Finished.WaitOne(0)) { await Task.Delay(1); } if (printXml) { Console.WriteLine($"STARTRESULTXML"); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); resultsXml.Save(Console.OpenStandardOutput()); Console.WriteLine(); Console.WriteLine($"ENDRESULTXML"); } var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0; return(failed ? 1 : 0); }
public static void DeserializedFactsAndTheoriesFromTheSameClassStillShareFixtures() { var code = @" using System; using System.Threading; using Xunit; public class TestClassFixture : IDisposable { public static long StaticConstructorCount = 0; public readonly long ConstructorInstance; public TestClassFixture() { ConstructorInstance = Interlocked.Increment(ref StaticConstructorCount); } public void Dispose() { Assert.Equal(1, StaticConstructorCount); } } public class TestClass : IClassFixture<TestClassFixture> { readonly TestClassFixture fixture; public TestClass(TestClassFixture fixture) { this.fixture = fixture; } [Fact] public void FactMethod() { Assert.Equal(1, fixture.ConstructorInstance); } [Theory] [InlineData(42)] public void TheoryMethod(int x) { Assert.Equal(1, fixture.ConstructorInstance); } } "; using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code)) { var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>(); var serializedTestCases = default(List <string>); var descriptors = default(List <TestCaseDescriptor>); using (var xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), assembly.FileName)) { xunit2.Find("TestClass", false, discoverySink, TestFrameworkOptions.ForDiscovery()); discoverySink.Finished.WaitOne(); var testCases = discoverySink.Messages .OfType <ITestCaseDiscoveryMessage>() .Select(x => x.TestCase) .ToList(); serializedTestCases = testCases.Select(x => xunit2.Serialize(x)).ToList(); descriptors = xunit2.GetTestCaseDescriptors(testCases, true); } using (var xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), assembly.FileName)) { var deserializations = default(List <ITestCase>); Action <List <KeyValuePair <string, ITestCase> > > callback = r => deserializations = r.Select(x => x.Value).ToList(); new TestCaseBulkDeserializer(xunit2, xunit2, serializedTestCases, callback); var executionSink = new SpyMessageSink <ITestAssemblyFinished>(); xunit2.RunTests(deserializations, executionSink, TestFrameworkOptions.ForExecution()); executionSink.Finished.WaitOne(); var passedTests = executionSink.Messages.OfType <ITestPassed>().ToList(); var failedTests = executionSink.Messages.OfType <ITestFailed>().ToList(); Assert.Equal(2, passedTests.Count); Assert.Empty(failedTests); } } }
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits) { WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''"); Log("Starting tests..."); var filters = new XunitFilters(); foreach (var trait in excludedTraits ?? Array.Empty <string>()) { ParseEqualSeparatedArgument(filters.ExcludedTraits, trait); } var configuration = new TestAssemblyConfiguration { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false }; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var discoverySink = new TestDiscoverySink(); var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testOptions = TestFrameworkOptions.ForExecution(configuration); var testSink = new TestMessageSink(); var controller = new Xunit2( AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false); discoveryOptions.SetSynchronousMessageReporting(true); testOptions.SetSynchronousMessageReporting(true); Log($"Discovering tests for {assemblyFileName}..."); var assembly = Assembly.LoadFrom(assemblyFileName); var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly); var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink); discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList(); Log($"Discovery finished."); Log(""); var summarySink = new DelegatingExecutionSummarySink( testSink, () => false, (completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); }); var resultsXmlAssembly = new XElement("assembly"); var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly); testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); }; testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); }; testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); }; testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); }; testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); }; controller.RunTests(testCasesToRun, resultsSink, testOptions); resultsSink.Finished.WaitOne(); var resultsXml = new XElement("assemblies"); resultsXml.Add(resultsXmlAssembly); Console.WriteLine(resultsXml.ToString()); Log(""); Log("Test results (Base64 encoded):"); var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString())); Log(base64, id: "results"); return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0); }