public void FindByTypesReturnsOnlyMethodsInTheGivenType() { var xml = @" <assembly> <class name='Type1'> <method name='Method1 Display Name' type='Type1' method='Method1'/> </class> <class name='Type2'> <method name='Type2.Method1' type='Type2' method='Method1'/> <method name='Type2.Method2' type='Type2' method='Method2'/> </class> </assembly>"; using (var xunit1 = new TestableXunit1()) { xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml)); var sink = new TestDiscoveryVisitor(); xunit1.Find("Type2", false, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => Assert.Equal("Type2.Method1", testCase.DisplayName), testCase => Assert.Equal("Type2.Method2", testCase.DisplayName) ); } }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions()); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !ParallelizeTestCollections, MaxParallelThreads = MaxParallelThreads }; controller.RunTests(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
public void TestCasesUseInformationFromSourceInformationProvider() { var xml = @" <assembly> <class name='Type2'> <method name='Type2.Method1' type='Type2' method='Method1'/> <method name='Type2.Method2' type='Type2' method='Method2'/> </class> </assembly>"; using (var xunit1 = new TestableXunit1()) { xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml)); xunit1.SourceInformationProvider .GetSourceInformation(null) .ReturnsForAnyArgs(callInfo => new SourceInformation { FileName = "File for " + callInfo.Arg <ITestCase>().DisplayName }); var sink = new TestDiscoveryVisitor(); xunit1.Find(true, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => Assert.Equal("File for Type2.Method1", testCase.SourceInformation.FileName), testCase => Assert.Equal("File for Type2.Method2", testCase.SourceInformation.FileName) ); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } lock (consoleLock) Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
IEnumerable <AssemblyRunInfo> DiscoverTestsInAssemblies() { var result = new List <AssemblyRunInfo>(); try { using (AssemblyHelper.SubscribeResolve()) { foreach (var assm in TestAssemblies) { // Xunit needs the file name var assemblyFileName = assm.GetName() .Name + ".dll"; var configuration = GetConfiguration(assm.GetName() .Name); var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); try { if (cancelled) { break; } using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false)) using (var sink = new TestDiscoveryVisitor()) { framework.Find(true, sink, discoveryOptions); sink.Finished.WaitOne(); result.Add(new AssemblyRunInfo { AssemblyFileName = assemblyFileName, Configuration = configuration, TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this)) .ToList() }); } } catch (Exception e) { Debug.WriteLine(e); } } } } catch (Exception e) { Debug.WriteLine(e); } return(result); }
protected TestableXunitTestFrameworkDiscoverer(IAssemblyInfo assembly, ISourceInformationProvider sourceProvider, IXunitTestCollectionFactory collectionFactory, IMessageAggregator messageAggregator) : base(assembly, sourceProvider ?? Substitute.For <ISourceInformationProvider>(), collectionFactory, messageAggregator) { Assembly = assembly; Visitor = new TestDiscoveryVisitor(); }
protected TestableXunitTestFrameworkDiscoverer(IAssemblyInfo assembly, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory) : base(assembly, sourceProvider ?? Substitute.For <ISourceInformationProvider>(), diagnosticMessageSink ?? new Xunit.NullMessageSink(), collectionFactory) { Assembly = assembly; Visitor = new TestDiscoveryVisitor(); }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor); discoveryVisitor.Finished.WaitOne(); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
public IList <ITestCase> GetTestCases() { var options = TestFrameworkOptions.ForDiscovery(environment.TestAssemblyConfiguration); Logger.LogVerbose("Starting discovery"); LogDiscoveryOptions(options); var visitor = new TestDiscoveryVisitor(runContext); discoverer.Find(false, visitor, options); visitor.Finished.WaitOne(); Logger.LogVerbose("Filtering test cases"); return(visitor.TestCases); }
IReadOnlyList <ITestCase> Discover(Action <IMessageSink> discoveryAction) { try { var visitor = new TestDiscoveryVisitor(); toDispose.Push(visitor); discoveryAction(visitor); visitor.Finished.WaitOne(); return(visitor.TestCases.ToList()); } catch (Exception ex) { testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error); return(new ITestCase[0]); } }
private IEnumerable<ITestCase> Discover(Action<IMessageSink> discoveryAction) { try { var visitor = new TestDiscoveryVisitor(); toDispose.Push(visitor); discoveryAction(visitor); visitor.Finished.WaitOne(); return visitor.TestCases.ToList(); } catch (Exception ex) { testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error); return Enumerable.Empty<ITestCase>(); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor); discoveryVisitor.Finished.WaitOne(); var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent); controller.Run(discoveryVisitor.TestCases, resultsVisitor); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Console.WriteLine(stackLine); } e = e.InnerException; } } return(assemblyElement); }
public void CollectsTestCases() { var visitor = new TestDiscoveryVisitor(); var testCase1 = Substitute.For<ITestCase>(); var testCase2 = Substitute.For<ITestCase>(); var testCase3 = Substitute.For<ITestCase>(); visitor.OnMessage(new DiscoveryMessage(testCase1)); visitor.OnMessage(new DiscoveryMessage(testCase2)); visitor.OnMessage(new DiscoveryMessage(testCase3)); visitor.OnMessage(Substitute.For<IMessageSinkMessage>()); // Ignored Assert.Collection(visitor.TestCases, msg => Assert.Same(testCase1, msg), msg => Assert.Same(testCase2, msg), msg => Assert.Same(testCase3, msg) ); }
public static void TheoriesAlwaysComeBackAsSingleXunitTheoryTestCase() { var sourceProvider = new NullSourceInformationProvider(); var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly()); var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider); var visitor = new TestDiscoveryVisitor(); discoverer.Find(typeof(ClassWithTheory).FullName, false, visitor, new XunitDiscoveryOptions()); visitor.Finished.WaitOne(); var testCase = Assert.Single(visitor.TestCases); Assert.IsType <XunitTheoryTestCase>(testCase); var deserialized = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(testCase)); Assert.IsType <XunitTheoryTestCase>(deserialized); }
public static void TheoryWithNonSerializableData_ReturnsAsASingleTestCase() { var sourceProvider = new NullSourceInformationProvider(); var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly()); var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider); var visitor = new TestDiscoveryVisitor(); discoverer.Find(typeof(ClassWithNonSerializableTheoryData).FullName, false, visitor, TestFrameworkOptions.ForDiscovery()); visitor.Finished.WaitOne(); var testCase = Assert.Single(visitor.TestCases); Assert.IsType <XunitTheoryTestCase>(testCase); var deserialized = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(testCase)); Assert.IsType <XunitTheoryTestCase>(deserialized); }
public void CollectsTestCases() { var visitor = new TestDiscoveryVisitor(); var testCase1 = Substitute.For <ITestCase>(); var testCase2 = Substitute.For <ITestCase>(); var testCase3 = Substitute.For <ITestCase>(); visitor.OnMessage(new DiscoveryMessage(testCase1)); visitor.OnMessage(new DiscoveryMessage(testCase2)); visitor.OnMessage(new DiscoveryMessage(testCase3)); visitor.OnMessage(Substitute.For <IMessageSinkMessage>()); // Ignored Assert.Collection(visitor.TestCases, msg => Assert.Same(testCase1, msg), msg => Assert.Same(testCase2, msg), msg => Assert.Same(testCase3, msg) ); }
private async Task AddAssemblies(IEnumerable <AssemblyAndConfigFile> assemblies) { var loadingDialog = new LoadingDialog { Owner = MainWindow.Instance }; try { using (AssemblyHelper.SubscribeResolve()) { loadingDialog.Show(); foreach (var assembly in assemblies) { loadingDialog.AssemblyFileName = assembly.AssemblyFileName; using (var xunit = new XunitFrontController( useAppDomain: true, assemblyFileName: assembly.AssemblyFileName, configFileName: assembly.ConfigFileName, diagnosticMessageSink: new DiagnosticMessageVisitor(), shadowCopy: false)) using (var testDiscoveryVisitor = new TestDiscoveryVisitor(xunit)) { await Task.Run(() => { xunit.Find(includeSourceInformation: false, messageSink: testDiscoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery()); testDiscoveryVisitor.Finished.WaitOne(); }); allTestCases.AddRange(testDiscoveryVisitor.TestCases); Assemblies.Add(new TestAssemblyViewModel(assembly)); } } } } catch (Exception ex) { MessageBox.Show(Application.Current.MainWindow, ex.ToString()); } finally { loadingDialog.Close(); } }
public static void SerializedTestsInSameCollectionRemainInSameCollection() { var sourceProvider = new NullSourceInformationProvider(); var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly()); var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider); var visitor = new TestDiscoveryVisitor(); discoverer.Find(typeof(ClassUnderTest).FullName, false, visitor, new XunitDiscoveryOptions()); visitor.Finished.WaitOne(); var first = visitor.TestCases[0]; var second = visitor.TestCases[1]; Assert.True(TestCollectionComparer.Instance.Equals(first.TestCollection, second.TestCollection)); var serializedFirst = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(first)); var serializedSecond = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(second)); Assert.NotSame(serializedFirst.TestCollection, serializedSecond.TestCollection); Assert.True(TestCollectionComparer.Instance.Equals(serializedFirst.TestCollection, serializedSecond.TestCollection)); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent, bool parallelizeTestCollections, int maxThreadCount) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent); controller.RunTests(discoveryVisitor.TestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
public static void TheoriesWithSerializableData_ReturnAsIndividualTestCases() { var sourceProvider = new NullSourceInformationProvider(); var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly()); var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider); var visitor = new TestDiscoveryVisitor(); discoverer.Find(typeof(ClassWithTheory).FullName, false, visitor, TestFrameworkOptions.ForDiscovery()); visitor.Finished.WaitOne(); var first = visitor.TestCases[0]; var second = visitor.TestCases[1]; Assert.NotEqual(first.UniqueID, second.UniqueID); Assert.True(TestCollectionComparer.Instance.Equals(first.TestMethod.TestClass.TestCollection, second.TestMethod.TestClass.TestCollection)); var serializedFirst = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(first)); var serializedSecond = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(second)); Assert.NotSame(serializedFirst.TestMethod.TestClass.TestCollection, serializedSecond.TestMethod.TestClass.TestCollection); Assert.True(TestCollectionComparer.Instance.Equals(serializedFirst.TestMethod.TestClass.TestCollection, serializedSecond.TestMethod.TestClass.TestCollection)); }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) return null; var assemblyElement = CreateAssemblyXElement(); try { using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions()); discoveryVisitor.Finished.WaitOne(); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !ParallelizeTestCollections, MaxParallelThreads = MaxParallelThreads }; controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml() { var xml = @" <assembly> <class name='Type1'> <method name='Method1 Display Name' type='Type1' method='Method1'/> </class> <class name='SpecialType'> <method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/> <method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'> <traits> <trait name='Trait1' value='Value1'/> <trait name='Trait2' value='Value2'/> </traits> </method> </class> </assembly>"; using (var xunit1 = new TestableXunit1()) { xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml)); var sink = new TestDiscoveryVisitor(); xunit1.Find(false, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => { Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("Method1", testCase.TestMethod.Method.Name); Assert.Equal("Method1 Display Name", testCase.DisplayName); Assert.Null(testCase.SkipReason); Assert.Empty(testCase.Traits); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName); Assert.Equal("I am not run", testCase.SkipReason); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName); Assert.Collection(testCase.Traits.Keys, key => { Assert.Equal("Trait1", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value1", value) ); }, key => { Assert.Equal("Trait2", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value2", value) ); } ); } ); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; if (diagnosticMessages) assembly.Configuration.DiagnosticMessages = true; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); if (!quiet) lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); else Console.WriteLine("Discovering: {0}", assemblyDisplayName); } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return assemblyElement; }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration) { if (cancel) return null; var assemblyElement = CreateAssemblyXElement(); try { if (DiagnosticMessages) configuration.DiagnosticMessages = true; var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var executionOptions = TestFrameworkOptions.ForExecution(configuration); if (maxParallelThreads.HasValue && maxParallelThreads.Value > -1) executionOptions.SetMaxParallelThreads(maxParallelThreads); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); if (configuration.DiagnosticMessagesOrDefault) Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); else Log.LogMessage(MessageImportance.High, " Discovering: {0}", assemblyDisplayName); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, configuration.DiagnosticMessagesOrDefault); using (var controller = CreateFrontController(assemblyFileName, configFileName, diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, discoveryMessageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", assemblyDisplayName); var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); if (filteredTestCases.Count == 0) completionMessages.TryAdd(assemblyDisplayName, new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool? parallelizeTestCollections, int? maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages ?? false) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.GetMethodDisplay(), !executionOptions.GetDisableParallelization(), executionOptions.GetMaxParallelThreads()); else Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) { Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
IEnumerable<AssemblyRunInfo> DiscoverTestsInAssemblies() { var result = new List<AssemblyRunInfo>(); try { using (AssemblyHelper.SubscribeResolve()) { foreach (var assm in TestAssemblies) { // Xunit needs the file name var assemblyFileName = assm.GetName() .Name + ".dll"; var configuration = GetConfiguration(assm.GetName() .Name); var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); try { if (cancelled) break; using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false)) using (var sink = new TestDiscoveryVisitor()) { framework.Find(true, sink, discoveryOptions); sink.Finished.WaitOne(); result.Add(new AssemblyRunInfo { AssemblyFileName = assemblyFileName, Configuration = configuration, TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this)) .ToList() }); } } catch (Exception e) { Debug.WriteLine(e); } } } } catch (Exception e) { Debug.WriteLine(e); } return result; }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) resultsVisitor = new FailSkipVisitor(resultsVisitor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
static int Main(string[] args) { // Handle RemoteExecutor if (args.Length > 0 && args[0] == "remote") { return(RemoteExecutor.Execute(args.Skip(1).ToArray())); } if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help") { PrintHeader(); PrintUsage(); return(2); } var commandLine = CommandLine.Parse(args); Console.CancelKeyPress += (sender, e) => { if (!cancel) { Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)"); cancel = true; e.Cancel = true; } }; if (commandLine.Debug) { Debugger.Launch(); } if (!commandLine.NoLogo) { PrintHeader(); } var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); int errorCount = 0; int failCount = 0; foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { break; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); // Discover & filter the tests Console.WriteLine($"Discovering: {assemblyName}"); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; Console.WriteLine($"Discovered: {assemblyName}"); // Run the filtered tests if (testCasesToRun == 0) { Console.WriteLine($"Info: {assemblyName} has no tests to run"); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } var assemblyElement = new XElement("assembly"); StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.Verbose, commandLine.FailSkips); xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); assembliesElement.Add(assemblyElement); // Set counters to determine the error code later. errorCount += resultsVisitor.ExecutionSummary.Errors; failCount += resultsVisitor.ExecutionSummary.Failed; Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception ex) { assembliesElement = new XElement("error"); assembliesElement.Add(ex); if (!commandLine.NoColor) { Console.ForegroundColor = ConsoleColor.Red; } Console.WriteLine($"error: {ex.Message}"); if (commandLine.DiagnosticMessages) { if (!commandLine.NoColor) { Console.ForegroundColor = ConsoleColor.DarkGray; } Console.WriteLine(ex.StackTrace); } } finally { if (!commandLine.NoColor) { Console.ResetColor(); } WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult(); } } if (cancel) { return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C } if (commandLine.Wait) { Console.WriteLine(); Console.Write("Press any key to continue..."); Console.ReadKey(); Console.WriteLine(); } if (errorCount > 0 || failCount > 0) { return(1); } return(0); }
static int Main(string[] args) { // Handle RemoteExecutor if (args.Length > 0 && args[0] == "remote") { return(RemoteExecutor.Execute(args.Skip(1).ToArray())); } var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); int errorCount = 0; int failCount = 0; foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { break; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); // Discover & filter the tests Console.WriteLine($"Discovering: {assemblyName}"); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; Console.WriteLine($"Discovered: {assemblyName}"); // Run the filtered tests if (testCasesToRun == 0) { Console.WriteLine($"Info: {assemblyName} has no tests to run"); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } var assemblyElement = new XElement("assembly"); StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.ShowProgress, commandLine.FailSkips); xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); assembliesElement.Add(assemblyElement); // Set counters to determine the error code later. errorCount += resultsVisitor.ExecutionSummary.Errors; failCount += resultsVisitor.ExecutionSummary.Failed; Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); Console.WriteLine(e); } finally { WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult(); } } if (errorCount > 0) { return(2); } else if (failCount > 0) { return(1); } return(0); }
public void TestCasesUseInformationFromSourceInformationProvider() { var xml = @" <assembly> <class name='Type2'> <method name='Type2.Method1' type='Type2' method='Method1'/> <method name='Type2.Method2' type='Type2' method='Method2'/> </class> </assembly>"; var xunit1 = new TestableXunit1(); xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml)); xunit1.SourceInformationProvider .GetSourceInformation(null) .ReturnsForAnyArgs(callInfo => new SourceInformation { FileName = "File for " + callInfo.Arg<ITestCase>().DisplayName }); var sink = new TestDiscoveryVisitor(); xunit1.Find(true, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => Assert.Equal("File for Type2.Method1", testCase.SourceInformation.FileName), testCase => Assert.Equal("File for Type2.Method2", testCase.SourceInformation.FileName) ); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return(assemblyElement); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; if (DiagnosticMessages) assembly.Configuration.DiagnosticMessages = true; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, discoveryOptions, executionOptions)); controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, executionOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
public async void RunTests(string arguments, StringBuilder sbLog, Action <string> uiLogger) { log = sbLog; var reporters = new List <IRunnerReporter>(); string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); log.AppendLine($"Args: {arguments}"); var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger(uiLogger)); var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; //log.AppendLine("testCasesToRun: " + testCasesToRun); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); //log.AppendLine("Finished running tests"); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); log.AppendLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Time: {resultsVisitor.ExecutionSummary.Time}"); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); } finally { await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement); await WriteLogs(Path.GetFileName(assembly.AssemblyFilename), log.ToString()); } } Application.Current.Exit(); }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) return null; var assemblyElement = CreateAssemblyXElement(); try { using (var controller = CreateFrontController(assemblyFileName, configFileName)) { var discoveryVisitor = new TestDiscoveryVisitor(); controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor); discoveryVisitor.Finished.WaitOne(); var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement); controller.Run(discoveryVisitor.TestCases, resultsVisitor); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } catch (Exception ex) { Exception e = ex; while (e != null) { Log.LogError(e.GetType().FullName + ": " + e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
private async void RunTests(string arguments) { var reporters = new List<IRunnerReporter>(); string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries); log = string.Empty; log += "Args: " + args + "\n"; var commandLine = CommandLine.Parse(args); if (commandLine.Debug) { Debugger.Launch(); } var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger()); var completionMessages = new ConcurrentDictionary<string, ExecutionSummary>(); var assembliesElement = new XElement("assemblies"); foreach (var assembly in commandLine.Project.Assemblies) { if (cancel) { return; } assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages; assembly.Configuration.AppDomain = AppDomainSupport.Denied; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetDisableParallelization(true); try { using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); xunit.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; log += "testCasesToRun: " + testCasesToRun + "\n"; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (commandLine.Serialize) { filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); var assemblyElement = new XElement("assembly"); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (commandLine.FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions); log += "finished running tests \n"; resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } } catch (Exception e) { assembliesElement = new XElement("error"); assembliesElement.Add(e); log += "logged exec errors: " + e + "\n"; } } await WriteResults(assembliesElement); await WriteLogs(log); Application.Current.Exit(); }
public void FindByTypesReturnsOnlyMethodsInTheGivenType() { var xml = @" <assembly> <class name='Type1'> <method name='Method1 Display Name' type='Type1' method='Method1'/> </class> <class name='Type2'> <method name='Type2.Method1' type='Type2' method='Method1'/> <method name='Type2.Method2' type='Type2' method='Method2'/> </class> </assembly>"; var xunit1 = new TestableXunit1(); xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml)); var sink = new TestDiscoveryVisitor(); xunit1.Find("Type2", false, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => Assert.Equal("Type2.Method1", testCase.DisplayName), testCase => Assert.Equal("Type2.Method2", testCase.DisplayName) ); }
XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, string visitor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { var discoveryOptions = new XunitDiscoveryOptions(assembly.Configuration); var executionOptions = new XunitExecutionOptions(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.MaxParallelThreads = maxThreadCount.GetValueOrDefault(); } if (parallelizeTestCollections.HasValue) { executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault(); } lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.MethodDisplay, !executionOptions.DisableParallelization, executionOptions.MaxParallelThreads); } else { Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestcases = null; if (designTime) { vsTestcases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); if (listTestCases) { lock (consoleLock) { if (designTime) { var sink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink)); foreach (var testcase in vsTestcases.Values) { if (sink != null) { sink.SendTest(testcase); } Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, visitor); if (designTime) { var sink = (ITestExecutionSink)_services.GetService(typeof(ITestExecutionSink)); resultsVisitor = new DesignTimeExecutionVisitor( sink, vsTestcases, resultsVisitor); } IList <ITestCase> filteredTestCases; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = (from t in vsTestcases where designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName) select t.Key) .ToList(); } if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml() { var xml = @" <assembly> <class name='Type1'> <method name='Method1 Display Name' type='Type1' method='Method1'/> </class> <class name='SpecialType'> <method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/> <method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'> <traits> <trait name='Trait1' value='Value1'/> <trait name='Trait2' value='Value2'/> </traits> </method> </class> </assembly>"; var xunit1 = new TestableXunit1(); xunit1.Executor .WhenForAnyArgs(x => x.EnumerateTests(null)) .Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml)); var sink = new TestDiscoveryVisitor(); xunit1.Find(false, sink); sink.Finished.WaitOne(); Assert.Collection(sink.TestCases, testCase => { Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("Method1", testCase.TestMethod.Method.Name); Assert.Equal("Method1 Display Name", testCase.DisplayName); Assert.Null(testCase.SkipReason); Assert.Empty(testCase.Traits); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName); Assert.Equal("I am not run", testCase.SkipReason); }, testCase => { Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name); Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name); Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName); Assert.Collection(testCase.Traits.Keys, key => { Assert.Equal("Trait1", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value1", value) ); }, key => { Assert.Equal("Trait2", key); Assert.Collection(testCase.Traits[key], value => Assert.Equal("Value2", value) ); } ); } ); }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { var discoveryOptions = new XunitDiscoveryOptions(configuration); var executionOptions = new XunitExecutionOptions(configuration); if (maxParallelThreads.HasValue) { executionOptions.MaxParallelThreads = maxParallelThreads.GetValueOrDefault(); } if (parallelizeTestCollections.HasValue) { executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault(); } if (configuration.DiagnosticMessages) { Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assemblyFileName), discoveryOptions.MethodDisplay, !executionOptions.DisableParallelization, executionOptions.MaxParallelThreads); } else { Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); } using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); if (filteredTestCases.Count == 0) { Log.LogError("{0} has no tests to run", Path.GetFileNameWithoutExtension(assemblyFileName)); ExitCode = 1; } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories when we're not running in Visual Studio if (!designTime) { assembly.Configuration.PreEnumerateTheories = false; } if (diagnosticMessages) { assembly.Configuration.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var sourceInformationProvider = new SourceInformationProviderAdapater(testHostServices?.SourceInformationProvider); using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { var includeSourceInformation = designTime && listTestCases; // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestCases = null; if (designTime) { vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } if (listTestCases) { lock (consoleLock) { if (designTime) { var sink = testHostServices.TestDiscoverySink; foreach (var testcase in vsTestCases.Values) { if (sink != null) { sink.SendTest(testcase); } Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } IExecutionVisitor resultsVisitor; if (designTime) { resultsVisitor = new DesignTimeExecutionVisitor(testHostServices.TestExecutionSink, vsTestCases, reporterMessageHandler); } else { resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); } IList <ITestCase> filteredTestCases; var testCasesDiscovered = discoveryVisitor.TestCases.Count; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList(); } var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch) { var result = new List <IGrouping <string, TestCase> >(); RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed)); } foreach (string assemblyFileName in sources) { var fileName = Path.GetFileName(assemblyFileName); try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName)); } } else { if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName)); } using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true)) using (var sink = new TestDiscoveryVisitor()) { framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions()); sink.Finished.WaitOne(); result.Add( new Grouping <string, TestCase>( assemblyFileName, sink.TestCases .GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name)) .SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(assemblyFileName, framework, testCase, settings, forceUniqueNames: group.Count() > 1))) .ToList() ) ); if (settings.MessageDisplay != MessageDisplay.None) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(assemblyFileName), sink.TestCases.Count)); } } } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e)); } } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed)); } return(result); } }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { if (DiagnosticMessages) { configuration.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); var executionOptions = TestFrameworkOptions.ForExecution(configuration); if (maxParallelThreads.HasValue && maxParallelThreads.Value > -1) { executionOptions.SetMaxParallelThreads(maxParallelThreads); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); if (configuration.DiagnosticMessagesOrDefault) { Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); } else { Log.LogMessage(MessageImportance.High, " Discovering: {0}", assemblyDisplayName); } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, configuration.DiagnosticMessagesOrDefault); using (var controller = CreateFrontController(assemblyFileName, configFileName, diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, discoveryMessageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", assemblyDisplayName); var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(assemblyDisplayName, new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent); controller.Run(discoveryVisitor.TestCases, resultsVisitor, new TestFrameworkOptions()); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Console.WriteLine(stackLine); e = e.InnerException; } } return assemblyElement; }
void RunTestsInAssembly(IFrameworkHandle frameworkHandle, List<IDisposable> toDispose, string assemblyFileName, IEnumerable<TestCase> testCases = null) { if (cancelled) return; var xunit2 = new Xunit2(SourceInformationProvider, assemblyFileName, configFileName: null, shadowCopy: true); toDispose.Add(xunit2); if (testCases == null) { using (var visitor = new TestDiscoveryVisitor()) { xunit2.Find(includeSourceInformation: true, messageSink: visitor); visitor.Finished.WaitOne(); testCases = visitor.TestCases.Select(tc => VsDiscoveryVisitor.CreateVsTestCase(frameworkHandle, assemblyFileName, xunit2, tc)).ToList(); } } var xunitTestCases = testCases.ToDictionary(tc => xunit2.Deserialize(tc.GetPropertyValue<string>(SerializedTestCaseProperty, null))); using (var executionVisitor = new VsExecutionVisitor(assemblyFileName, frameworkHandle, xunitTestCases, () => cancelled)) { xunit2.Run(xunitTestCases.Keys.ToList(), executionVisitor); executionVisitor.Finished.WaitOne(); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } if (diagnosticMessages) { assembly.Configuration.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); if (!quiet) { lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); } else { Console.WriteLine("Discovering: {0}", assemblyDisplayName); } } } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) { lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
/// <summary> /// Starts the process of running all the xUnit.net v1 tests in the assembly. /// </summary> /// <param name="messageSink">The message sink to report results back to.</param> public void Run(IMessageSink messageSink) { var discoverySink = new TestDiscoveryVisitor(); toDispose.Push(discoverySink); Find(false, discoverySink); discoverySink.Finished.WaitOne(); Run(discoverySink.TestCases, messageSink); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (FailSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); if (resultsVisitor.ExecutionSummary.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent); controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
protected virtual void ExecuteAssembly(string assemblyFilename, string configFileName, MSBuildVisitor resultsVisitor) { try { Log.LogMessage(MessageImportance.High, "Test assembly: {0}", assemblyFilename); using (var controller = CreateFrontController(assemblyFilename, configFileName)) { var discoveryVisitor = new TestDiscoveryVisitor(); controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor); discoveryVisitor.Finished.WaitOne(); controller.Run(discoveryVisitor.TestCases, resultsVisitor); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Exception e = ex; while (e != null) { Log.LogError(e.GetType().FullName + ": " + e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } }