protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions()); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !ParallelizeTestCollections, MaxParallelThreads = MaxParallelThreads }; controller.RunTests(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } lock (consoleLock) Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
public static void TestOptions_MaxThreads() { var options = new XunitExecutionOptions { MaxParallelThreads = 3 }; var runner = TestableXunitTestAssemblyRunner.Create(options: options); var result = runner.GetTestFrameworkEnvironment(); Assert.EndsWith("[collection-per-class, parallel (3 threads)]", result); }
public static void TestOptions_NonParallel() { var options = new XunitExecutionOptions { DisableParallelization = true }; var runner = TestableXunitTestAssemblyRunner.Create(options: options); var result = runner.GetTestFrameworkEnvironment(); Assert.EndsWith("[collection-per-class, non-parallel]", result); }
void RunTestsInAssembly(IDiscoveryContext discoveryContext, IFrameworkHandle frameworkHandle, List <IDisposable> toDispose, string assemblyFileName, IEnumerable <TestCase> testCases, XunitVisualStudioSettings settings, Stopwatch stopwatch) { if (cancelled) { return; } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } #if WIN8_STORE // For store apps, the files are copied to the AppX dir, we need to load it from there assemblyFileName = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), Path.GetFileName(assemblyFileName)); #elif WINDOWS_PHONE_APP // For WPA Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true); lock (toDispose) toDispose.Add(controller); var xunitTestCases = testCases.ToDictionary(tc => controller.Deserialize(tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))); using (var executionVisitor = new VsExecutionVisitor(discoveryContext, frameworkHandle, xunitTestCases, () => cancelled)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !settings.ParallelizeTestCollections, MaxParallelThreads = settings.MaxParallelThreads }; controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions); executionVisitor.Finished.WaitOne(); } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution finished: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } }
public static void TestOptionsOverrideAttribute() { var attribute = Mocks.CollectionBehaviorAttribute(disableTestParallelization: true, maxParallelThreads: 127); var options = new XunitExecutionOptions { DisableParallelization = false, MaxParallelThreads = 3 }; var assembly = Mocks.TestAssembly(new[] { attribute }); var runner = TestableXunitTestAssemblyRunner.Create(assembly: assembly, options: options); var result = runner.GetTestFrameworkEnvironment(); Assert.EndsWith("[collection-per-class, parallel (3 threads)]", result); }
public static async void NonParallel() { var passing = Mocks.XunitTestCase <ClassUnderTest>("Passing"); var other = Mocks.XunitTestCase <ClassUnderTest>("Other"); var options = new XunitExecutionOptions { DisableParallelization = true }; var runner = TestableXunitTestAssemblyRunner.Create(testCases: new[] { passing, other }, options: options); await runner.RunAsync(); var threadIDs = runner.TestCasesRun.Select(x => x.Item1).ToList(); Assert.Equal(threadIDs[0], threadIDs[1]); }
void RunTestsInAssembly(IDiscoveryContext discoveryContext, IFrameworkHandle frameworkHandle, List <IDisposable> toDispose, string assemblyFileName, IEnumerable <TestCase> testCases, XunitVisualStudioSettings settings, Stopwatch stopwatch) { if (cancelled) { return; } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true); lock (toDispose) toDispose.Add(controller); var xunitTestCases = testCases.ToDictionary(tc => controller.Deserialize(tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))); using (var executionVisitor = new VsExecutionVisitor(discoveryContext, frameworkHandle, xunitTestCases, () => cancelled)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !settings.ParallelizeTestCollections, MaxParallelThreads = settings.MaxParallelThreads }; controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions); executionVisitor.Finished.WaitOne(); } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution finished: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName))); } }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent, bool parallelizeTestCollections, int maxThreadCount) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent); controller.RunTests(discoveryVisitor.TestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent); controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName) { if (cancel) return null; var assemblyElement = CreateAssemblyXElement(); try { using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions()); discoveryVisitor.Finished.WaitOne(); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var executionOptions = new XunitExecutionOptions { DisableParallelization = !ParallelizeTestCollections, MaxParallelThreads = MaxParallelThreads }; controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration) { if (cancel) { return(null); } var assemblyElement = CreateAssemblyXElement(); try { var discoveryOptions = new XunitDiscoveryOptions(configuration); var executionOptions = new XunitExecutionOptions(configuration); if (maxParallelThreads.HasValue) { executionOptions.MaxParallelThreads = maxParallelThreads.GetValueOrDefault(); } if (parallelizeTestCollections.HasValue) { executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault(); } if (configuration.DiagnosticMessages) { Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assemblyFileName), discoveryOptions.MethodDisplay, !executionOptions.DisableParallelization, executionOptions.MaxParallelThreads); } else { Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); } using (var controller = CreateFrontController(assemblyFileName, configFileName)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName)); using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement)) { var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); if (filteredTestCases.Count == 0) { Log.LogError("{0} has no tests to run", Path.GetFileNameWithoutExtension(assemblyFileName)); ExitCode = 1; } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, string visitor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { var discoveryOptions = new XunitDiscoveryOptions(assembly.Configuration); var executionOptions = new XunitExecutionOptions(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.MaxParallelThreads = maxThreadCount.GetValueOrDefault(); } if (parallelizeTestCollections.HasValue) { executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault(); } lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.MethodDisplay, !executionOptions.DisableParallelization, executionOptions.MaxParallelThreads); } else { Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestcases = null; if (designTime) { vsTestcases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); if (listTestCases) { lock (consoleLock) { if (designTime) { var sink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink)); foreach (var testcase in vsTestcases.Values) { if (sink != null) { sink.SendTest(testcase); } Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, visitor); if (designTime) { var sink = (ITestExecutionSink)_services.GetService(typeof(ITestExecutionSink)); resultsVisitor = new DesignTimeExecutionVisitor( sink, vsTestcases, resultsVisitor); } IList <ITestCase> filteredTestCases; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = (from t in vsTestcases where designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName) select t.Key) .ToList(); } if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
void ExecuteAssembly(object consoleLock, string assemblyName, CommandLine options) { if (cancel) { return; } try { var name = new AssemblyName(assemblyName); var assembly = Reflector.Wrap(Assembly.Load(name)); var framework = GetFramework(assembly); var discoverer = framework.GetDiscoverer(assembly); var executor = framework.GetExecutor(name); var discoveryVisitor = new TestDiscoveryVisitor(); discoverer.Find(includeSourceInformation: true, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); if (options.List) { ITestDiscoverySink discoverySink = null; if (options.DesignTime) { discoverySink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink)); } lock (consoleLock) { foreach (var test in discoveryVisitor.TestCases) { if (discoverySink != null) { discoverySink.SendTest(test.ToDesignTimeTest()); } Console.WriteLine(test.DisplayName); } } return; } var executionOptions = new XunitExecutionOptions { DisableParallelization = !options.ParallelizeTestCollections, MaxParallelThreads = options.MaxParallelThreads }; var resultsVisitor = CreateVisitor(consoleLock, options); var tests = discoveryVisitor.TestCases; if (options.Tests != null && options.Tests.Count > 0) { tests = tests.Where(t => IsTestNameMatch(t, options.Tests)).ToList(); } executor.RunTests(tests, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); // When executing under TeamCity, we record the results in a format TeamCity understands, but do not return an error code. // This causes TeamCity to treat the step as completed, but the build as failed. We'll work around this by special casing the TeamCityVisitor var teamCityVisitor = resultsVisitor as TeamCityVisitor; if (teamCityVisitor != null) { failed = teamCityVisitor.Failed > 0; } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); Console.WriteLine(ex); failed = true; } }