static XElement ExecuteAssembly(XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor); discoveryVisitor.Finished.WaitOne(); var resultsVisitor = CreateVisitor(assemblyElement, teamCity, silent); controller.Run(discoveryVisitor.TestCases, resultsVisitor); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Console.WriteLine(stackLine); e = e.InnerException; } } return assemblyElement; }
public List<IResultXmlTransform> GetTransforms(XunitProjectAssembly assembly) { List<IResultXmlTransform> results = new List<IResultXmlTransform>(); foreach (var output in assembly.Output) { if (output.Key == "xml") results.Add(new NullTransformer(output.Value)); else { TransformConfigurationElement configItem = GetTransformConfigItem(output.Key); if (configItem == null) throw new ArgumentException(String.Format("unknown output transform: {0}", output.Key)); string xslFilename = Path.Combine(executablePath, configItem.XslFile); if (!FileExists(xslFilename)) throw new ArgumentException(String.Format("cannot find transform XSL file '{0}' for transform '{1}'", xslFilename, output.Key)); results.Add(new XslStreamTransformer(xslFilename, output.Value)); } } return results; }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; #if NETCOREAPP1_0 || NETCOREAPP2_0 using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename)) #endif using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.IfAvailable : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; #if NETCOREAPP1_0 using (NetCoreAssemblyHelper.SubscribeResolve(Path.GetDirectoryName(assembly.AssemblyFilename))) #endif using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, string visitor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { var discoveryOptions = new XunitDiscoveryOptions(assembly.Configuration); var executionOptions = new XunitExecutionOptions(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.MaxParallelThreads = maxThreadCount.GetValueOrDefault(); } if (parallelizeTestCollections.HasValue) { executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault(); } lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.MethodDisplay, !executionOptions.DisableParallelization, executionOptions.MaxParallelThreads); } else { Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } } using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestcases = null; if (designTime) { vsTestcases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); if (listTestCases) { lock (consoleLock) { if (designTime) { var sink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink)); foreach (var testcase in vsTestcases.Values) { if (sink != null) { sink.SendTest(testcase); } Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, visitor); if (designTime) { var sink = (ITestExecutionSink)_services.GetService(typeof(ITestExecutionSink)); resultsVisitor = new DesignTimeExecutionVisitor( sink, vsTestcases, resultsVisitor); } IList <ITestCase> filteredTestCases; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = (from t in vsTestcases where designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName) select t.Key) .ToList(); } if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) { assembly.Configuration.AppDomain = AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) { resultsVisitor = new FailSkipVisitor(resultsVisitor); } controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return(assemblyElement); }
private static ISourceInformationProvider GetSourceInformationProviderAdapater(XunitProjectAssembly assembly) { var directoryPath = Path.GetDirectoryName(assembly.AssemblyFilename); var assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var pdbPath = Path.Combine(directoryPath, assemblyName + FileNameSuffixes.DotNet.ProgramDatabase); return(File.Exists(pdbPath) ? new SourceInformationProviderAdapater(new SourceInformationProvider(pdbPath, null)) : null); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList() }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); // Force unique names if there is more than 1 testcase with the same name foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1)) { foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames) { discoveredTestCaseWithDuplicateName.ForceUniqueName(); } } foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; if (DiagnosticMessages) assembly.Configuration.DiagnosticMessages = true; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, discoveryOptions, executionOptions)); controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, executionOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); if (resultsVisitor.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, IMessageSink reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if PLATFORM_DOTNET // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) }) .Where(tc => tc.xunit != null) .ToDictionary(tc => tc.xunit, tc => tc.vs); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled)) { controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions); executionSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.UseAppDomain = appDomains.GetValueOrDefault(); } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var useAppDomain = assembly.Configuration.UseAppDomainOrDefault; using (var controller = new XunitFrontController(useAppDomain, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, useAppDomain, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); if (resultsVisitor.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
/// <summary> /// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description. /// </summary> /// <param name="typeName"></param> /// <param name="testReportName"></param> private void ExecuteTests(string typeName = null, string testReportName = null) { ExecutionSummary executionSummary = null; XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = assemblyLocation, ConfigFilename = null }; XElement assembliesElement = new XElement("assemblies"); XElement assemblyElement = new XElement("assembly"); // Logger var verboserReporter = new XunitReporters.VerboseReporter(); IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true); IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger)); // Option setup ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null); ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null); executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel); executionOptions.SetDiagnosticMessages(true); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var clockTime = Stopwatch.StartNew(); bool cancel = false; using (var controller = new XunitFrontController( appDomainSupport: AppDomainSupport.Denied, assemblyFileName: assembly.AssemblyFilename, configFileName: null, shadowCopy: shadowCopy, shadowCopyFolder: null, sourceInformationProvider: null, diagnosticMessageSink: null)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting( assembly: assembly, appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy: shadowCopy, discoveryOptions: discoveryOptions)); if (typeName != null) { controller.Find(typeName, false, discoverySink, discoveryOptions); } else { controller.Find(false, discoverySink, discoveryOptions); } discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; }); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); assembliesElement.Add(assemblyElement); } } clockTime.Stop(); assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); if (executionSummary != null) { Console.WriteLine(); KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary); reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > { kvpExecutionSummary })); if (testReportName != null) { // Create folder if it does not exist FileInfo fi = new FileInfo(testReportName); DirectoryInfo directory = fi.Directory; if (!directory.Exists) { Directory.CreateDirectory(directory.FullName); } CreateXmlTestReport(assembliesElement, fi); CreateHtmlTestReport(assembliesElement, fi); } Console.WriteLine(); Console.WriteLine(); } }
public static List<IResultXmlTransform> GetAssemblyTransforms(XunitProjectAssembly assembly) { return instance.GetTransforms(assembly); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; if (diagnosticMessages) assembly.Configuration.DiagnosticMessages = true; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); if (!quiet) lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); else Console.WriteLine("Discovering: {0}", assemblyDisplayName); } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return assemblyElement; }
public static List <IResultXmlTransform> GetAssemblyTransforms(XunitProjectAssembly assembly) { return(instance.GetTransforms(assembly)); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) return null; var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (SerializeTestCases) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); if (longRunningSeconds > 0) resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); if (FailSkips) resultsSink = new DelegatingFailSkipSink(resultsSink); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) ExitCode = 1; } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) Log.LogError(stackLine); e = e.InnerException; } ExitCode = -1; } return assemblyElement; }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool? parallelizeTestCollections, int? maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages ?? false) Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.GetMethodDisplay(), !executionOptions.GetDisableParallelization(), executionOptions.GetMaxParallelThreads()); else Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) { Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
private XElement ExecuteAssembly( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, bool needsXml, [NotNull] XunitFilters filters) { if (assembly == null) { throw new ArgumentNullException(nameof(assembly)); } if (options == null) { throw new ArgumentNullException(nameof(options)); } if (filters == null) { throw new ArgumentNullException(nameof(filters)); } if (_cancel) { return(null); } var internalDiagnosticMessages = options.InternalDiagnosticMessages; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(assembly.AssemblyFilename) || !ValidateFileExists(assembly.ConfigFilename)) { return(null); } ConfigureAssembly(assembly, options); // Setup discovery and execution options with command-line overrides var executionOptions = ConfiguExecutionOptions(assembly, options); var assemblyDisplayName = assembly.GetFileNameWithoutExtension().EmptyIfNull(); void LogAction(MessageHandlerArgs <IDiagnosticMessage> args, string assemblyName) => Console.WriteLine($"{assemblyDisplayName}: {args.Message.Message}"); var diagnosticMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var internalDiagnosticsMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink)) using (var controller = new XunitFrontController( appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => _cancel)) { ExecuteTests(assembly, options, filters, controller, discoverySink, executionOptions, assemblyElement, diagnosticMessageSink); } } catch (Exception ex) { _failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
ExecutionSummary RunTestsInAssembly(AssemblyRunInfo runInfo) { //var assemblyDisplayName = "(unknown assembly)"; //try //{ var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; //assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); //var configuration = runInfo.Configuration; //var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainSupport.Denied; //var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; runInfo.Configuration.PreEnumerateTheories = true; //if (runSettings.DisableAppDomain) // appDomain = AppDomainSupport.Denied; //#if WINDOWS_UAP // // For AppX Apps, use the package location // assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); //#endif //var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); //var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName /*, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink*/)) { // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); var testCases = GetTestCases(runInfo, controller); var discoveredTestCases = GetVsTestCases(assemblyFileName, controller, testCases); var testCasesMap = new Dictionary <string, VsTestCase>(StringComparer.Ordinal); var uniqueTestCases = new List <ITestCase>(); foreach (var discoveredTestCase in discoveredTestCases) { var uniqueID = discoveredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { //logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, discoveredTestCase.VSTestCase); uniqueTestCases.Add(discoveredTestCase.TestCase); } } var recorder = new TrxTestResultRecorder(); using (var resultsSink = new RunnerExecutionSink(testCasesMap, recorder, this.blameWriter, this.resultsWriter)) { controller.RunTests(uniqueTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); return(resultsSink.ExecutionSummary); } //var assembliesElement = new XElement("assemblies"); //var assemblyElement = new XElement("assembly"); //using (var executionSink = new DelegatingXmlCreationSink(new BlameCrashExecutionSink(blameWriter), assemblyElement)) //{ // controller.RunTests(testCases, executionSink, executionOptions); // executionSink.Finished.WaitOne(); // //assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); // assembliesElement.Add(assemblyElement); // assembliesElement.Save(resultsWriter); // return executionSink.ExecutionSummary; //} } //} //catch (Exception ex) //{ // logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); //} }
private void ExecuteTests( [NotNull] XunitProjectAssembly assembly, [NotNull] TestRunOptions options, [NotNull] XunitFilters filters, [NotNull] XunitFrontController controller, [NotNull] TestDiscoverySink discoverySink, [NotNull] ITestFrameworkExecutionOptions executionOptions, [CanBeNull] XElement assemblyElement, [NotNull] DiagnosticMessageSink diagnosticMessageSink) { var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var serialize = options.Serialize; var failSkips = options.FailSkips; var stopOnFail = options.StopOnFail; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); // Discover & filter the tests _reporterMessageHandler.OnMessage( new TestAssemblyDiscoveryStarting( assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { _completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary()); } else { if (serialize) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink( _reporterMessageHandler, () => _cancel, (path, summary) => _completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0) { return; } Console.WriteLine("Canceling due to test failure..."); _cancel = true; } }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, XunitFilters filters, bool designTime, bool listTestCases, IReadOnlyList <string> designTimeFullyQualifiedNames) { if (_cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories when we're not running in Visual Studio if (!designTime) { assembly.Configuration.PreEnumerateTheories = false; } if (diagnosticMessages) { assembly.Configuration.DiagnosticMessages = true; } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly); using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { var includeSourceInformation = designTime && listTestCases; // Discover & filter the tests _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions)); controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); IDictionary <ITestCase, VsTestCase> vsTestCases = null; if (designTime) { vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases); } if (listTestCases) { lock (consoleLock) { if (designTime) { foreach (var testcase in vsTestCases.Values) { _testDiscoverySink?.SendTestFound(testcase); Console.WriteLine(testcase.FullyQualifiedName); } } else { foreach (var testcase in discoveryVisitor.TestCases) { Console.WriteLine(testcase.DisplayName); } } } return(assemblyElement); } IExecutionVisitor resultsVisitor; if (designTime) { resultsVisitor = new DesignTimeExecutionVisitor(_testExecutionSink, vsTestCases, _reporterMessageHandler); } else { resultsVisitor = new XmlAggregateVisitor(_reporterMessageHandler, _completionMessages, assemblyElement, () => _cancel); } IList <ITestCase> filteredTestCases; var testCasesDiscovered = discoveryVisitor.TestCases.Count; if (!designTime || designTimeFullyQualifiedNames.Count == 0) { filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); } else { filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList(); } var testCasesToRun = filteredTestCases.Count; _reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); if (filteredTestCases.Count == 0) { _completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { _reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); _reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { _failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool?parallelizeTestCollections, int?maxThreadCount, ExtendedXunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } lock (consoleLock) { if (assembly.Configuration.DiagnosticMessages ?? false) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), discoveryOptions.GetMethodDisplay(), !executionOptions.GetDisableParallelization(), executionOptions.GetMaxParallelThreads()); } else { Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } } using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); lock (consoleLock) { Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress); var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { lock (consoleLock) { Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); Console.ForegroundColor = ConsoleColor.Gray; } } else { controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault()); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); lock (consoleLock) { if (assembly.Configuration.DiagnosticMessagesOrDefault) { Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})", assemblyDisplayName, discoveryOptions.GetMethodDisplayOrDefault(), !executionOptions.GetDisableParallelizationOrDefault(), executionOptions.GetMaxParallelThreadsOrDefault()); } else if (!quiet) { Console.WriteLine("Discovering: {0}", assemblyDisplayName); } } var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions); discoveryVisitor.Finished.WaitOne(); if (!quiet) { lock (consoleLock) Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)); } var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); if (filteredTestCases.Count == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message); e = e.InnerException; } } return(assemblyElement); }
async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, bool needsXml, _IMessageSink reporterMessageHandler) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); // The normal default is true here, but we want it to be false for us by default if (!assembly.Configuration.PreEnumerateTheories.HasValue) { discoveryOptions.SetPreEnumerateTheories(false); } var assemblyDisplayName = assembly.AssemblyDisplayName; var noColor = assembly.Project.Configuration.NoColorOrDefault; var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; var assemblyInfo = new ReflectionAssemblyInfo(testAssembly); await using var disposalTracker = new DisposalTracker(); var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo); disposalTracker.Add(testFramework); var discoverySink = new TestDiscoverySink(() => cancel); // Discover & filter the tests var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo); var discoveryStarting = new TestAssemblyDiscoveryStarting { AppDomain = AppDomainOption.NotAvailable, Assembly = assembly, DiscoveryOptions = discoveryOptions, ShadowCopy = false }; reporterMessageHandler.OnMessage(discoveryStarting); testDiscoverer.Find(discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; var discoveryFinished = new TestAssemblyDiscoveryFinished { Assembly = assembly, DiscoveryOptions = discoveryOptions, TestCasesDiscovered = testCasesDiscovered, TestCasesToRun = testCasesToRun }; reporterMessageHandler.OnMessage(discoveryFinished); // Run the filtered tests if (testCasesToRun == 0) { testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary()); } else { var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (assembly.Configuration.FailSkipsOrDefault) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var executor = testFramework.GetExecutor(assemblyInfo); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler.OnMessage(executionFinished); if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (assembly.Configuration.InternalDiagnosticMessagesOrDefault) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
void DiscoverTests <TVisitor>(IEnumerable <string> sources, LoggerHelper logger, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null) where TVisitor : IVsDiscoverySink, IDisposable { try { RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { var reporterMessageHandler = new DefaultRunnerReporterWithTypes().CreateMessageHandler(new VisualStudioRunnerLogger(logger)); foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources) { var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath; #if !PLATFORM_DOTNET assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath); #endif var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var configuration = LoadConfiguration(assemblyFileName); var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = configuration.ShadowCopyOrDefault; try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault); using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions); var totalTests = visitor.Finish(); visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } } catch (Exception e) { var ex = e.Unwrap(); var fileNotFound = ex as FileNotFoundException; #if !PLATFORM_DOTNET var fileLoad = ex as FileLoadException; #endif if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (fileNotFound != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !PLATFORM_DOTNET else if (fileLoad != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } } } } catch (Exception e) { logger.LogWarning("Exception discovering tests: {0}", e.Unwrap()); } }
static XunitProject GetSingleAssemblyProject(Dictionary<string, string> transforms, string assemblyFile, string configFile, bool noShadow) { XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFile, ConfigFilename = configFile, ShadowCopy = !noShadow }; foreach (var transform in transforms) assembly.Output.Add(transform.Key, transform.Value); XunitProject project = new XunitProject(); project.AddAssembly(assembly); return project; }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit {CrossPlatform.Version}"; if (NeedsXml) { assembliesElement = new XElement("assemblies"); } switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return(false); } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporter = GetReporter(); if (reporter == null) { return(false); } logger = new MSBuildLogger(Log); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger)); if (!NoLogo) { Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); } var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) { configFileName = null; } var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) { projectAssembly.Configuration.ShadowCopy = shadowCopy; } project.Add(projectAssembly); } if (WorkingFolder != null) { Directory.SetCurrentDirectory(WorkingFolder); } var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) { parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); } if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) { assembliesElement.Add(assemblyElement); } } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) { assembliesElement.Add(assemblyElement); } } } clockTime.Stop(); if (assembliesElement != null) { assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); } if (completionMessages.Count > 0) { reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) { assembliesElement.Save(new FileStream(Xml.GetMetadata("FullPath"), FileMode.OpenOrCreate)); } if (XmlV1 != null) { CrossPlatform.Transform(logger, "XmlV1", "xUnit1.xslt", assembliesElement, XmlV1); } if (Html != null) { CrossPlatform.Transform(logger, "Html", "HTML.xslt", assembliesElement, Html); } if (NUnit != null) { CrossPlatform.Transform(logger, "NUnit", "NUnitXml.xslt", assembliesElement, NUnit); } } // ExitCode is set to 1 for test failures and -1 for Exceptions. return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures)); }
public PerformanceTestDiscoveryVisitor(XunitProjectAssembly assembly, XunitFilters filters, IMessageSink diagnosticMessageSink) { _assembly = assembly; _filters = filters; _diagnosticMessageSink = diagnosticMessageSink; }
public void AddBaseline(XunitProjectAssembly assembly) { _baselineAssemblies.Add(assembly); }
public IEnumerable <AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings, ITestFeedbackProvider channel) { var logger = new XUnitLogger(channel); XunitProject project = new XunitProject(); var runner = settings.Assembly; // Set assembly externally as XUnit screws up the casing logger.SetCurrentAssembly(runner.Assembly); XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = runner.Assembly, ConfigFilename = null, ShadowCopy = false }; project.AddAssembly(assembly); foreach (XunitProjectAssembly asm in project.Assemblies) { using (ExecutorWrapper wrapper = new ExecutorWrapper(asm.AssemblyFilename, asm.ConfigFilename, asm.ShadowCopy)) { try { var xunitRunner = new TestRunner(wrapper, logger); //Run all tests if (runner.Tests.Count() == 0 && runner.Members.Count() == 0 && runner.Namespaces.Count() == 0) { xunitRunner.RunAssembly(); } //Run tests if (runner.Tests.Count() > 0) { foreach (var test in runner.Tests) { xunitRunner.RunTest(test.Substring(0, test.LastIndexOf(".")), test.Substring(test.LastIndexOf(".") + 1, test.Length - (test.LastIndexOf(".") + 1))); } } //Run members if (runner.Members.Count() > 0) { foreach (var member in runner.Members) { xunitRunner.RunClass(member); } } //Run namespaces if (runner.Namespaces.Count() > 0) { var loadedAssembly = Assembly.LoadFrom(runner.Assembly); var types = loadedAssembly.GetExportedTypes(); loadedAssembly = null; foreach (var ns in runner.Namespaces) { foreach (Type type in types) { if (ns == null || type.Namespace == ns) { xunitRunner.RunClass(type.FullName); } } } } } catch (ArgumentException ex) { Console.WriteLine(ex.Message); } } } return(logger.Results); }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}"; if (NeedsXml) assembliesElement = new XElement("assemblies"); switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return false; } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporter = GetReporter(); if (reporter == null) return false; logger = new MSBuildLogger(Log); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger)); if (!NoLogo) Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) configFileName = null; var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) projectAssembly.Configuration.ShadowCopy = shadowCopy; project.Add(projectAssembly); } if (WorkingFolder != null) Directory.SetCurrentDirectory(WorkingFolder); var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) assembliesElement.Add(assemblyElement); } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) assembliesElement.Add(assemblyElement); } } clockTime.Stop(); if (assembliesElement != null) assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); if (completionMessages.Count > 0) reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) assembliesElement.Save(Xml.GetMetadata("FullPath")); if (XmlV1 != null) Transform("xUnit1.xslt", assembliesElement, XmlV1); if (Html != null) Transform("HTML.xslt", assembliesElement, Html); if (NUnit != null) Transform("NUnitXml.xslt", assembliesElement, NUnit); } // ExitCode is set to 1 for test failures and -1 for Exceptions. return ExitCode == 0 || (ExitCode == 1 && IgnoreFailures); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, RunSettings runSettings, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (runSettings.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { try { var serializations = runInfo.TestCases .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)) .ToList(); if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}", serializations.Count, Environment.NewLine, string.Join(Environment.NewLine, serializations.Select(x => $" {x}"))); } var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations); if (deserializedTestCasesByUniqueId == null) { logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize"); } else { for (int idx = 0; idx < runInfo.TestCases.Count; ++idx) { try { var kvp = deserializedTestCasesByUniqueId[idx]; var vsTestCase = runInfo.TestCases[idx]; if (kvp.Value == null) { logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key); } else { testCasesMap.Add(kvp.Key, vsTestCase); testCases.Add(kvp.Value); } } catch (Exception ex) { logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex); } } } } catch (Exception ex) { logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex); } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (runSettings.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); if (TeamCity) { Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead."); return false; } if (Verbose) { Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead."); return false; } XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}"; if (NeedsXml) assembliesElement = new XElement("assemblies"); switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return false; } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporters = GetAvailableRunnerReporters(); var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled); if (reporter == null && !string.IsNullOrWhiteSpace(Reporter)) { reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase)); if (reporter == null) { var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList(); if (switchableReporters.Count == 0) Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter); else Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters)); return false; } } if (reporter == null) reporter = new DefaultRunnerReporter(); logger = new MSBuildLogger(Log); reporterMessageHandler = reporter.CreateMessageHandler(logger); if (!NoLogo) Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) configFileName = null; var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) projectAssembly.Configuration.ShadowCopy = shadowCopy; project.Add(projectAssembly); } if (WorkingFolder != null) Directory.SetCurrentDirectory(WorkingFolder); var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) assembliesElement.Add(assemblyElement); } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) assembliesElement.Add(assemblyElement); } } clockTime.Stop(); if (completionMessages.Count > 0) reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) assembliesElement.Save(Xml.GetMetadata("FullPath")); if (XmlV1 != null) Transform("xUnit1.xslt", assembliesElement, XmlV1); if (Html != null) Transform("HTML.xslt", assembliesElement, Html); if (NUnit != null) Transform("NUnitXml.xslt", assembliesElement, NUnit); } // ExitCode is set to 1 for test failures and -1 for Exceptions. return ExitCode == 0 || (ExitCode == 1 && IgnoreFailures); }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); if (TeamCity) { Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead."); return(false); } if (Verbose) { Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead."); return(false); } XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}"; if (NeedsXml) { assembliesElement = new XElement("assemblies"); } switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return(false); } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporters = GetAvailableRunnerReporters(); var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled); if (reporter == null && !string.IsNullOrWhiteSpace(Reporter)) { reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase)); if (reporter == null) { var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList(); if (switchableReporters.Count == 0) { Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter); } else { Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters)); } return(false); } } if (reporter == null) { reporter = new DefaultRunnerReporterWithTypes(); } logger = new MSBuildLogger(Log); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger)); if (!NoLogo) { Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); } var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) { configFileName = null; } var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) { projectAssembly.Configuration.ShadowCopy = shadowCopy; } project.Add(projectAssembly); } if (WorkingFolder != null) { Directory.SetCurrentDirectory(WorkingFolder); } var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) { parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); } if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) { assembliesElement.Add(assemblyElement); } } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) { assembliesElement.Add(assemblyElement); } } } clockTime.Stop(); if (completionMessages.Count > 0) { reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) { assembliesElement.Save(Xml.GetMetadata("FullPath")); } if (XmlV1 != null) { Transform("xUnit1.xslt", assembliesElement, XmlV1); } if (Html != null) { Transform("HTML.xslt", assembliesElement, Html); } if (NUnit != null) { Transform("NUnitXml.xslt", assembliesElement, NUnit); } } // ExitCode is set to 1 for test failures and -1 for Exceptions. return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures)); }
static XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool serialize, bool needsXml, bool? parallelizeTestCollections, int? maxThreadCount, bool diagnosticMessages, bool noColor, bool noAppDomain, bool failSkips, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; if (noAppDomain) assembly.Configuration.AppDomain = AppDomainSupport.Denied; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue) executionOptions.SetMaxParallelThreads(maxThreadCount); if (parallelizeTestCollections.HasValue) executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions)); controller.Find(false, discoveryVisitor, discoveryOptions); discoveryVisitor.Finished.WaitOne(); var testCasesDiscovered = discoveryVisitor.TestCases.Count; var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); else { if (serialize) filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel); if (failSkips) resultsVisitor = new FailSkipVisitor(resultsVisitor); controller.RunTests(filteredTestCases, resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary)); } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); e = e.InnerException; } } return assemblyElement; }
XElement ExecuteAssembly(object consoleLock, XunitProjectAssembly assembly, bool needsXml, bool?parallelizeTestCollections, int?maxThreadCount, bool diagnosticMessages, bool noColor, bool failSkips, bool stopOnFail, XunitFilters filters, bool internalDiagnosticMessages) { if (cancel) { return(null); } var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.ConfigFilename)) { return(null); } // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= diagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages; // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); executionOptions.SetStopOnTestFail(stopOnFail); if (maxThreadCount.HasValue) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault()); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor); var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename); var entryAssembly = Assembly.GetEntryAssembly(); var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly); var discoverySink = new TestDiscoverySink(() => cancel); using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions)); testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); } var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { executionSummary = new ExecutionSummary(); } else { reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink)); } if (failSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using var executor = testFramework.GetExecutor(entryAssembly.GetName()); executor.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); executionSummary = resultsSink.ExecutionSummary; reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0) { Console.WriteLine("Canceling due to test failure..."); cancel = true; } } } catch (Exception ex) { failed = true; var e = ex; while (e != null) { Console.WriteLine($"{e.GetType().FullName}: {e.Message}"); if (internalDiagnosticMessages) { Console.WriteLine(e.StackTrace); } e = e.InnerException; } } return(assemblyElement); }
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters) { if (cancel) return null; var assemblyElement = needsXml ? new XElement("assembly") : null; try { if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename)) return null; using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy)) using (var discoveryVisitor = new TestDiscoveryVisitor()) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions()); discoveryVisitor.Finished.WaitOne(); var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount }; var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent); controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions); resultsVisitor.Finished.WaitOne(); } } catch (Exception ex) { Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message); failed = true; } return assemblyElement; }
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework, LoggerHelper logger, TestPlatformContext testPlatformContext, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete, string assemblyFileName, bool shadowCopy, TestAssemblyConfiguration configuration) where TVisitor : IVsDiscoverySink, IDisposable { var reporterMessageHandler = GetRunnerReporter(new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger)); var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); try { if (cancelled) { return(false); } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions)); framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions); var totalTests = visitor.Finish(); visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor); reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } catch (Exception e) { var ex = e.Unwrap(); if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (ex is FileNotFoundException fileNotFound) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !WINDOWS_UAP else if (ex is FileLoadException fileLoad) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } return(true); }