public static DiagnosticMessageSink ForInternalDiagnostics(TaskLoggingHelper log, bool showDiagnostics) { var result = new DiagnosticMessageSink(); if (showDiagnostics) { result.Diagnostics.DiagnosticMessageEvent += args => log.LogMessage("{0}", args.Message.Message); } return(result); }
public static DiagnosticMessageSink ForDiagnostics(TaskLoggingHelper log, string assemblyDisplayName, bool showDiagnostics) { var result = new DiagnosticMessageSink(); if (showDiagnostics) { result.Diagnostics.DiagnosticMessageEvent += args => log.LogWarning("{0}: {1}", assemblyDisplayName, args.Message.Message); } return(result); }
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { // Turn off pre-enumeration of theories, since there is no theory selection UI in this runner assembly.Configuration.PreEnumerateTheories = false; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied; } // Setup discovery and execution options with command-line overrides var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename); var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) using (var discoverySink = new TestDiscoverySink(() => cancel)) { // Discover & filter the tests reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); controller.Find(false, discoverySink, discoveryOptions); discoverySink.Finished.WaitOne(); var testCasesDiscovered = discoverySink.TestCases.Count; var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList(); var testCasesToRun = filteredTestCases.Count; reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun)); // Run the filtered tests if (testCasesToRun == 0) { completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary()); } else { if (SerializeTestCases) { filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList(); } IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); controller.RunTests(filteredTestCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); if (resultsSink.ExecutionSummary.Failed != 0) { ExitCode = 1; } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
protected virtual async ValueTask <XElement?> ExecuteAssembly( XunitProjectAssembly assembly, AppDomainSupport?appDomains) { if (cancel) { return(null); } var assemblyElement = NeedsXml ? new XElement("assembly") : null; try { assembly.Configuration.PreEnumerateTheories = PreEnumerateTheories; assembly.Configuration.DiagnosticMessages |= DiagnosticMessages; assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages; if (appDomains.HasValue) { assembly.Configuration.AppDomain = appDomains; } // Setup discovery and execution options with command-line overrides var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration); var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration); if (maxThreadCount.HasValue && maxThreadCount.Value > -1) { executionOptions.SetMaxParallelThreads(maxThreadCount); } if (parallelizeTestCollections.HasValue) { executionOptions.SetDisableParallelization(!parallelizeTestCollections); } if (stopOnFail.HasValue) { executionOptions.SetStopOnTestFail(stopOnFail); } var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename) !; var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault); var appDomainSupport = assembly.Configuration.AppDomainOrDefault; var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink); var executionStarting = new TestAssemblyExecutionStarting { Assembly = assembly, ExecutionOptions = executionOptions }; reporterMessageHandler !.OnMessage(executionStarting); IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler !, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary)); if (assemblyElement != null) { resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement); } if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink); } if (FailSkips) { resultsSink = new DelegatingFailSkipSink(resultsSink); } using (resultsSink) { var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters); controller.FindAndRun(resultsSink, settings); resultsSink.Finished.WaitOne(); var executionFinished = new TestAssemblyExecutionFinished { Assembly = assembly, ExecutionOptions = executionOptions, ExecutionSummary = resultsSink.ExecutionSummary }; reporterMessageHandler !.OnMessage(executionFinished); if (resultsSink.ExecutionSummary.Failed != 0 || resultsSink.ExecutionSummary.Errors != 0) { ExitCode = 1; if (stopOnFail == true) { Log.LogMessage(MessageImportance.High, "Canceling due to test failure..."); Cancel(); } } } } catch (Exception ex) { var e = ex; while (e != null) { Log.LogError("{0}: {1}", e.GetType().FullName, e.Message); if (e.StackTrace != null) { foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { Log.LogError(stackLine); } } e = e.InnerException; } ExitCode = -1; } return(assemblyElement); }
public override bool Execute() { Guard.ArgumentNotNull(nameof(Assemblies), Assemblies); RemotingUtility.CleanUpRegisteredChannels(); XElement?assembliesElement = null; if (NeedsXml) { assembliesElement = new XElement("assemblies"); } var appDomains = default(AppDomainSupport?); switch (AppDomains?.ToLowerInvariant()) { case null: break; case "ifavailable": appDomains = AppDomainSupport.IfAvailable; break; case "true": case "required": appDomains = AppDomainSupport.Required; break; case "false": case "denied": appDomains = AppDomainSupport.Denied; break; default: Log.LogError("AppDomains value '{0}' is invalid: must be 'ifavailable', 'required', or 'denied'", AppDomains); return(false); } switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return(false); } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(Log, InternalDiagnosticMessages); using (AssemblyHelper.SubscribeResolveForAssembly(typeof(xunit), internalDiagnosticsMessageSink)) { var reporter = GetReporter(); if (reporter == null) { return(false); } try { logger = new MSBuildLogger(Log); reporterMessageHandler = reporter.CreateMessageHandler(logger, internalDiagnosticsMessageSink).GetAwaiter().GetResult(); if (!NoLogo) { Log.LogMessage(MessageImportance.High, $"xUnit.net v3 MSBuild Runner v{ThisAssembly.AssemblyInformationalVersion} ({IntPtr.Size * 8}-bit {RuntimeInformation.FrameworkDescription})"); } var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) { configFileName = null; } var targetFramework = AssemblyUtility.GetTargetFramework(assemblyFileName); var projectAssembly = new XunitProjectAssembly(project) { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName, TargetFramework = targetFramework }; ConfigReader.Load(projectAssembly.Configuration, assemblyFileName, configFileName); if (shadowCopy.HasValue) { projectAssembly.Configuration.ShadowCopy = shadowCopy; } project.Add(projectAssembly); } if (WorkingFolder != null) { Directory.SetCurrentDirectory(WorkingFolder); } var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) { parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); } if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly, appDomains).AsTask())); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.WhereNotNull()) { assembliesElement !.Add(assemblyElement); } } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly, appDomains); if (assemblyElement != null) { assembliesElement !.Add(assemblyElement); } } } clockTime.Stop(); if (assembliesElement != null) { assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); } if (completionMessages.Count > 0) { var summaries = new TestExecutionSummaries { ElapsedClockTime = clockTime.Elapsed }; foreach (var completionMessage in completionMessages.OrderBy(kvp => kvp.Key)) { summaries.Add(completionMessage.Key, completionMessage.Value); } reporterMessageHandler.OnMessage(summaries); } } finally { reporter.DisposeAsync().GetAwaiter().GetResult(); } } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml && assembliesElement != null) { if (Xml != null) { TransformFactory.Transform("xml", assembliesElement, Xml.GetMetadata("FullPath")); } if (XmlV1 != null) { TransformFactory.Transform("xmlv1", assembliesElement, XmlV1.GetMetadata("FullPath")); } if (Html != null) { TransformFactory.Transform("html", assembliesElement, Html.GetMetadata("FullPath")); } if (NUnit != null) { TransformFactory.Transform("nunit", assembliesElement, NUnit.GetMetadata("FullPath")); } if (JUnit != null) { TransformFactory.Transform("junit", assembliesElement, JUnit.GetMetadata("FullPath")); } } // ExitCode is set to 1 for test failures and -1 for Exceptions. return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures)); }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit {CrossPlatform.Version}"; if (NeedsXml) { assembliesElement = new XElement("assemblies"); } var appDomains = default(AppDomainSupport?); switch (AppDomains?.ToLowerInvariant()) // Using ToLowerInvariant() here for back compat for when this was a boolean { case null: break; case "ifavailable": appDomains = AppDomainSupport.IfAvailable; break; case "true": case "required": appDomains = AppDomainSupport.Required; break; case "false": case "denied": appDomains = AppDomainSupport.Denied; break; default: Log.LogError("AppDomains value '{0}' is invalid: must be 'ifavailable', 'required', or 'denied'", AppDomains); return(false); } switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return(false); } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(Log, InternalDiagnosticMessages); using (AssemblyHelper.SubscribeResolveForAssembly(typeof(xunit), internalDiagnosticsMessageSink)) { var reporter = GetReporter(); if (reporter == null) { return(false); } logger = new MSBuildLogger(Log); reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger)); if (!NoLogo) { var versionAttribute = typeof(xunit).GetTypeInfo().Assembly.GetCustomAttribute <AssemblyInformationalVersionAttribute>(); Log.LogMessage(MessageImportance.High, $"xUnit.net MSBuild Runner v{versionAttribute.InformationalVersion} ({environment})"); } var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) { configFileName = null; } var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) { projectAssembly.Configuration.ShadowCopy = shadowCopy; } project.Add(projectAssembly); } if (WorkingFolder != null) { Directory.SetCurrentDirectory(WorkingFolder); } var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) { parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); } if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly, appDomains))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) { assembliesElement.Add(assemblyElement); } } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly, appDomains); if (assemblyElement != null) { assembliesElement.Add(assemblyElement); } } } clockTime.Stop(); if (assembliesElement != null) { assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture))); } if (completionMessages.Count > 0) { reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) { using (var xmlStream = new FileStream(Xml.GetMetadata("FullPath"), FileMode.OpenOrCreate, FileAccess.Write)) assembliesElement.Save(xmlStream); } if (XmlV1 != null) { CrossPlatform.Transform(logger, "XmlV1", "xUnit1.xslt", assembliesElement, XmlV1); } if (Html != null) { CrossPlatform.Transform(logger, "Html", "HTML.xslt", assembliesElement, Html); } if (NUnit != null) { CrossPlatform.Transform(logger, "NUnit", "NUnitXml.xslt", assembliesElement, NUnit); } if (JUnit != null) { CrossPlatform.Transform(logger, "JUnit", "JUnitXml.xslt", assembliesElement, JUnit); } } // ExitCode is set to 1 for test failures and -1 for Exceptions. return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures)); }