/// <summary>
/// Runs the tests with the given options
/// </summary>
/// <param name="options">The test run options</param>
/// <returns>The number of failed tests, or -1 if cancelled</returns>
public int EntryPoint([NotNull] TestRunOptions options)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
try
{
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
var defaultDirectory = Directory.GetCurrentDirectory();
if (!defaultDirectory.EndsWith(new string(new[] { Path.DirectorySeparatorChar }), StringComparison.Ordinal))
{
}
/*
* if (options.Debug)
* {
* Debugger.Launch();
* }*/
_reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(options.Reporter.CreateMessageHandler(_logger));
if (!options.NoLogo)
{
PrintHeader();
}
var failCount = RunProject(options);
if (_cancel)
{
return(-1);
}
return(failCount);
}
catch (ArgumentException ex)
{
Console.WriteLine($"error: {ex.Message}");
Console.WriteLine(ex.StackTrace);
throw;
}
catch (BadImageFormatException ex)
{
Console.WriteLine(ex.Message);
throw;
}
}
void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, Func <List <AssemblyRunInfo> > testCaseAccessor)
{
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);
try
{
RemotingUtility.CleanUpRegisteredChannels();
#if NET35 || NETCOREAPP1_0
// Reads settings like disabling appdomains, parallel etc.
// Do this first before invoking any thing else to ensure correct settings for the run
RunSettingsHelper.ReadRunSettings(runContext?.RunSettings?.SettingsXml);
#endif
cancelled = false;
var assemblies = testCaseAccessor();
var parallelizeAssemblies = !RunSettingsHelper.DisableParallelization && assemblies.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault);
var reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(GetRunnerReporter(assemblies.Select(ari => ari.AssemblyFileName))
.CreateMessageHandler(new VisualStudioRunnerLogger(logger)));
using (AssemblyHelper.SubscribeResolve())
if (parallelizeAssemblies)
{
assemblies
.Select(runInfo => RunTestsInAssemblyAsync(runContext, frameworkHandle, logger, reporterMessageHandler, runInfo))
.ToList()
.ForEach(@event => @event.WaitOne());
}
else
{
assemblies
.ForEach(runInfo => RunTestsInAssembly(runContext, frameworkHandle, logger, reporterMessageHandler, runInfo));
}
}
catch (Exception ex)
{
logger.LogError("Catastrophic failure: {0}", ex);
}
}
void RunTests(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
Func <List <AssemblyRunInfo> > getRunInfos)
{
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);
try
{
RemotingUtility.CleanUpRegisteredChannels();
cancelled = false;
var runInfos = getRunInfos();
var parallelizeAssemblies = !RunSettingsHelper.DisableParallelization && runInfos.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault);
var reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(GetRunnerReporter(runInfos.Select(ari => ari.AssemblyFileName))
.CreateMessageHandler(new VisualStudioRunnerLogger(logger)));
#if NET452
using (AssemblyHelper.SubscribeResolve())
#endif
{
if (parallelizeAssemblies)
{
runInfos
.Select(runInfo => RunTestsInAssemblyAsync(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo))
.ToList()
.ForEach(@event => @event.WaitOne());
}
else
{
runInfos
.ForEach(runInfo => RunTestsInAssembly(runContext, frameworkHandle, logger, testPlatformContext, reporterMessageHandler, runInfo));
}
}
}
catch (Exception ex)
{
logger.LogError("Catastrophic failure: {0}", ex);
}
}
public static int Main(string[] args)
{
try
{
var reporters = GetAvailableRunnerReporters();
if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help")
{
PrintHeader();
PrintUsage(reporters);
return(2);
}
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
Console.CancelKeyPress += (sender, e) =>
{
if (!cancel)
{
Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)");
cancel = true;
e.Cancel = true;
}
};
var defaultDirectory = Directory.GetCurrentDirectory();
if (!defaultDirectory.EndsWith(new string(new[] { Path.DirectorySeparatorChar }), StringComparison.Ordinal))
{
defaultDirectory += Path.DirectorySeparatorChar;
}
var commandLine = CommandLine.Parse(reporters, args);
if (commandLine.Debug)
{
Debugger.Launch();
}
logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
if (!commandLine.NoLogo)
{
PrintHeader();
}
var failCount = RunProject(commandLine.Project, commandLine.Serialize, commandLine.ParallelizeAssemblies,
commandLine.ParallelizeTestCollections, commandLine.MaxParallelThreads,
commandLine.DiagnosticMessages, commandLine.NoColor, commandLine.NoAppDomain,
commandLine.FailSkips, commandLine.InternalDiagnosticMessages);
if (commandLine.Wait)
{
Console.WriteLine();
Console.Write("Press any key to continue...");
Console.ReadKey();
Console.WriteLine();
}
return(failCount > 0 ? 1 : 0);
}
catch (ArgumentException ex)
{
Console.WriteLine($"error: {ex.Message}");
return(3);
}
catch (BadImageFormatException ex)
{
Console.WriteLine(ex.Message);
return(4);
}
finally
{
Console.ResetColor();
}
}
public int Run(string[] args)
{
try
{
var reporters = GetAvailableRunnerReporters();
if (args.Length == 0 || args.Any(arg => arg == "-?" || arg == "-help"))
{
PrintHeader();
PrintUsage(reporters);
return(2);
}
#if !NETCOREAPP1_0
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
#endif
var commandLine = CommandLine.Parse(reporters, args);
#if !NETCOREAPP1_0
if (commandLine.Debug)
{
Debugger.Launch();
}
#else
if (commandLine.Debug)
{
Console.WriteLine("Debug support is not available in .NET Core.");
return(2);
}
#endif
logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
if (!commandLine.NoLogo)
{
PrintHeader();
}
var testsToRun = commandLine.DesignTimeTestUniqueNames;
if (commandLine.Port.HasValue)
{
socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
var ipEndPoint = new IPEndPoint(IPAddress.Loopback, commandLine.Port.Value);
socket.Connect(ipEndPoint);
var networkStream = new NetworkStream(socket);
UseTestSinksWithSockets(networkStream);
if (commandLine.WaitCommand)
{
var reader = new BinaryReader(networkStream);
testExecutionSink.SendWaitingCommand();
var rawMessage = reader.ReadString();
var message = JsonConvert.DeserializeObject <Message>(rawMessage);
testsToRun = message.Payload.ToObject <RunTestsMessage>().Tests;
}
}
else
{
UseTestSinksWithStandardOutputStreams();
}
var failCount = RunProject(commandLine.Project, commandLine.ParallelizeAssemblies, commandLine.ParallelizeTestCollections,
commandLine.MaxParallelThreads, commandLine.DiagnosticMessages, commandLine.NoColor,
commandLine.AppDomains, commandLine.FailSkips, commandLine.DesignTime, commandLine.List,
testsToRun);
if (commandLine.Wait)
{
WaitForInput();
}
return(failCount > 0 ? 1 : 0);
}
catch (ArgumentException ex)
{
Console.WriteLine("error: {0}", ex.Message);
return(3);
}
catch (BadImageFormatException ex)
{
Console.WriteLine("{0}", ex.Message);
return(4);
}
finally
{
Console.ResetColor();
}
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
AppDomainSupport?appDomains,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary), logger);
if (assemblyElement != null)
{
logger.LogMessage($"Adding DelegatingXmlCreationSink");
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement, logger);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
logger.LogMessage($"ExecuteAssembly: {assemblyElement}");
return(assemblyElement);
}
public int EntryPoint(string[] args)
{
commandLine = CommandLine.Parse(args);
try
{
var reporters = GetAvailableRunnerReporters();
if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help")
{
PrintHeader();
PrintUsage(reporters);
return(2);
}
if (commandLine.Project.Assemblies.Count == 0)
{
throw new ArgumentException("must specify at least one assembly");
}
#if NET452
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
#endif
Console.CancelKeyPress += (sender, e) =>
{
if (!cancel)
{
Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)");
cancel = true;
e.Cancel = true;
}
};
var defaultDirectory = Directory.GetCurrentDirectory();
if (!defaultDirectory.EndsWith(new string(new[] { Path.DirectorySeparatorChar }), StringComparison.Ordinal))
{
defaultDirectory += Path.DirectorySeparatorChar;
}
var reporter = commandLine.ChooseReporter(reporters);
#if DEBUG
if (commandLine.Pause)
{
Console.Write("Press any key to start execution...");
Console.ReadKey(true);
Console.WriteLine();
}
#endif
if (commandLine.Debug)
{
Debugger.Launch();
}
logger = new ConsoleRunnerLogger(!commandLine.NoColor, consoleLock);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!commandLine.NoLogo)
{
PrintHeader();
}
var failCount = RunProject(commandLine.Project, commandLine.Serialize, commandLine.ParallelizeAssemblies,
commandLine.ParallelizeTestCollections, commandLine.MaxParallelThreads,
commandLine.DiagnosticMessages, commandLine.NoColor, commandLine.AppDomains,
commandLine.FailSkips, commandLine.StopOnFail, commandLine.InternalDiagnosticMessages);
if (cancel)
{
return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C
}
if (commandLine.Wait)
{
Console.WriteLine();
Console.Write("Press any key to continue...");
Console.ReadKey();
Console.WriteLine();
}
return(failCount > 0 ? 1 : 0);
}
catch (Exception ex)
{
if (!commandLine.NoColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.Red);
}
Console.WriteLine($"error: {ex.Message}");
if (commandLine.InternalDiagnosticMessages)
{
if (!commandLine.NoColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.DarkGray);
}
Console.WriteLine(ex.StackTrace);
}
return(ex is ArgumentException ? 3 : 4);
}
finally
{
if (!commandLine.NoColor)
{
ConsoleHelper.ResetColor();
}
}
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporter = GetReporter();
if (reporter == null)
{
return(false);
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
assembliesElement.Save(Xml.GetMetadata("FullPath"));
}
if (XmlV1 != null)
{
Transform("xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
Transform("HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit {CrossPlatform.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
var appDomains = default(AppDomainSupport?);
switch (AppDomains?.ToLowerInvariant()) // Using ToLowerInvariant() here for back compat for when this was a boolean
{
case null:
break;
case "ifavailable":
appDomains = AppDomainSupport.IfAvailable;
break;
case "true":
case "required":
appDomains = AppDomainSupport.Required;
break;
case "false":
case "denied":
appDomains = AppDomainSupport.Denied;
break;
default:
Log.LogError("AppDomains value '{0}' is invalid: must be 'ifavailable', 'required', or 'denied'", AppDomains);
return(false);
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(Log, InternalDiagnosticMessages);
using (AssemblyHelper.SubscribeResolveForAssembly(typeof(xunit), internalDiagnosticsMessageSink))
{
var reporter = GetReporter();
if (reporter == null)
{
return(false);
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
var versionAttribute = typeof(xunit).GetTypeInfo().Assembly.GetCustomAttribute <AssemblyInformationalVersionAttribute>();
Log.LogMessage(MessageImportance.High, $"xUnit.net MSBuild Runner v{versionAttribute.InformationalVersion} ({environment})");
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly, appDomains)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly, appDomains);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
using (var xmlStream = new FileStream(Xml.GetMetadata("FullPath"), FileMode.OpenOrCreate, FileAccess.Write))
assembliesElement.Save(xmlStream);
}
if (XmlV1 != null)
{
CrossPlatform.Transform(logger, "XmlV1", "xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
CrossPlatform.Transform(logger, "Html", "HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
CrossPlatform.Transform(logger, "NUnit", "NUnitXml.xslt", assembliesElement, NUnit);
}
if (JUnit != null)
{
CrossPlatform.Transform(logger, "JUnit", "JUnitXml.xslt", assembliesElement, JUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor);
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename);
var entryAssembly = Assembly.GetEntryAssembly();
var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly);
var discoverySink = new TestDiscoverySink(() => cancel);
using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions));
testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
}
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using var executor = testFramework.GetExecutor(entryAssembly.GetName());
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
executionSummary = resultsSink.ExecutionSummary;
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public int EntryPoint(string[] args)
{
var assemblyUnderTest = Assembly.GetEntryAssembly();
commandLine = CommandLine.Parse(assemblyUnderTest.GetLocalCodeBase(), args);
try
{
var reporters = GetAvailableRunnerReporters();
if (args.Length > 0 && (args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help"))
{
PrintHeader();
PrintUsage(reporters);
return(2);
}
// TODO: What is the portable version of this?
#if NETFRAMEWORK
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
#endif
Console.CancelKeyPress += (sender, e) =>
{
if (!cancel)
{
Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)");
cancel = true;
e.Cancel = true;
}
};
var defaultDirectory = Directory.GetCurrentDirectory();
if (!defaultDirectory.EndsWith(new string(new[] { Path.DirectorySeparatorChar }), StringComparison.Ordinal))
{
defaultDirectory += Path.DirectorySeparatorChar;
}
var reporter = commandLine.ChooseReporter(reporters);
if (commandLine.Pause)
{
Console.Write("Press any key to start execution...");
Console.ReadKey(true);
Console.WriteLine();
}
if (commandLine.Debug)
{
Debugger.Launch();
}
logger = new ConsoleRunnerLogger(!commandLine.NoColor, consoleLock);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!commandLine.NoLogo)
{
PrintHeader();
}
// TODO: Will need more things here, like filters and output transform, when they're back
var failCount = RunProject(commandLine.Project,
commandLine.ParallelizeTestCollections, commandLine.MaxParallelThreads,
commandLine.DiagnosticMessages, commandLine.NoColor,
commandLine.FailSkips, commandLine.StopOnFail, commandLine.InternalDiagnosticMessages);
if (cancel)
{
return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C
}
if (commandLine.Wait)
{
Console.WriteLine();
Console.Write("Press any key to continue...");
Console.ReadKey();
Console.WriteLine();
}
return(failCount > 0 ? 1 : 0);
}
catch (Exception ex)
{
if (!commandLine.NoColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.Red);
}
Console.WriteLine($"error: {ex.Message}");
if (commandLine.InternalDiagnosticMessages)
{
if (!commandLine.NoColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.DarkGray);
}
Console.WriteLine(ex.StackTrace);
}
return(ex is ArgumentException ? 3 : 4);
}
finally
{
if (!commandLine.NoColor)
{
ConsoleHelper.ResetColor();
}
}
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
if (TeamCity)
{
Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead.");
return(false);
}
if (Verbose)
{
Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead.");
return(false);
}
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporters = GetAvailableRunnerReporters();
var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled);
if (reporter == null && !string.IsNullOrWhiteSpace(Reporter))
{
reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase));
if (reporter == null)
{
var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList();
if (switchableReporters.Count == 0)
{
Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter);
}
else
{
Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters));
}
return(false);
}
}
if (reporter == null)
{
reporter = new DefaultRunnerReporterWithTypes();
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
assembliesElement.Save(Xml.GetMetadata("FullPath"));
}
if (XmlV1 != null)
{
Transform("xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
Transform("HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
/// <summary>
/// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description.
/// </summary>
/// <param name="typeName"></param>
/// <param name="testReportName"></param>
private void ExecuteTests(string typeName = null, string testReportName = null)
{
ExecutionSummary executionSummary = null;
XunitProjectAssembly assembly = new XunitProjectAssembly
{
AssemblyFilename = assemblyLocation,
ConfigFilename = null
};
XElement assembliesElement = new XElement("assemblies");
XElement assemblyElement = new XElement("assembly");
// Logger
var verboserReporter = new XunitReporters.VerboseReporter();
IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true);
IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger));
// Option setup
ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null);
ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null);
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel);
executionOptions.SetDiagnosticMessages(true);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var clockTime = Stopwatch.StartNew();
bool cancel = false;
using (var controller = new XunitFrontController(
appDomainSupport: AppDomainSupport.Denied,
assemblyFileName: assembly.AssemblyFilename,
configFileName: null,
shadowCopy: shadowCopy,
shadowCopyFolder: null,
sourceInformationProvider: null,
diagnosticMessageSink: null))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(
assembly: assembly,
appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy: shadowCopy,
discoveryOptions: discoveryOptions));
if (typeName != null)
{
controller.Find(typeName, false, discoverySink, discoveryOptions);
}
else
{
controller.Find(false, discoverySink, discoveryOptions);
}
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; });
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
if (executionSummary != null)
{
Console.WriteLine();
KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary);
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > {
kvpExecutionSummary
}));
if (testReportName != null)
{
// Create folder if it does not exist
FileInfo fi = new FileInfo(testReportName);
DirectoryInfo directory = fi.Directory;
if (!directory.Exists)
{
Directory.CreateDirectory(directory.FullName);
}
CreateXmlTestReport(assembliesElement, fi);
CreateHtmlTestReport(assembliesElement, fi);
}
Console.WriteLine();
Console.WriteLine();
}
}
public static void RunTests(Options o)
{
var nullMessage = new Xunit.NullMessageSink();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false))
{
var tv = new TestDiscoverySink();
var excludeTestCaseSet = new TestDiscoverySink();
c.Find(true, tv, discoveryOptions);
tv.Finished.WaitOne();
foreach (var tc in tv.TestCases)
{
var method = tc.TestMethod.Method;
var attributes = method.GetCustomAttributes(typeof(FactAttribute));
foreach (ReflectionAttributeInfo at in attributes)
{
var result = at.GetNamedArgument<string>("Skip");
if (result != null)
{
Console.WriteLine("SKIPPY! {0} because {1}", method, result);
}
if (o.TestType != TestType.All)
{
if (!at.ToString().EndsWith(o.TestType.ToString()))
{
excludeTestCaseSet.TestCases.Add(tc);
}
}
}
}
foreach (var tc in excludeTestCaseSet.TestCases)
{
tv.TestCases.Remove(tc);
}
Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count);
//core execution Sink
int testCaseCount = tv.TestCases.Count;
Stream file = new FileStream(".\\result.xml", FileMode.Create);
int totalResult = 0;
int totalErrors = 0;
int totalFailed = 0;
int totalSkipped = 0;
for (int i = 0; i < testCaseCount; i++)
{
IExecutionSink resultsSink;
ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
IMessageSinkWithTypes reporterMessageHandler;
var reporters = GetAvailableRunnerReporters();
var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll");
IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
var xmlElement = new XElement("TestResult");
resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true);
var message = new Xunit.NullMessageSink();
var executionOptions = TestFrameworkOptions.ForExecution();
c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions);
resultsSink.Finished.WaitOne(o.TimeOut);
tv.TestCases.RemoveAt(0);
totalResult++;
totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors;
totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed;
totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped;
xmlElement.Save(file);
file.Flush();
}
file.Dispose();
Console.WriteLine("Total tests: " + totalResult);
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("Error tests: " + totalErrors);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed tests: " + totalFailed);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Skipped tests: " + totalSkipped);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped));
Console.ResetColor();
}
}
