int RunProject(XunitProject project,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool failSkips,
bool stopOnFail,
bool internalDiagnosticMessages)
{
XElement assembliesElement = null;
var clockTime = Stopwatch.StartNew();
var xmlTransformers = TransformFactory.GetXmlTransformers(project);
var needsXml = xmlTransformers.Count > 0;
if (needsXml)
{
assembliesElement = new XElement("assemblies");
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
var assembly = project.Assemblies.Single();
var assemblyElement = ExecuteAssembly(consoleLock, assembly, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, failSkips, stopOnFail, project.Filters, internalDiagnosticMessages);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
var summary = new KeyValuePair <string, ExecutionSummary>(Path.GetFileNameWithoutExtension(assembly.AssemblyFilename), executionSummary);
var summaries = new List <KeyValuePair <string, ExecutionSummary> > {
summary
};
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, summaries));
Directory.SetCurrentDirectory(originalWorkingFolder);
xmlTransformers.ForEach(transformer => transformer(assembliesElement));
return(failed ? 1 : executionSummary.Failed);
}
static void DispatchLongRunningTestsSummaryAsDiagnosticMessage(LongRunningTestsSummary summary, IMessageSinkWithTypes diagnosticMessageSink)
{
var messages = summary.TestCases.Select(pair => $"[Long Running Test] '{pair.Key.DisplayName}', Elapsed: {pair.Value:hh\\:mm\\:ss}");
var message = string.Join(Environment.NewLine, messages.ToArray());
diagnosticMessageSink.OnMessage(new DiagnosticMessage(message));
}
static void DispatchLongRunningTestsSummaryAsDiagnosticMessage(LongRunningTestsSummary summary, IMessageSinkWithTypes diagnosticMessageSink)
{
var messages = summary.TestCases.Select(pair => $"[Long Running Test] '{pair.Key.DisplayName}', Elapsed: {pair.Value:hh\\:mm\\:ss}");
var message = string.Join(Environment.NewLine, messages.ToArray());
diagnosticMessageSink.OnMessage(new DiagnosticMessage(message));
}
static int RunProject(XunitProject project,
bool serialize,
bool?parallelizeAssemblies,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
bool internalDiagnosticMessages)
{
XElement assembliesElement = null;
var clockTime = Stopwatch.StartNew();
var xmlTransformers = TransformFactory.GetXmlTransformers(project);
var needsXml = xmlTransformers.Count > 0;
var consoleLock = new object();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (needsXml)
{
assembliesElement = new XElement("assemblies");
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, noAppDomain, failSkips, project.Filters, internalDiagnosticMessages)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(consoleLock, assembly, serialize, needsXml, parallelizeTestCollections, maxThreadCount, diagnosticMessages, noColor, noAppDomain, failSkips, project.Filters, internalDiagnosticMessages);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(originalWorkingFolder);
xmlTransformers.ForEach(transformer => transformer(assembliesElement));
return(failed ? 1 : completionMessages.Values.Sum(summary => summary.Failed));
}
int RunProject(XunitProject project,
bool?parallelizeAssemblies,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
AppDomainSupport?appDomains,
bool failSkips,
bool designTime,
bool list,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
XElement assembliesElement = null;
var xmlTransformers = TransformFactory.GetXmlTransformers(project);
var needsXml = xmlTransformers.Count > 0;
var consoleLock = new object();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly =>
{
var assm = (XunitProjectAssembly2)assembly;
return(assm.ConfigFilename != null ?
assm.Configuration.ParallelizeAssemblyOrDefault :
assm.ConfigurationStream.ParallelizeAssemblyOrDefault);
});
}
if (needsXml)
{
assembliesElement = new XElement("assemblies");
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var clockTime = Stopwatch.StartNew();
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => TaskRun(() =>
ExecuteAssembly(
consoleLock,
(XunitProjectAssembly2)assembly,
needsXml,
parallelizeTestCollections,
maxThreadCount,
diagnosticMessages,
noColor,
appDomains,
failSkips,
project.Filters,
designTime,
list,
designTimeFullyQualifiedNames)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(
consoleLock,
(XunitProjectAssembly2)assembly,
needsXml,
parallelizeTestCollections,
maxThreadCount,
diagnosticMessages,
noColor,
appDomains,
failSkips,
project.Filters,
designTime,
list,
designTimeFullyQualifiedNames);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
SendTestCompletedIfNecessary(designTime, list);
clockTime.Stop();
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(originalWorkingFolder);
foreach (var transformer in xmlTransformers)
{
transformer(assembliesElement);
}
return(failed ? 1 : completionMessages.Values.Sum(summary => summary.Failed));
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
// Apply any filtering
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
// We are in Run Specific tests scenario, the `TestCase` objects are available.
// Query the `TestCase` objects to find XunitTestCase objects.
foreach (var vstestCase in runInfo.TestCases)
{
var xunitTestCase = Deserialize(logger, controller, vstestCase);
if (xunitTestCase != null)
{
testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
testCases.Add(xunitTestCase);
}
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (FailSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporter = GetReporter();
if (reporter == null)
{
return(false);
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
assembliesElement.Save(Xml.GetMetadata("FullPath"));
}
if (XmlV1 != null)
{
Transform("xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
Transform("HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
if (NeedsXml)
assembliesElement = new XElement("assemblies");
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return false;
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporter = GetReporter();
if (reporter == null)
return false;
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
configFileName = null;
var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName };
if (shadowCopy.HasValue)
projectAssembly.Configuration.ShadowCopy = shadowCopy;
project.Add(projectAssembly);
}
if (WorkingFolder != null)
Directory.SetCurrentDirectory(WorkingFolder);
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
assembliesElement.Add(assemblyElement);
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
if (assembliesElement != null)
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
if (completionMessages.Count > 0)
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
assembliesElement.Save(Xml.GetMetadata("FullPath"));
if (XmlV1 != null)
Transform("xUnit1.xslt", assembliesElement, XmlV1);
if (Html != null)
Transform("HTML.xslt", assembliesElement, Html);
if (NUnit != null)
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return ExitCode == 0 || (ExitCode == 1 && IgnoreFailures);
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
.Where(tc => tc.xunit != null)
.ToDictionary(tc => tc.xunit, tc => tc.vs);
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(xunitTestCases.Keys.ToList(), resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit {CrossPlatform.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
var appDomains = default(AppDomainSupport?);
switch (AppDomains?.ToLowerInvariant()) // Using ToLowerInvariant() here for back compat for when this was a boolean
{
case null:
break;
case "ifavailable":
appDomains = AppDomainSupport.IfAvailable;
break;
case "true":
case "required":
appDomains = AppDomainSupport.Required;
break;
case "false":
case "denied":
appDomains = AppDomainSupport.Denied;
break;
default:
Log.LogError("AppDomains value '{0}' is invalid: must be 'ifavailable', 'required', or 'denied'", AppDomains);
return(false);
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(Log, InternalDiagnosticMessages);
using (AssemblyHelper.SubscribeResolveForAssembly(typeof(xunit), internalDiagnosticsMessageSink))
{
var reporter = GetReporter();
if (reporter == null)
{
return(false);
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
var versionAttribute = typeof(xunit).GetTypeInfo().Assembly.GetCustomAttribute <AssemblyInformationalVersionAttribute>();
Log.LogMessage(MessageImportance.High, $"xUnit.net MSBuild Runner v{versionAttribute.InformationalVersion} ({environment})");
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly, appDomains)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly, appDomains);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
using (var xmlStream = new FileStream(Xml.GetMetadata("FullPath"), FileMode.OpenOrCreate, FileAccess.Write))
assembliesElement.Save(xmlStream);
}
if (XmlV1 != null)
{
CrossPlatform.Transform(logger, "XmlV1", "xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
CrossPlatform.Transform(logger, "Html", "HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
CrossPlatform.Transform(logger, "NUnit", "NUnitXml.xslt", assembliesElement, NUnit);
}
if (JUnit != null)
{
CrossPlatform.Transform(logger, "JUnit", "JUnitXml.xslt", assembliesElement, JUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
if (TeamCity)
{
Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead.");
return(false);
}
if (Verbose)
{
Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead.");
return(false);
}
XElement assembliesElement = null;
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
if (NeedsXml)
{
assembliesElement = new XElement("assemblies");
}
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = -1;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return(false);
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporters = GetAvailableRunnerReporters();
var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled);
if (reporter == null && !string.IsNullOrWhiteSpace(Reporter))
{
reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase));
if (reporter == null)
{
var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList();
if (switchableReporters.Count == 0)
{
Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter);
}
else
{
Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters));
}
return(false);
}
}
if (reporter == null)
{
reporter = new DefaultRunnerReporterWithTypes();
}
logger = new MSBuildLogger(Log);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(reporter.CreateMessageHandler(logger));
if (!NoLogo)
{
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
}
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
{
configFileName = null;
}
var projectAssembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName, ConfigFilename = configFileName
};
if (shadowCopy.HasValue)
{
projectAssembly.Configuration.ShadowCopy = shadowCopy;
}
project.Add(projectAssembly);
}
if (WorkingFolder != null)
{
Directory.SetCurrentDirectory(WorkingFolder);
}
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
{
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
}
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
{
assembliesElement.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
{
assembliesElement.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (completionMessages.Count > 0)
{
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
{
assembliesElement.Save(Xml.GetMetadata("FullPath"));
}
if (XmlV1 != null)
{
Transform("xUnit1.xslt", assembliesElement, XmlV1);
}
if (Html != null)
{
Transform("HTML.xslt", assembliesElement, Html);
}
if (NUnit != null)
{
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
}
// ExitCode is set to 1 for test failures and -1 for Exceptions.
return(ExitCode == 0 || (ExitCode == 1 && IgnoreFailures));
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (runSettings.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
try
{
var serializations = runInfo.TestCases
.Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
.ToList();
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
serializations.Count,
Environment.NewLine,
string.Join(Environment.NewLine, serializations.Select(x => $" {x}")));
}
var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);
if (deserializedTestCasesByUniqueId == null)
{
logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
}
else
{
for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
{
try
{
var kvp = deserializedTestCasesByUniqueId[idx];
var vsTestCase = runInfo.TestCases[idx];
if (kvp.Value == null)
{
logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
}
else
{
testCasesMap.Add(kvp.Key, vsTestCase);
testCases.Add(kvp.Value);
}
}
catch (Exception ex)
{
logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
}
}
}
}
catch (Exception ex)
{
logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (runSettings.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
private void ExecuteTests(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
[NotNull] XunitFilters filters,
[NotNull] XunitFrontController controller,
[NotNull] TestDiscoverySink discoverySink,
[NotNull] ITestFrameworkExecutionOptions executionOptions,
[CanBeNull] XElement assemblyElement,
[NotNull] DiagnosticMessageSink diagnosticMessageSink)
{
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var serialize = options.Serialize;
var failSkips = options.FailSkips;
var stopOnFail = options.StopOnFail;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
// Discover & filter the tests
_reporterMessageHandler.OnMessage(
new TestAssemblyDiscoveryStarting(
assembly,
controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy,
discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
_completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
_reporterMessageHandler,
() => _cancel,
(path, summary) => _completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
{
return;
}
Console.WriteLine("Canceling due to test failure...");
_cancel = true;
}
}
/// <summary>
/// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description.
/// </summary>
/// <param name="typeName"></param>
/// <param name="testReportName"></param>
private void ExecuteTests(string typeName = null, string testReportName = null)
{
ExecutionSummary executionSummary = null;
XunitProjectAssembly assembly = new XunitProjectAssembly
{
AssemblyFilename = assemblyLocation,
ConfigFilename = null
};
XElement assembliesElement = new XElement("assemblies");
XElement assemblyElement = new XElement("assembly");
// Logger
var verboserReporter = new XunitReporters.VerboseReporter();
IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true);
IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger));
// Option setup
ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null);
ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null);
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel);
executionOptions.SetDiagnosticMessages(true);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var clockTime = Stopwatch.StartNew();
bool cancel = false;
using (var controller = new XunitFrontController(
appDomainSupport: AppDomainSupport.Denied,
assemblyFileName: assembly.AssemblyFilename,
configFileName: null,
shadowCopy: shadowCopy,
shadowCopyFolder: null,
sourceInformationProvider: null,
diagnosticMessageSink: null))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(
assembly: assembly,
appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy: shadowCopy,
discoveryOptions: discoveryOptions));
if (typeName != null)
{
controller.Find(typeName, false, discoverySink, discoveryOptions);
}
else
{
controller.Find(false, discoverySink, discoveryOptions);
}
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; });
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
if (executionSummary != null)
{
Console.WriteLine();
KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary);
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > {
kvpExecutionSummary
}));
if (testReportName != null)
{
// Create folder if it does not exist
FileInfo fi = new FileInfo(testReportName);
DirectoryInfo directory = fi.Directory;
if (!directory.Exists)
{
Directory.CreateDirectory(directory.FullName);
}
CreateXmlTestReport(assembliesElement, fi);
CreateHtmlTestReport(assembliesElement, fi);
}
Console.WriteLine();
Console.WriteLine();
}
}