XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly2 assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
AppDomainSupport?appDomain,
bool failSkips,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// if we had a config file use it
var config = assembly.ConfigFilename != null ? assembly.Configuration : assembly.ConfigurationStream;
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
config.PreEnumerateTheories = false;
}
if (appDomain.HasValue)
{
config.AppDomain = appDomain.GetValueOrDefault();
}
if (diagnosticMessages)
{
config.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(config);
var executionOptions = TestFrameworkOptions.ForExecution(config);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, config.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = config.AppDomainOrDefault;
var shadowCopy = config.ShadowCopyOrDefault;
var longRunningSeconds = config.LongRunningTestSecondsOrDefault;
var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly);
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink, sourceInformationProvider: sourceInformationProvider))
using (var discoverySink = new TestDiscoverySink())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoverySink, discoveryOptions: discoveryOptions);
discoverySink.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoverySink.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
foreach (var testcase in vsTestCases.Values)
{
testDiscoverySink?.SendTestFound(testcase);
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoverySink.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionSink resultsSink;
if (designTime)
{
resultsSink = new DesignTimeExecutionSink(testExecutionSink, vsTestCases, reporterMessageHandler);
}
else
{
resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds);
}
if (failSkips)
{
resultsSink = new FailSkipSink(resultsSink);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoverySink.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionSink resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds);
if (FailSkips)
{
resultsSink = new FailSkipSink(resultsSink);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds);
if (failSkips)
{
resultsSink = new FailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
public static void RunTests(Options o)
{
var nullMessage = new Xunit.NullMessageSink();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false))
{
var tv = new TestDiscoverySink();
var excludeTestCaseSet = new TestDiscoverySink();
c.Find(true, tv, discoveryOptions);
tv.Finished.WaitOne();
foreach (var tc in tv.TestCases)
{
var method = tc.TestMethod.Method;
var attributes = method.GetCustomAttributes(typeof(FactAttribute));
foreach (ReflectionAttributeInfo at in attributes)
{
var result = at.GetNamedArgument<string>("Skip");
if (result != null)
{
Console.WriteLine("SKIPPY! {0} because {1}", method, result);
}
if (o.TestType != TestType.All)
{
if (!at.ToString().EndsWith(o.TestType.ToString()))
{
excludeTestCaseSet.TestCases.Add(tc);
}
}
}
}
foreach (var tc in excludeTestCaseSet.TestCases)
{
tv.TestCases.Remove(tc);
}
Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count);
//core execution Sink
int testCaseCount = tv.TestCases.Count;
Stream file = new FileStream(".\\result.xml", FileMode.Create);
int totalResult = 0;
int totalErrors = 0;
int totalFailed = 0;
int totalSkipped = 0;
for (int i = 0; i < testCaseCount; i++)
{
IExecutionSink resultsSink;
ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
IMessageSinkWithTypes reporterMessageHandler;
var reporters = GetAvailableRunnerReporters();
var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll");
IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
var xmlElement = new XElement("TestResult");
resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true);
var message = new Xunit.NullMessageSink();
var executionOptions = TestFrameworkOptions.ForExecution();
c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions);
resultsSink.Finished.WaitOne(o.TimeOut);
tv.TestCases.RemoveAt(0);
totalResult++;
totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors;
totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed;
totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped;
xmlElement.Save(file);
file.Flush();
}
file.Dispose();
Console.WriteLine("Total tests: " + totalResult);
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("Error tests: " + totalErrors);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed tests: " + totalFailed);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Skipped tests: " + totalSkipped);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped));
Console.ResetColor();
}
}
