protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions());
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !ParallelizeTestCollections,
MaxParallelThreads = MaxParallelThreads
};
controller.RunTests(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
lock (consoleLock)
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var executionOptions = new XunitExecutionOptions {
DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount
};
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
public static void TestOptions_MaxThreads()
{
var options = new XunitExecutionOptions {
MaxParallelThreads = 3
};
var runner = TestableXunitTestAssemblyRunner.Create(options: options);
var result = runner.GetTestFrameworkEnvironment();
Assert.EndsWith("[collection-per-class, parallel (3 threads)]", result);
}
public static void TestOptions_NonParallel()
{
var options = new XunitExecutionOptions {
DisableParallelization = true
};
var runner = TestableXunitTestAssemblyRunner.Create(options: options);
var result = runner.GetTestFrameworkEnvironment();
Assert.EndsWith("[collection-per-class, non-parallel]", result);
}
void RunTestsInAssembly(IDiscoveryContext discoveryContext,
IFrameworkHandle frameworkHandle,
List <IDisposable> toDispose,
string assemblyFileName,
IEnumerable <TestCase> testCases,
XunitVisualStudioSettings settings,
Stopwatch stopwatch)
{
if (cancelled)
{
return;
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
#if WIN8_STORE
// For store apps, the files are copied to the AppX dir, we need to load it from there
assemblyFileName = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), Path.GetFileName(assemblyFileName));
#elif WINDOWS_PHONE_APP
// For WPA Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true);
lock (toDispose)
toDispose.Add(controller);
var xunitTestCases = testCases.ToDictionary(tc => controller.Deserialize(tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)));
using (var executionVisitor = new VsExecutionVisitor(discoveryContext, frameworkHandle, xunitTestCases, () => cancelled))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !settings.ParallelizeTestCollections,
MaxParallelThreads = settings.MaxParallelThreads
};
controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions);
executionVisitor.Finished.WaitOne();
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution finished: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
}
public static void TestOptionsOverrideAttribute()
{
var attribute = Mocks.CollectionBehaviorAttribute(disableTestParallelization: true, maxParallelThreads: 127);
var options = new XunitExecutionOptions {
DisableParallelization = false, MaxParallelThreads = 3
};
var assembly = Mocks.TestAssembly(new[] { attribute });
var runner = TestableXunitTestAssemblyRunner.Create(assembly: assembly, options: options);
var result = runner.GetTestFrameworkEnvironment();
Assert.EndsWith("[collection-per-class, parallel (3 threads)]", result);
}
public static async void NonParallel()
{
var passing = Mocks.XunitTestCase <ClassUnderTest>("Passing");
var other = Mocks.XunitTestCase <ClassUnderTest>("Other");
var options = new XunitExecutionOptions {
DisableParallelization = true
};
var runner = TestableXunitTestAssemblyRunner.Create(testCases: new[] { passing, other }, options: options);
await runner.RunAsync();
var threadIDs = runner.TestCasesRun.Select(x => x.Item1).ToList();
Assert.Equal(threadIDs[0], threadIDs[1]);
}
void RunTestsInAssembly(IDiscoveryContext discoveryContext,
IFrameworkHandle frameworkHandle,
List <IDisposable> toDispose,
string assemblyFileName,
IEnumerable <TestCase> testCases,
XunitVisualStudioSettings settings,
Stopwatch stopwatch)
{
if (cancelled)
{
return;
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true);
lock (toDispose)
toDispose.Add(controller);
var xunitTestCases = testCases.ToDictionary(tc => controller.Deserialize(tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)));
using (var executionVisitor = new VsExecutionVisitor(discoveryContext, frameworkHandle, xunitTestCases, () => cancelled))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !settings.ParallelizeTestCollections,
MaxParallelThreads = settings.MaxParallelThreads
};
controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions);
executionVisitor.Finished.WaitOne();
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution finished: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent, bool parallelizeTestCollections, int maxThreadCount)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var executionOptions = new XunitExecutionOptions {
DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount
};
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent);
controller.RunTests(discoveryVisitor.TestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount };
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent);
controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return assemblyElement;
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
return null;
var assemblyElement = CreateAssemblyXElement();
try
{
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions());
discoveryVisitor.Finished.WaitOne();
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !ParallelizeTestCollections,
MaxParallelThreads = MaxParallelThreads
};
controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
var discoveryOptions = new XunitDiscoveryOptions(configuration);
var executionOptions = new XunitExecutionOptions(configuration);
if (maxParallelThreads.HasValue)
{
executionOptions.MaxParallelThreads = maxParallelThreads.GetValueOrDefault();
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault();
}
if (configuration.DiagnosticMessages)
{
Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assemblyFileName),
discoveryOptions.MethodDisplay,
!executionOptions.DisableParallelization,
executionOptions.MaxParallelThreads);
}
else
{
Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
}
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
Log.LogError("{0} has no tests to run", Path.GetFileNameWithoutExtension(assemblyFileName));
ExitCode = 1;
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
XElement ExecuteAssembly(object consoleLock,
string defaultDirectory,
XunitProjectAssembly assembly,
bool needsXml,
string visitor,
bool?parallelizeTestCollections,
int?maxThreadCount,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
var discoveryOptions = new XunitDiscoveryOptions(assembly.Configuration);
var executionOptions = new XunitExecutionOptions(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.MaxParallelThreads = maxThreadCount.GetValueOrDefault();
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault();
}
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.MethodDisplay,
!executionOptions.DisableParallelization,
executionOptions.MaxParallelThreads);
}
else
{
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestcases = null;
if (designTime)
{
vsTestcases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
var sink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink));
foreach (var testcase in vsTestcases.Values)
{
if (sink != null)
{
sink.SendTest(testcase);
}
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, visitor);
if (designTime)
{
var sink = (ITestExecutionSink)_services.GetService(typeof(ITestExecutionSink));
resultsVisitor = new DesignTimeExecutionVisitor(
sink,
vsTestcases,
resultsVisitor);
}
IList <ITestCase> filteredTestCases;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = (from t in vsTestcases
where designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)
select t.Key)
.ToList();
}
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
void ExecuteAssembly(object consoleLock, string assemblyName, CommandLine options)
{
if (cancel)
{
return;
}
try
{
var name = new AssemblyName(assemblyName);
var assembly = Reflector.Wrap(Assembly.Load(name));
var framework = GetFramework(assembly);
var discoverer = framework.GetDiscoverer(assembly);
var executor = framework.GetExecutor(name);
var discoveryVisitor = new TestDiscoveryVisitor();
discoverer.Find(includeSourceInformation: true, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
if (options.List)
{
ITestDiscoverySink discoverySink = null;
if (options.DesignTime)
{
discoverySink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink));
}
lock (consoleLock)
{
foreach (var test in discoveryVisitor.TestCases)
{
if (discoverySink != null)
{
discoverySink.SendTest(test.ToDesignTimeTest());
}
Console.WriteLine(test.DisplayName);
}
}
return;
}
var executionOptions = new XunitExecutionOptions {
DisableParallelization = !options.ParallelizeTestCollections, MaxParallelThreads = options.MaxParallelThreads
};
var resultsVisitor = CreateVisitor(consoleLock, options);
var tests = discoveryVisitor.TestCases;
if (options.Tests != null && options.Tests.Count > 0)
{
tests = tests.Where(t => IsTestNameMatch(t, options.Tests)).ToList();
}
executor.RunTests(tests, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
// When executing under TeamCity, we record the results in a format TeamCity understands, but do not return an error code.
// This causes TeamCity to treat the step as completed, but the build as failed. We'll work around this by special casing the TeamCityVisitor
var teamCityVisitor = resultsVisitor as TeamCityVisitor;
if (teamCityVisitor != null)
{
failed = teamCityVisitor.Failed > 0;
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
Console.WriteLine(ex);
failed = true;
}
}
