public void FindByTypesReturnsOnlyMethodsInTheGivenType()
{
var xml = @"
<assembly>
<class name='Type1'>
<method name='Method1 Display Name' type='Type1' method='Method1'/>
</class>
<class name='Type2'>
<method name='Type2.Method1' type='Type2' method='Method1'/>
<method name='Type2.Method2' type='Type2' method='Method2'/>
</class>
</assembly>";
using (var xunit1 = new TestableXunit1())
{
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml));
var sink = new TestDiscoveryVisitor();
xunit1.Find("Type2", false, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase => Assert.Equal("Type2.Method1", testCase.DisplayName),
testCase => Assert.Equal("Type2.Method2", testCase.DisplayName)
);
}
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions());
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !ParallelizeTestCollections,
MaxParallelThreads = MaxParallelThreads
};
controller.RunTests(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
public void TestCasesUseInformationFromSourceInformationProvider()
{
var xml = @"
<assembly>
<class name='Type2'>
<method name='Type2.Method1' type='Type2' method='Method1'/>
<method name='Type2.Method2' type='Type2' method='Method2'/>
</class>
</assembly>";
using (var xunit1 = new TestableXunit1())
{
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml));
xunit1.SourceInformationProvider
.GetSourceInformation(null)
.ReturnsForAnyArgs(callInfo => new SourceInformation {
FileName = "File for " + callInfo.Arg <ITestCase>().DisplayName
});
var sink = new TestDiscoveryVisitor();
xunit1.Find(true, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase => Assert.Equal("File for Type2.Method1", testCase.SourceInformation.FileName),
testCase => Assert.Equal("File for Type2.Method2", testCase.SourceInformation.FileName)
);
}
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
lock (consoleLock)
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var executionOptions = new XunitExecutionOptions {
DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount
};
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
IEnumerable <AssemblyRunInfo> DiscoverTestsInAssemblies()
{
var result = new List <AssemblyRunInfo>();
try
{
using (AssemblyHelper.SubscribeResolve())
{
foreach (var assm in TestAssemblies)
{
// Xunit needs the file name
var assemblyFileName = assm.GetName()
.Name + ".dll";
var configuration = GetConfiguration(assm.GetName()
.Name);
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
try
{
if (cancelled)
{
break;
}
using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false))
using (var sink = new TestDiscoveryVisitor())
{
framework.Find(true, sink, discoveryOptions);
sink.Finished.WaitOne();
result.Add(new AssemblyRunInfo
{
AssemblyFileName = assemblyFileName,
Configuration = configuration,
TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this))
.ToList()
});
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
}
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
return(result);
}
protected TestableXunitTestFrameworkDiscoverer(IAssemblyInfo assembly,
ISourceInformationProvider sourceProvider,
IXunitTestCollectionFactory collectionFactory,
IMessageAggregator messageAggregator)
: base(assembly,
sourceProvider ?? Substitute.For <ISourceInformationProvider>(),
collectionFactory,
messageAggregator)
{
Assembly = assembly;
Visitor = new TestDiscoveryVisitor();
}
protected TestableXunitTestFrameworkDiscoverer(IAssemblyInfo assembly,
ISourceInformationProvider sourceProvider,
IMessageSink diagnosticMessageSink,
IXunitTestCollectionFactory collectionFactory)
: base(assembly,
sourceProvider ?? Substitute.For <ISourceInformationProvider>(),
diagnosticMessageSink ?? new Xunit.NullMessageSink(),
collectionFactory)
{
Assembly = assembly;
Visitor = new TestDiscoveryVisitor();
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor);
discoveryVisitor.Finished.WaitOne();
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
public IList <ITestCase> GetTestCases()
{
var options = TestFrameworkOptions.ForDiscovery(environment.TestAssemblyConfiguration);
Logger.LogVerbose("Starting discovery");
LogDiscoveryOptions(options);
var visitor = new TestDiscoveryVisitor(runContext);
discoverer.Find(false, visitor, options);
visitor.Finished.WaitOne();
Logger.LogVerbose("Filtering test cases");
return(visitor.TestCases);
}
IReadOnlyList <ITestCase> Discover(Action <IMessageSink> discoveryAction)
{
try
{
var visitor = new TestDiscoveryVisitor();
toDispose.Push(visitor);
discoveryAction(visitor);
visitor.Finished.WaitOne();
return(visitor.TestCases.ToList());
}
catch (Exception ex)
{
testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error);
return(new ITestCase[0]);
}
}
private IEnumerable<ITestCase> Discover(Action<IMessageSink> discoveryAction)
{
try
{
var visitor = new TestDiscoveryVisitor();
toDispose.Push(visitor);
discoveryAction(visitor);
visitor.Finished.WaitOne();
return visitor.TestCases.ToList();
}
catch (Exception ex)
{
testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error);
return Enumerable.Empty<ITestCase>();
}
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor);
discoveryVisitor.Finished.WaitOne();
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent);
controller.Run(discoveryVisitor.TestCases, resultsVisitor);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Console.WriteLine(stackLine);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public void CollectsTestCases()
{
var visitor = new TestDiscoveryVisitor();
var testCase1 = Substitute.For<ITestCase>();
var testCase2 = Substitute.For<ITestCase>();
var testCase3 = Substitute.For<ITestCase>();
visitor.OnMessage(new DiscoveryMessage(testCase1));
visitor.OnMessage(new DiscoveryMessage(testCase2));
visitor.OnMessage(new DiscoveryMessage(testCase3));
visitor.OnMessage(Substitute.For<IMessageSinkMessage>()); // Ignored
Assert.Collection(visitor.TestCases,
msg => Assert.Same(testCase1, msg),
msg => Assert.Same(testCase2, msg),
msg => Assert.Same(testCase3, msg)
);
}
public static void TheoriesAlwaysComeBackAsSingleXunitTheoryTestCase()
{
var sourceProvider = new NullSourceInformationProvider();
var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly());
var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider);
var visitor = new TestDiscoveryVisitor();
discoverer.Find(typeof(ClassWithTheory).FullName, false, visitor, new XunitDiscoveryOptions());
visitor.Finished.WaitOne();
var testCase = Assert.Single(visitor.TestCases);
Assert.IsType <XunitTheoryTestCase>(testCase);
var deserialized = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(testCase));
Assert.IsType <XunitTheoryTestCase>(deserialized);
}
public static void TheoryWithNonSerializableData_ReturnsAsASingleTestCase()
{
var sourceProvider = new NullSourceInformationProvider();
var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly());
var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider);
var visitor = new TestDiscoveryVisitor();
discoverer.Find(typeof(ClassWithNonSerializableTheoryData).FullName, false, visitor, TestFrameworkOptions.ForDiscovery());
visitor.Finished.WaitOne();
var testCase = Assert.Single(visitor.TestCases);
Assert.IsType <XunitTheoryTestCase>(testCase);
var deserialized = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(testCase));
Assert.IsType <XunitTheoryTestCase>(deserialized);
}
public void CollectsTestCases()
{
var visitor = new TestDiscoveryVisitor();
var testCase1 = Substitute.For <ITestCase>();
var testCase2 = Substitute.For <ITestCase>();
var testCase3 = Substitute.For <ITestCase>();
visitor.OnMessage(new DiscoveryMessage(testCase1));
visitor.OnMessage(new DiscoveryMessage(testCase2));
visitor.OnMessage(new DiscoveryMessage(testCase3));
visitor.OnMessage(Substitute.For <IMessageSinkMessage>()); // Ignored
Assert.Collection(visitor.TestCases,
msg => Assert.Same(testCase1, msg),
msg => Assert.Same(testCase2, msg),
msg => Assert.Same(testCase3, msg)
);
}
private async Task AddAssemblies(IEnumerable <AssemblyAndConfigFile> assemblies)
{
var loadingDialog = new LoadingDialog {
Owner = MainWindow.Instance
};
try
{
using (AssemblyHelper.SubscribeResolve())
{
loadingDialog.Show();
foreach (var assembly in assemblies)
{
loadingDialog.AssemblyFileName = assembly.AssemblyFileName;
using (var xunit = new XunitFrontController(
useAppDomain: true,
assemblyFileName: assembly.AssemblyFileName,
configFileName: assembly.ConfigFileName,
diagnosticMessageSink: new DiagnosticMessageVisitor(),
shadowCopy: false))
using (var testDiscoveryVisitor = new TestDiscoveryVisitor(xunit))
{
await Task.Run(() =>
{
xunit.Find(includeSourceInformation: false, messageSink: testDiscoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery());
testDiscoveryVisitor.Finished.WaitOne();
});
allTestCases.AddRange(testDiscoveryVisitor.TestCases);
Assemblies.Add(new TestAssemblyViewModel(assembly));
}
}
}
}
catch (Exception ex)
{
MessageBox.Show(Application.Current.MainWindow, ex.ToString());
}
finally
{
loadingDialog.Close();
}
}
public static void SerializedTestsInSameCollectionRemainInSameCollection()
{
var sourceProvider = new NullSourceInformationProvider();
var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly());
var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider);
var visitor = new TestDiscoveryVisitor();
discoverer.Find(typeof(ClassUnderTest).FullName, false, visitor, new XunitDiscoveryOptions());
visitor.Finished.WaitOne();
var first = visitor.TestCases[0];
var second = visitor.TestCases[1];
Assert.True(TestCollectionComparer.Instance.Equals(first.TestCollection, second.TestCollection));
var serializedFirst = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(first));
var serializedSecond = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(second));
Assert.NotSame(serializedFirst.TestCollection, serializedSecond.TestCollection);
Assert.True(TestCollectionComparer.Instance.Equals(serializedFirst.TestCollection, serializedSecond.TestCollection));
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent, bool parallelizeTestCollections, int maxThreadCount)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var executionOptions = new XunitExecutionOptions {
DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount
};
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent);
controller.RunTests(discoveryVisitor.TestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
public static void TheoriesWithSerializableData_ReturnAsIndividualTestCases()
{
var sourceProvider = new NullSourceInformationProvider();
var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly());
var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider);
var visitor = new TestDiscoveryVisitor();
discoverer.Find(typeof(ClassWithTheory).FullName, false, visitor, TestFrameworkOptions.ForDiscovery());
visitor.Finished.WaitOne();
var first = visitor.TestCases[0];
var second = visitor.TestCases[1];
Assert.NotEqual(first.UniqueID, second.UniqueID);
Assert.True(TestCollectionComparer.Instance.Equals(first.TestMethod.TestClass.TestCollection, second.TestMethod.TestClass.TestCollection));
var serializedFirst = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(first));
var serializedSecond = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(second));
Assert.NotSame(serializedFirst.TestMethod.TestClass.TestCollection, serializedSecond.TestMethod.TestClass.TestCollection);
Assert.True(TestCollectionComparer.Instance.Equals(serializedFirst.TestMethod.TestClass.TestCollection, serializedSecond.TestMethod.TestClass.TestCollection));
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
return null;
var assemblyElement = CreateAssemblyXElement();
try
{
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new XunitDiscoveryOptions());
discoveryVisitor.Finished.WaitOne();
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var executionOptions = new XunitExecutionOptions
{
DisableParallelization = !ParallelizeTestCollections,
MaxParallelThreads = MaxParallelThreads
};
controller.Run(discoveryVisitor.TestCases.Where(Filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml()
{
var xml = @"
<assembly>
<class name='Type1'>
<method name='Method1 Display Name' type='Type1' method='Method1'/>
</class>
<class name='SpecialType'>
<method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/>
<method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'>
<traits>
<trait name='Trait1' value='Value1'/>
<trait name='Trait2' value='Value2'/>
</traits>
</method>
</class>
</assembly>";
using (var xunit1 = new TestableXunit1())
{
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml));
var sink = new TestDiscoveryVisitor();
xunit1.Find(false, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase =>
{
Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("Method1", testCase.TestMethod.Method.Name);
Assert.Equal("Method1 Display Name", testCase.DisplayName);
Assert.Null(testCase.SkipReason);
Assert.Empty(testCase.Traits);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName);
Assert.Equal("I am not run", testCase.SkipReason);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName);
Assert.Collection(testCase.Traits.Keys,
key =>
{
Assert.Equal("Trait1", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value1", value)
);
},
key =>
{
Assert.Equal("Trait2", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value2", value)
);
}
);
}
);
}
}
static XElement ExecuteAssembly(object consoleLock,
string defaultDirectory,
XunitProjectAssembly assembly,
bool quiet,
bool serialize,
bool needsXml,
bool teamCity,
bool appVeyor,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
if (diagnosticMessages)
assembly.Configuration.DiagnosticMessages = true;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
if (!quiet)
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessagesOrDefault)
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
else
Console.WriteLine("Discovering: {0}", assemblyDisplayName);
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
if (!quiet)
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return assemblyElement;
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration)
{
if (cancel)
return null;
var assemblyElement = CreateAssemblyXElement();
try
{
if (DiagnosticMessages)
configuration.DiagnosticMessages = true;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
if (maxParallelThreads.HasValue && maxParallelThreads.Value > -1)
executionOptions.SetMaxParallelThreads(maxParallelThreads);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
if (configuration.DiagnosticMessagesOrDefault)
Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
else
Log.LogMessage(MessageImportance.High, " Discovering: {0}", assemblyDisplayName);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, configuration.DiagnosticMessagesOrDefault);
using (var controller = CreateFrontController(assemblyFileName, configFileName, diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, discoveryMessageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", assemblyDisplayName);
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
if (filteredTestCases.Count == 0)
completionMessages.TryAdd(assemblyDisplayName, new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool? parallelizeTestCollections, int? maxThreadCount, XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages ?? false)
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.GetMethodDisplay(),
!executionOptions.GetDisableParallelization(),
executionOptions.GetMaxParallelThreads());
else
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
{
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return assemblyElement;
}
IEnumerable<AssemblyRunInfo> DiscoverTestsInAssemblies()
{
var result = new List<AssemblyRunInfo>();
try
{
using (AssemblyHelper.SubscribeResolve())
{
foreach (var assm in TestAssemblies)
{
// Xunit needs the file name
var assemblyFileName = assm.GetName()
.Name + ".dll";
var configuration = GetConfiguration(assm.GetName()
.Name);
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
try
{
if (cancelled)
break;
using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false))
using (var sink = new TestDiscoveryVisitor())
{
framework.Find(true, sink, discoveryOptions);
sink.Finished.WaitOne();
result.Add(new AssemblyRunInfo
{
AssemblyFileName = assemblyFileName,
Configuration = configuration,
TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this))
.ToList()
});
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
}
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
return result;
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (failSkips)
resultsVisitor = new FailSkipVisitor(resultsVisitor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return assemblyElement;
}
static int Main(string[] args)
{
// Handle RemoteExecutor
if (args.Length > 0 && args[0] == "remote")
{
return(RemoteExecutor.Execute(args.Skip(1).ToArray()));
}
if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help")
{
PrintHeader();
PrintUsage();
return(2);
}
var commandLine = CommandLine.Parse(args);
Console.CancelKeyPress += (sender, e) =>
{
if (!cancel)
{
Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)");
cancel = true;
e.Cancel = true;
}
};
if (commandLine.Debug)
{
Debugger.Launch();
}
if (!commandLine.NoLogo)
{
PrintHeader();
}
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
int errorCount = 0;
int failCount = 0;
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
break;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
// Discover & filter the tests
Console.WriteLine($"Discovering: {assemblyName}");
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
Console.WriteLine($"Discovered: {assemblyName}");
// Run the filtered tests
if (testCasesToRun == 0)
{
Console.WriteLine($"Info: {assemblyName} has no tests to run");
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
var assemblyElement = new XElement("assembly");
StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.Verbose, commandLine.FailSkips);
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
assembliesElement.Add(assemblyElement);
// Set counters to determine the error code later.
errorCount += resultsVisitor.ExecutionSummary.Errors;
failCount += resultsVisitor.ExecutionSummary.Failed;
Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception ex)
{
assembliesElement = new XElement("error");
assembliesElement.Add(ex);
if (!commandLine.NoColor)
{
Console.ForegroundColor = ConsoleColor.Red;
}
Console.WriteLine($"error: {ex.Message}");
if (commandLine.DiagnosticMessages)
{
if (!commandLine.NoColor)
{
Console.ForegroundColor = ConsoleColor.DarkGray;
}
Console.WriteLine(ex.StackTrace);
}
}
finally
{
if (!commandLine.NoColor)
{
Console.ResetColor();
}
WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult();
}
}
if (cancel)
{
return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C
}
if (commandLine.Wait)
{
Console.WriteLine();
Console.Write("Press any key to continue...");
Console.ReadKey();
Console.WriteLine();
}
if (errorCount > 0 || failCount > 0)
{
return(1);
}
return(0);
}
static int Main(string[] args)
{
// Handle RemoteExecutor
if (args.Length > 0 && args[0] == "remote")
{
return(RemoteExecutor.Execute(args.Skip(1).ToArray()));
}
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
int errorCount = 0;
int failCount = 0;
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
break;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
// Discover & filter the tests
Console.WriteLine($"Discovering: {assemblyName}");
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
Console.WriteLine($"Discovered: {assemblyName}");
// Run the filtered tests
if (testCasesToRun == 0)
{
Console.WriteLine($"Info: {assemblyName} has no tests to run");
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
var assemblyElement = new XElement("assembly");
StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.ShowProgress, commandLine.FailSkips);
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
assembliesElement.Add(assemblyElement);
// Set counters to determine the error code later.
errorCount += resultsVisitor.ExecutionSummary.Errors;
failCount += resultsVisitor.ExecutionSummary.Failed;
Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
Console.WriteLine(e);
}
finally
{
WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult();
}
}
if (errorCount > 0)
{
return(2);
}
else if (failCount > 0)
{
return(1);
}
return(0);
}
public void TestCasesUseInformationFromSourceInformationProvider()
{
var xml = @"
<assembly>
<class name='Type2'>
<method name='Type2.Method1' type='Type2' method='Method1'/>
<method name='Type2.Method2' type='Type2' method='Method2'/>
</class>
</assembly>";
var xunit1 = new TestableXunit1();
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml));
xunit1.SourceInformationProvider
.GetSourceInformation(null)
.ReturnsForAnyArgs(callInfo => new SourceInformation { FileName = "File for " + callInfo.Arg<ITestCase>().DisplayName });
var sink = new TestDiscoveryVisitor();
xunit1.Find(true, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase => Assert.Equal("File for Type2.Method1", testCase.SourceInformation.FileName),
testCase => Assert.Equal("File for Type2.Method2", testCase.SourceInformation.FileName)
);
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (failSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
return null;
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
if (DiagnosticMessages)
assembly.Configuration.DiagnosticMessages = true;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, discoveryOptions, executionOptions));
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, executionOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
public async void RunTests(string arguments, StringBuilder sbLog, Action <string> uiLogger)
{
log = sbLog;
var reporters = new List <IRunnerReporter>();
string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
log.AppendLine($"Args: {arguments}");
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger(uiLogger));
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
//log.AppendLine("testCasesToRun: " + testCasesToRun);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
var assemblyElement = new XElement("assembly");
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (commandLine.FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
//log.AppendLine("Finished running tests");
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
assembliesElement.Add(assemblyElement);
log.AppendLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
}
finally
{
await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement);
await WriteLogs(Path.GetFileName(assembly.AssemblyFilename), log.ToString());
}
}
Application.Current.Exit();
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName)
{
if (cancel)
return null;
var assemblyElement = CreateAssemblyXElement();
try
{
using (var controller = CreateFrontController(assemblyFileName, configFileName))
{
var discoveryVisitor = new TestDiscoveryVisitor();
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor);
discoveryVisitor.Finished.WaitOne();
var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement);
controller.Run(discoveryVisitor.TestCases, resultsVisitor);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
catch (Exception ex)
{
Exception e = ex;
while (e != null)
{
Log.LogError(e.GetType().FullName + ": " + e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
private async void RunTests(string arguments)
{
var reporters = new List<IRunnerReporter>();
string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries);
log = string.Empty;
log += "Args: " + args + "\n";
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger());
var completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
log += "testCasesToRun: " + testCasesToRun + "\n";
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
var assemblyElement = new XElement("assembly");
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (commandLine.FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
log += "finished running tests \n";
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
log += "logged exec errors: " + e + "\n";
}
}
await WriteResults(assembliesElement);
await WriteLogs(log);
Application.Current.Exit();
}
public void FindByTypesReturnsOnlyMethodsInTheGivenType()
{
var xml = @"
<assembly>
<class name='Type1'>
<method name='Method1 Display Name' type='Type1' method='Method1'/>
</class>
<class name='Type2'>
<method name='Type2.Method1' type='Type2' method='Method1'/>
<method name='Type2.Method2' type='Type2' method='Method2'/>
</class>
</assembly>";
var xunit1 = new TestableXunit1();
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml));
var sink = new TestDiscoveryVisitor();
xunit1.Find("Type2", false, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase => Assert.Equal("Type2.Method1", testCase.DisplayName),
testCase => Assert.Equal("Type2.Method2", testCase.DisplayName)
);
}
XElement ExecuteAssembly(object consoleLock,
string defaultDirectory,
XunitProjectAssembly assembly,
bool needsXml,
string visitor,
bool?parallelizeTestCollections,
int?maxThreadCount,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
var discoveryOptions = new XunitDiscoveryOptions(assembly.Configuration);
var executionOptions = new XunitExecutionOptions(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.MaxParallelThreads = maxThreadCount.GetValueOrDefault();
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault();
}
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.MethodDisplay,
!executionOptions.DisableParallelization,
executionOptions.MaxParallelThreads);
}
else
{
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestcases = null;
if (designTime)
{
vsTestcases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
var sink = (ITestDiscoverySink)_services.GetService(typeof(ITestDiscoverySink));
foreach (var testcase in vsTestcases.Values)
{
if (sink != null)
{
sink.SendTest(testcase);
}
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, visitor);
if (designTime)
{
var sink = (ITestExecutionSink)_services.GetService(typeof(ITestExecutionSink));
resultsVisitor = new DesignTimeExecutionVisitor(
sink,
vsTestcases,
resultsVisitor);
}
IList <ITestCase> filteredTestCases;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = (from t in vsTestcases
where designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)
select t.Key)
.ToList();
}
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml()
{
var xml = @"
<assembly>
<class name='Type1'>
<method name='Method1 Display Name' type='Type1' method='Method1'/>
</class>
<class name='SpecialType'>
<method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/>
<method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'>
<traits>
<trait name='Trait1' value='Value1'/>
<trait name='Trait2' value='Value2'/>
</traits>
</method>
</class>
</assembly>";
var xunit1 = new TestableXunit1();
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg<ICallbackEventHandler>().RaiseCallbackEvent(xml));
var sink = new TestDiscoveryVisitor();
xunit1.Find(false, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase =>
{
Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("Method1", testCase.TestMethod.Method.Name);
Assert.Equal("Method1 Display Name", testCase.DisplayName);
Assert.Null(testCase.SkipReason);
Assert.Empty(testCase.Traits);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName);
Assert.Equal("I am not run", testCase.SkipReason);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName);
Assert.Collection(testCase.Traits.Keys,
key =>
{
Assert.Equal("Trait1", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value1", value)
);
},
key =>
{
Assert.Equal("Trait2", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value2", value)
);
}
);
}
);
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
var discoveryOptions = new XunitDiscoveryOptions(configuration);
var executionOptions = new XunitExecutionOptions(configuration);
if (maxParallelThreads.HasValue)
{
executionOptions.MaxParallelThreads = maxParallelThreads.GetValueOrDefault();
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.DisableParallelization = !parallelizeTestCollections.GetValueOrDefault();
}
if (configuration.DiagnosticMessages)
{
Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assemblyFileName),
discoveryOptions.MethodDisplay,
!executionOptions.DisableParallelization,
executionOptions.MaxParallelThreads);
}
else
{
Log.LogMessage(MessageImportance.High, " Discovering: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
}
using (var controller = CreateFrontController(assemblyFileName, configFileName))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", Path.GetFileNameWithoutExtension(assemblyFileName));
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
Log.LogError("{0} has no tests to run", Path.GetFileNameWithoutExtension(assemblyFileName));
ExitCode = 1;
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
assembly.Configuration.PreEnumerateTheories = false;
}
if (diagnosticMessages)
{
assembly.Configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var sourceInformationProvider = new SourceInformationProviderAdapater(testHostServices?.SourceInformationProvider);
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
var sink = testHostServices.TestDiscoverySink;
foreach (var testcase in vsTestCases.Values)
{
if (sink != null)
{
sink.SendTest(testcase);
}
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionVisitor resultsVisitor;
if (designTime)
{
resultsVisitor = new DesignTimeExecutionVisitor(testHostServices.TestExecutionSink, vsTestCases, reporterMessageHandler);
}
else
{
resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch)
{
var result = new List <IGrouping <string, TestCase> >();
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed));
}
foreach (string assemblyFileName in sources)
{
var fileName = Path.GetFileName(assemblyFileName);
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName));
}
}
else
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName));
}
using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true))
using (var sink = new TestDiscoveryVisitor())
{
framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions());
sink.Finished.WaitOne();
result.Add(
new Grouping <string, TestCase>(
assemblyFileName,
sink.TestCases
.GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name))
.SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(assemblyFileName, framework, testCase, settings, forceUniqueNames: group.Count() > 1)))
.ToList()
)
);
if (settings.MessageDisplay != MessageDisplay.None)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(assemblyFileName), sink.TestCases.Count));
}
}
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e));
}
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed));
}
return(result);
}
}
protected virtual XElement ExecuteAssembly(string assemblyFileName, string configFileName, TestAssemblyConfiguration configuration)
{
if (cancel)
{
return(null);
}
var assemblyElement = CreateAssemblyXElement();
try
{
if (DiagnosticMessages)
{
configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
if (maxParallelThreads.HasValue && maxParallelThreads.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxParallelThreads);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
if (configuration.DiagnosticMessagesOrDefault)
{
Log.LogMessage(MessageImportance.High, " Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
}
else
{
Log.LogMessage(MessageImportance.High, " Discovering: {0}", assemblyDisplayName);
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, configuration.DiagnosticMessagesOrDefault);
using (var controller = CreateFrontController(assemblyFileName, configFileName, diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, discoveryMessageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
Log.LogMessage(MessageImportance.High, " Discovered: {0}", assemblyDisplayName);
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(assemblyDisplayName, new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
using (var resultsVisitor = CreateVisitor(assemblyFileName, assemblyElement))
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent);
controller.Run(discoveryVisitor.TestCases, resultsVisitor, new TestFrameworkOptions());
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Console.WriteLine(stackLine);
e = e.InnerException;
}
}
return assemblyElement;
}
void RunTestsInAssembly(IFrameworkHandle frameworkHandle, List<IDisposable> toDispose, string assemblyFileName, IEnumerable<TestCase> testCases = null)
{
if (cancelled)
return;
var xunit2 = new Xunit2(SourceInformationProvider, assemblyFileName, configFileName: null, shadowCopy: true);
toDispose.Add(xunit2);
if (testCases == null)
{
using (var visitor = new TestDiscoveryVisitor())
{
xunit2.Find(includeSourceInformation: true, messageSink: visitor);
visitor.Finished.WaitOne();
testCases = visitor.TestCases.Select(tc => VsDiscoveryVisitor.CreateVsTestCase(frameworkHandle, assemblyFileName, xunit2, tc)).ToList();
}
}
var xunitTestCases = testCases.ToDictionary(tc => xunit2.Deserialize(tc.GetPropertyValue<string>(SerializedTestCaseProperty, null)));
using (var executionVisitor = new VsExecutionVisitor(assemblyFileName, frameworkHandle, xunitTestCases, () => cancelled))
{
xunit2.Run(xunitTestCases.Keys.ToList(), executionVisitor);
executionVisitor.Finished.WaitOne();
}
}
static XElement ExecuteAssembly(object consoleLock,
string defaultDirectory,
XunitProjectAssembly assembly,
bool quiet,
bool serialize,
bool needsXml,
bool teamCity,
bool appVeyor,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
if (diagnosticMessages)
{
assembly.Configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
if (!quiet)
{
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessagesOrDefault)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
}
else
{
Console.WriteLine("Discovering: {0}", assemblyDisplayName);
}
}
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
if (!quiet)
{
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
/// <summary>
/// Starts the process of running all the xUnit.net v1 tests in the assembly.
/// </summary>
/// <param name="messageSink">The message sink to report results back to.</param>
public void Run(IMessageSink messageSink)
{
var discoverySink = new TestDiscoveryVisitor();
toDispose.Push(discoverySink);
Find(false, discoverySink);
discoverySink.Finished.WaitOne();
Run(discoverySink.TestCases, messageSink);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
if (resultsVisitor.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount };
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent);
controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return assemblyElement;
}
protected virtual void ExecuteAssembly(string assemblyFilename, string configFileName, MSBuildVisitor resultsVisitor)
{
try
{
Log.LogMessage(MessageImportance.High, "Test assembly: {0}", assemblyFilename);
using (var controller = CreateFrontController(assemblyFilename, configFileName))
{
var discoveryVisitor = new TestDiscoveryVisitor();
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor);
discoveryVisitor.Finished.WaitOne();
controller.Run(discoveryVisitor.TestCases, resultsVisitor);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Exception e = ex;
while (e != null)
{
Log.LogError(e.GetType().FullName + ": " + e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
}
