public async Task<RunSummary> RunAsync()
{
var summary = new RunSummary();
await this.aggregator.RunAsync(async () =>
{
if (!this.cancellationTokenSource.IsCancellationRequested)
{
var testClassInstance = this.CreateScenarioClass();
if (!this.cancellationTokenSource.IsCancellationRequested)
{
await this.BeforeScenarioMethodInvokedAsync();
if (!this.cancellationTokenSource.IsCancellationRequested && !this.aggregator.HasExceptions)
{
summary.Aggregate(await this.InvokeScenarioMethodAsync(testClassInstance));
}
await this.AfterScenarioMethodInvokedAsync();
}
var disposable = testClassInstance as IDisposable;
if (disposable != null)
{
this.timer.Aggregate(() => this.aggregator.Run(disposable.Dispose));
}
}
summary.Time += this.timer.Total;
});
return summary;
}
/// <inheritdoc/>
public void Run(IEnumerable<ITestCase> testMethods, IMessageSink messageSink)
{
bool cancelled = false;
var totalSummary = new RunSummary();
string currentDirectory = Directory.GetCurrentDirectory();
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (messageSink.OnMessage(new TestAssemblyStarting()))
{
var classGroups = testMethods.Cast<XunitTestCase>().GroupBy(tc => tc.Class).ToList();
if (classGroups.Count > 0)
{
var collectionSummary = new RunSummary();
if (messageSink.OnMessage(new TestCollectionStarting()))
{
foreach (var group in classGroups)
{
var classSummary = new RunSummary();
if (!messageSink.OnMessage(new TestClassStarting { ClassName = group.Key.Name }))
cancelled = true;
else
{
cancelled = RunTestClass(messageSink, group, classSummary);
collectionSummary.Aggregate(classSummary);
}
if (!messageSink.OnMessage(new TestClassFinished { Assembly = assemblyInfo, ClassName = group.Key.Name, TestsRun = classSummary.Total }))
cancelled = true;
if (cancelled)
break;
}
}
messageSink.OnMessage(new TestCollectionFinished { Assembly = assemblyInfo, TestsRun = collectionSummary.Total });
totalSummary.Aggregate(collectionSummary);
}
}
messageSink.OnMessage(new TestAssemblyFinished
{
Assembly = assemblyInfo,
TestsRun = totalSummary.Total,
TestsFailed = totalSummary.Failed,
TestsSkipped = totalSummary.Skipped,
ExecutionTime = totalSummary.Time
});
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
/// <inheritdoc/>
public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
{
var cancellationTokenSource = new CancellationTokenSource();
var totalSummary = new RunSummary();
string currentDirectory = Directory.GetCurrentDirectory();
var ordererAttribute = assemblyInfo.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
var orderer = ordererAttribute != null ? GetTestCaseOrderer(ordererAttribute) : new DefaultTestCaseOrderer();
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (OnMessage(messageSink, new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now,
displayName,
XunitTestFrameworkDiscoverer.DisplayName)))
{
IList<RunSummary> summaries;
// TODO: Contract for Run() states that null "testCases" means "run everything".
if (disableParallelization)
{
summaries = new List<RunSummary>();
foreach (var collectionGroup in testCases.Cast<XunitTestCase>().GroupBy(tc => tc.TestCollection))
summaries.Add(RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, orderer, cancellationTokenSource));
}
else
{
var tasks = testCases.Cast<XunitTestCase>()
.GroupBy(tc => tc.TestCollection)
.Select(collectionGroup => Task.Run(() => RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, orderer, cancellationTokenSource)))
.ToArray();
summaries = await Task.WhenAll(tasks);
}
totalSummary.Time = summaries.Sum(s => s.Time);
totalSummary.Total = summaries.Sum(s => s.Total);
totalSummary.Failed = summaries.Sum(s => s.Failed);
totalSummary.Skipped = summaries.Sum(s => s.Skipped);
}
OnMessage(messageSink, new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped));
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
private void ContinueObservedTask(Task task) {
Lazy<IDisposable> disposable;
if (!_observedTasks.TryRemove(task, out disposable)) {
return;
}
if (!task.IsFaulted) {
return;
}
var runSummary = new RunSummary { Total = 1, Failed = 1, Time = (decimal)_stopwatch.Elapsed.TotalSeconds };
_messageBus.QueueMessage(new TestFailed(new XunitTest(_testCase, _testCase.DisplayName), runSummary.Time, string.Empty, task.Exception));
_runSummaryTcs.SetResult(runSummary);
}
public async Task<RunSummary> RunAsync()
{
if (!string.IsNullOrEmpty(this.skipReason))
{
this.messageBus.Queue(
this.scenario, test => new TestSkipped(test, this.skipReason), this.cancellationTokenSource);
return new RunSummary { Total = 1, Skipped = 1 };
}
else
{
var summary = new RunSummary();
var output = string.Empty;
var childAggregator = new ExceptionAggregator(this.parentAggregator);
if (!childAggregator.HasExceptions)
{
var tuple = await childAggregator.RunAsync(() => this.InvokeScenarioAsync(childAggregator));
summary.Aggregate(tuple.Item1);
output = tuple.Item2;
}
var exception = childAggregator.ToException();
if (exception != null)
{
summary.Total++;
summary.Failed++;
this.messageBus.Queue(
this.scenario,
test => new TestFailed(test, summary.Time, output, exception),
this.cancellationTokenSource);
}
else if (summary.Total == 0)
{
summary.Total++;
this.messageBus.Queue(
this.scenario, test => new TestPassed(test, summary.Time, output), this.cancellationTokenSource);
}
return summary;
}
}
protected override async Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
{
var summary = new RunSummary();
var defaultCases = new List<IXunitTestCase>();
var observationTestCases = new List<ObservationTestCase>();
foreach (var testCase in testCases)
{
if (testCase is ObservationTestCase)
observationTestCases.Add(testCase as ObservationTestCase);
else
defaultCases.Add(testCase);
}
if (observationTestCases.Any())
{
summary.Aggregate(
await new ObservationTestCollectionRunner(
testCollection,
observationTestCases,
DiagnosticMessageSink,
messageBus,
TestCaseOrderer,
new ExceptionAggregator(Aggregator),
cancellationTokenSource
)
.RunAsync()
);
}
if (defaultCases.Any())
{
summary.Aggregate(
await base.RunTestCollectionAsync(messageBus, testCollection, defaultCases, cancellationTokenSource)
);
}
return summary;
}
/// <inheritdoc/>
public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
{
var cancellationTokenSource = new CancellationTokenSource();
var totalSummary = new RunSummary();
string currentDirectory = Directory.GetCurrentDirectory();
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now,
String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version),
XunitTestFrameworkDiscoverer.DisplayName)))
{
// TODO: Contract for Run() states that null "testCases" means "run everything".
var tasks =
testCases.Cast<XunitTestCase>()
.GroupBy(tc => tc.TestCollection)
.Select(collectionGroup => Task.Run(() => RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, cancellationTokenSource)))
.ToArray();
var summaries = await Task.WhenAll(tasks);
totalSummary.Time = summaries.Sum(s => s.Time);
totalSummary.Total = summaries.Sum(s => s.Total);
totalSummary.Failed = summaries.Sum(s => s.Failed);
totalSummary.Skipped = summaries.Sum(s => s.Skipped);
}
messageSink.OnMessage(new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped));
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
protected override async Task<RunSummary> RunTestAsync()
{
var dataDiscoveryException = this.GetDataDiscoveryException();
if (dataDiscoveryException != null)
return this.RunTest_DataDiscoveryException();
var runSummary = new RunSummary();
var testRunners = this.GetTestRunners();
foreach (var testRunner in testRunners)
runSummary.Aggregate(await RunTestAsync(testRunner));
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
var cleanupAggregator = this.GetCleanupAggregator();
var toDispose = this.GetToDispose();
foreach (var disposable in toDispose)
timer.Aggregate(() => cleanupAggregator.Run(() => disposable.Dispose()));
runSummary.Time += timer.Total;
return runSummary;
}
/// <summary>
/// Runs the test.
/// </summary>
/// <returns>Returns summary information about the test that was run.</returns>
public async Task <RunSummary> RunAsync()
{
var runSummary = new RunSummary {
Total = 1
};
var output = string.Empty;
if (!MessageBus.QueueMessage(new TestStarting(Test)))
{
CancellationTokenSource.Cancel();
}
else
{
AfterTestStarting();
if (!string.IsNullOrEmpty(SkipReason))
{
runSummary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(Test, SkipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator));
runSummary.Time = tuple.Item1;
output = tuple.Item2;
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(Test, runSummary.Time, output);
}
else
{
testResult = new TestFailed(Test, runSummary.Time, output, exception);
runSummary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
Aggregator.Clear();
BeforeTestFinished();
if (Aggregator.HasExceptions)
{
if (!MessageBus.QueueMessage(new TestCleanupFailure(Test, Aggregator.ToException())))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(runSummary);
}
/// <inheritdoc/>
protected override async Task<RunSummary> RunTestAsync()
{
var testRunners = new List<XunitTestRunner>();
var toDispose = new List<IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType<IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
var test = new XunitTest(TestCase, DisplayName);
if (!MessageBus.QueueMessage(new TestStarting(test)))
CancellationTokenSource.Cancel();
else
{
if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, ex.Unwrap())))
CancellationTokenSource.Cancel();
}
if (!MessageBus.QueueMessage(new TestFinished(test, 0, null)))
CancellationTokenSource.Cancel();
return new RunSummary { Total = 1, Failed = 1 };
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
runSummary.Aggregate(await testRunner.RunAsync());
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
runSummary.Time += timer.Total;
return runSummary;
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public override async Task<RunSummary> RunAsync
#pragma warning restore CS1998
( IMessageSink diagnosticMessageSink
, IMessageBus bus
, object[] constructorArguments
, ExceptionAggregator aggregator
, CancellationTokenSource cancellationTokenSource
)
{ var summary = new RunSummary();
var test = new XunitTest(this, DisplayName);
if (Inv == null)
{ var msg = "Return Type must be Invariant.";
bus.QueueMessage
( new TestFailed(test, 0, null, new Exception(msg))
);
summary.Aggregate(new RunSummary { Total = 1, Failed = 1 });
return summary;
}
var output = new TestOutputHelper();
var timer = new ExecutionTimer();
output.Initialize(bus, test);
bus.QueueMessage(new TestStarting(test));
InvariantResult result;
timer.Aggregate(() => result = Inv.Results(Config(output)).First());
var xresult = ToXunitResult(test, result, timer.Total);
bus.QueueMessage(xresult.Item1);
summary.Aggregate(xresult.Item2);
return summary;
}
private static void RunTestClass(IMessageBus messageBus,
ITestCollection collection,
Dictionary <Type, object> collectionFixtureMappings,
IReflectionTypeInfo testClass,
IEnumerable <XunitTestCase> testCases,
ITestCaseOrderer orderer,
RunSummary classSummary,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var testClassType = testClass.Type;
var fixtureMappings = new Dictionary <Type, object>();
var constructorArguments = new List <object>();
var ordererAttribute = testClass.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
{
orderer = GetTestCaseOrderer(ordererAttribute);
}
if (testClassType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture <>)))
{
aggregator.Add(new TestClassException("A test class may not be decorated with ICollectionFixture<> (decorate the test collection class instead)."));
}
foreach (var interfaceType in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture <>)))
{
CreateFixture(interfaceType, aggregator, fixtureMappings);
}
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture <>)))
{
CreateFixture(interfaceType, aggregator, fixtureMappings);
}
}
var isStaticClass = testClassType.IsAbstract && testClassType.IsSealed;
if (!isStaticClass)
{
var ctors = testClassType.GetConstructors();
if (ctors.Length != 1)
{
aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
}
else
{
var ctor = ctors.Single();
var unusedArguments = new List <string>();
foreach (var paramInfo in ctor.GetParameters())
{
object fixture;
if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture) || collectionFixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
{
constructorArguments.Add(fixture);
}
else
{
unusedArguments.Add(String.Format("{0} {1}", paramInfo.ParameterType.Name, paramInfo.Name));
}
}
if (unusedArguments.Count > 0)
{
aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
}
}
}
var orderedTestCases = orderer.OrderTestCases(testCases);
var methodGroups = orderedTestCases.GroupBy(tc => tc.Method);
foreach (var method in methodGroups)
{
if (!messageBus.QueueMessage(new TestMethodStarting(collection, testClass.Name, method.Key.Name)))
{
cancellationTokenSource.Cancel();
}
else
{
RunTestMethod(messageBus, constructorArguments.ToArray(), method, classSummary, aggregator, cancellationTokenSource);
}
if (!messageBus.QueueMessage(new TestMethodFinished(collection, testClass.Name, method.Key.Name)))
{
cancellationTokenSource.Cancel();
}
if (cancellationTokenSource.IsCancellationRequested)
{
break;
}
}
foreach (var fixture in fixtureMappings.Values.OfType <IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageBus.QueueMessage(new ErrorMessage(ex.Unwrap())))
{
cancellationTokenSource.Cancel();
}
}
}
}
private RunSummary RunTestCollection(IMessageBus messageBus,
ITestCollection collection,
IEnumerable <XunitTestCase> testCases,
ITestCaseOrderer orderer,
CancellationTokenSource cancellationTokenSource)
{
var collectionSummary = new RunSummary();
var collectionFixtureMappings = new Dictionary <Type, object>();
var aggregator = new ExceptionAggregator();
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture <>)))
{
CreateFixture(interfaceType, aggregator, collectionFixtureMappings);
}
var ordererAttribute = collection.CollectionDefinition.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
{
orderer = GetTestCaseOrderer(ordererAttribute);
}
}
if (messageBus.QueueMessage(new TestCollectionStarting(collection)))
{
foreach (var testCasesByClass in testCases.GroupBy(tc => tc.Class))
{
var classSummary = new RunSummary();
if (!messageBus.QueueMessage(new TestClassStarting(collection, testCasesByClass.Key.Name)))
{
cancellationTokenSource.Cancel();
}
else
{
RunTestClass(messageBus, collection, collectionFixtureMappings, (IReflectionTypeInfo)testCasesByClass.Key, testCasesByClass, orderer, classSummary, aggregator, cancellationTokenSource);
collectionSummary.Aggregate(classSummary);
}
if (!messageBus.QueueMessage(new TestClassFinished(collection, testCasesByClass.Key.Name, classSummary.Time, classSummary.Total, classSummary.Failed, classSummary.Skipped)))
{
cancellationTokenSource.Cancel();
}
if (cancellationTokenSource.IsCancellationRequested)
{
break;
}
}
}
foreach (var fixture in collectionFixtureMappings.Values.OfType <IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageBus.QueueMessage(new ErrorMessage(ex.Unwrap())))
{
cancellationTokenSource.Cancel();
}
}
}
messageBus.QueueMessage(new TestCollectionFinished(collection, collectionSummary.Time, collectionSummary.Total, collectionSummary.Failed, collectionSummary.Skipped));
return(collectionSummary);
}
public async Task<RunSummary> RunScenarioAsync()
{
var runSummary = new RunSummary { Total = 1 };
var output = string.Empty;
if (!MessageBus.QueueMessage(new TestStarting(Test)))
CancellationTokenSource.Cancel();
else
{
AfterTestStarting();
if (!string.IsNullOrEmpty(SkipReason))
{
runSummary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(Test, SkipReason)))
CancellationTokenSource.Cancel();
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator));
runSummary.Time = tuple.Item1;
output = tuple.Item2;
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
testResult = new TestPassed(Test, runSummary.Time, output);
else if (exception is IgnoreException)
{
testResult = new TestSkipped(Test, exception.Message);
runSummary.Skipped++;
}
else
{
testResult = new TestFailed(Test, runSummary.Time, output, exception);
runSummary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
if (!MessageBus.QueueMessage(testResult))
CancellationTokenSource.Cancel();
}
Aggregator.Clear();
BeforeTestFinished();
if (Aggregator.HasExceptions)
if (!MessageBus.QueueMessage(new TestCleanupFailure(Test, Aggregator.ToException())))
CancellationTokenSource.Cancel();
}
if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output)))
CancellationTokenSource.Cancel();
return runSummary;
}
public void Aggregate(RunSummary other, IMetricCollector collector)
{
Aggregate(other);
TimeElapsed = collector.TimeElapsed;
MemoryDelta = collector.MemoryDelta;
}
public void Aggregate(RunSummary other)
{
Total += other.Total;
Failed += other.Failed;
Skipped += other.Skipped;
Time += other.Time;
}
private static async Task RunTestClassAsync(IMessageBus messageBus,
ITestCollection collection,
Dictionary<Type, object> collectionFixtureMappings,
IReflectionTypeInfo testClass,
IEnumerable<XunitTestCase> testCases,
ITestCaseOrderer orderer,
RunSummary classSummary,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var testClassType = testClass.Type;
var fixtureMappings = new Dictionary<Type, object>();
var constructorArguments = new List<object>();
var ordererAttribute = testClass.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
orderer = GetTestCaseOrderer(ordererAttribute);
if (testClassType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
aggregator.Add(new TestClassException("A test class may not be decorated with ICollectionFixture<> (decorate the test collection class instead)."));
foreach (var interfaceType in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
CreateFixture(interfaceType, aggregator, fixtureMappings);
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
CreateFixture(interfaceType, aggregator, fixtureMappings);
}
var isStaticClass = testClassType.IsAbstract && testClassType.IsSealed;
if (!isStaticClass)
{
var ctors = testClassType.GetConstructors();
if (ctors.Length != 1)
{
aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
}
else
{
var ctor = ctors.Single();
var unusedArguments = new List<string>();
foreach (var paramInfo in ctor.GetParameters())
{
object fixture;
if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture) || collectionFixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
constructorArguments.Add(fixture);
else
unusedArguments.Add(String.Format("{0} {1}", paramInfo.ParameterType.Name, paramInfo.Name));
}
if (unusedArguments.Count > 0)
aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
}
}
var orderedTestCases = orderer.OrderTestCases(testCases);
var methodGroups = orderedTestCases.GroupBy(tc => tc.Method);
foreach (var method in methodGroups)
{
if (!messageBus.QueueMessage(new TestMethodStarting(collection, testClass.Name, method.Key.Name)))
cancellationTokenSource.Cancel();
else
await RunTestMethodAsync(messageBus, constructorArguments.ToArray(), method, classSummary, aggregator, cancellationTokenSource);
if (!messageBus.QueueMessage(new TestMethodFinished(collection, testClass.Name, method.Key.Name)))
cancellationTokenSource.Cancel();
if (cancellationTokenSource.IsCancellationRequested)
break;
}
foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageBus.QueueMessage(new ErrorMessage(ex.Unwrap())))
cancellationTokenSource.Cancel();
}
}
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
var testRunners = new List <XunitTestRunner>();
var toDispose = new List <IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
testRunners.Add(new XunitTestRunner(TestCase, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, theoryDisplayName, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
if (!MessageBus.QueueMessage(new TestStarting(TestCase, DisplayName)))
{
CancellationTokenSource.Cancel();
}
else
{
if (!MessageBus.QueueMessage(new TestFailed(TestCase, DisplayName, 0, null, ex.Unwrap())))
{
CancellationTokenSource.Cancel();
}
}
if (!MessageBus.QueueMessage(new TestFinished(TestCase, DisplayName, 0, null)))
{
CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
{
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
private async Task<RunSummary> InvokeStepsAsync(
ICollection<IStepDefinition> backGroundStepDefinitions, ICollection<IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType<IFilter<IStepDefinition>>();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List<Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return summary;
}
private async Task<RunSummary> InvokeScenarioMethodAsync(object scenarioClassInstance)
{
var backgroundStepDefinitions = new List<IStepDefinition>();
var scenarioStepDefinitions = new List<IStepDefinition>();
await this.aggregator.RunAsync(async () =>
{
try
{
foreach (
var backgroundMethod in
this.scenario.TestCase.TestMethod.Method.Type.GetMethods(false)
.Where(candidate => candidate.GetCustomAttributes(typeof(BackgroundAttribute)).Any())
.Select(method => method.ToRuntimeMethod()))
{
await this.timer.AggregateAsync(() => backgroundMethod.InvokeAsync(scenarioClassInstance, null));
}
backgroundStepDefinitions.AddRange(CurrentThread.StepDefinitions);
}
catch (Exception ex)
{
}
finally
{
CurrentThread.StepDefinitions.Clear();
}
try
{
await this.timer.AggregateAsync(() =>
this.scenarioMethod.InvokeAsync(scenarioClassInstance, this.scenarioMethodArguments));
scenarioStepDefinitions.AddRange(CurrentThread.StepDefinitions);
}
finally
{
CurrentThread.StepDefinitions.Clear();
}
});
var runSummary = new RunSummary { Time = this.timer.Total };
if (!this.aggregator.HasExceptions)
{
runSummary.Aggregate(await this.InvokeStepsAsync(backgroundStepDefinitions, scenarioStepDefinitions));
}
return runSummary;
}
protected override async Task<RunSummary> RunTestAsync()
{
if (this.dataDiscoveryException != null)
{
this.MessageBus.Queue(
new XunitTest(this.TestCase, this.DisplayName),
test => new TestFailed(test, 0, null, this.dataDiscoveryException.Unwrap()),
this.CancellationTokenSource);
return new RunSummary { Total = 1, Failed = 1 };
}
var summary = new RunSummary();
foreach (var scenarioRunner in this.scenarioRunners)
{
summary.Aggregate(await scenarioRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in this.disposables)
{
timer.Aggregate(() => this.cleanupAggregator.Run(() => disposable.Dispose()));
}
summary.Time += timer.Total;
return summary;
}
private async Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus,
ITestCollection collection,
IEnumerable<XunitTestCase> testCases,
ITestCaseOrderer orderer,
CancellationTokenSource cancellationTokenSource)
{
var collectionSummary = new RunSummary();
var collectionFixtureMappings = new Dictionary<Type, object>();
var aggregator = new ExceptionAggregator();
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
CreateFixture(interfaceType, aggregator, collectionFixtureMappings);
var ordererAttribute = collection.CollectionDefinition.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
orderer = GetTestCaseOrderer(ordererAttribute);
}
if (messageBus.QueueMessage(new TestCollectionStarting(collection)))
{
foreach (var testCasesByClass in testCases.GroupBy(tc => tc.Class))
{
var classSummary = new RunSummary();
if (!messageBus.QueueMessage(new TestClassStarting(collection, testCasesByClass.Key.Name)))
cancellationTokenSource.Cancel();
else
{
await RunTestClassAsync(messageBus, collection, collectionFixtureMappings, (IReflectionTypeInfo)testCasesByClass.Key, testCasesByClass, orderer, classSummary, aggregator, cancellationTokenSource);
collectionSummary.Aggregate(classSummary);
}
if (!messageBus.QueueMessage(new TestClassFinished(collection, testCasesByClass.Key.Name, classSummary.Time, classSummary.Total, classSummary.Failed, classSummary.Skipped)))
cancellationTokenSource.Cancel();
if (cancellationTokenSource.IsCancellationRequested)
break;
}
}
foreach (var fixture in collectionFixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageBus.QueueMessage(new ErrorMessage(ex.Unwrap())))
cancellationTokenSource.Cancel();
}
}
messageBus.QueueMessage(new TestCollectionFinished(collection, collectionSummary.Time, collectionSummary.Total, collectionSummary.Failed, collectionSummary.Skipped));
return collectionSummary;
}
private static bool RunTestClass(IMessageSink messageSink, IGrouping<ITypeInfo, XunitTestCase> group, RunSummary classSummary)
{
bool cancelled = false;
var aggregator = new ExceptionAggregator();
Type testClassType = ((IReflectionTypeInfo)group.Key).Type;
Dictionary<Type, object> fixtureMappings = new Dictionary<Type, object>();
List<object> constructorArguments = new List<object>();
// TODO: Read class fixtures from test collection
foreach (var iface in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
{
Type fixtureType = iface.GetGenericArguments().Single();
object fixture = null;
aggregator.Run(() => fixture = Activator.CreateInstance(fixtureType));
fixtureMappings.Add(fixtureType, fixture);
}
var ctors = testClassType.GetConstructors();
if (ctors.Length != 1)
{
aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
}
else
{
var ctor = ctors.Single();
List<string> unusedArguments = new List<string>();
foreach (var paramInfo in ctor.GetParameters())
{
object fixture;
if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
constructorArguments.Add(fixture);
else
unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name);
}
if (unusedArguments.Count > 0)
aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
}
var methodGroups = group.GroupBy(tc => tc.Method);
foreach (var method in methodGroups)
{
if (!messageSink.OnMessage(new TestMethodStarting { ClassName = group.Key.Name, MethodName = method.Key.Name }))
cancelled = true;
else
cancelled = RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator);
if (!messageSink.OnMessage(new TestMethodFinished { ClassName = group.Key.Name, MethodName = method.Key.Name }))
cancelled = true;
if (cancelled)
break;
}
foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
cancelled = true;
}
}
return cancelled;
}
private static async Task RunTestMethodAsync(IMessageBus messageBus,
object[] constructorArguments,
IEnumerable<XunitTestCase> testCases,
RunSummary classSummary,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
foreach (var testCase in testCases)
{
using (var delegatingBus = new DelegatingMessageBus<ITestCaseFinished>(messageBus))
{
await testCase.RunAsync(delegatingBus, constructorArguments, aggregator, cancellationTokenSource);
delegatingBus.Finished.WaitOne();
classSummary.Total += delegatingBus.FinalMessage.TestsRun;
classSummary.Failed += delegatingBus.FinalMessage.TestsFailed;
classSummary.Skipped += delegatingBus.FinalMessage.TestsSkipped;
classSummary.Time += delegatingBus.FinalMessage.ExecutionTime;
}
if (cancellationTokenSource.IsCancellationRequested)
break;
}
}
private static bool RunTestMethod(IMessageSink messageSink, object[] constructorArguments, IEnumerable<XunitTestCase> testCases, RunSummary classSummary, ExceptionAggregator aggregator)
{
bool cancelled = false;
foreach (XunitTestCase testCase in testCases)
{
var delegatingSink = new DelegatingMessageSink<ITestCaseFinished>(messageSink);
// REVIEW: testCase.Run() returning bool implies synchronous behavior, which will probably
// not be true once we start supporting parallelization. This could be achieved by always
// using a delegating sink (like above) and watching for cancellation there, then checking
// for the cancellation result in the delegating sink after work is finished.
cancelled = testCase.Run(delegatingSink, constructorArguments, aggregator);
delegatingSink.Finished.WaitOne();
classSummary.Total += delegatingSink.FinalMessage.TestsRun;
classSummary.Failed += delegatingSink.FinalMessage.TestsFailed;
classSummary.Skipped += delegatingSink.FinalMessage.TestsSkipped;
classSummary.Time += delegatingSink.FinalMessage.ExecutionTime;
if (cancelled)
break;
}
return cancelled;
}
/// <inheritdoc/>
public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink, ITestFrameworkOptions executionOptions)
{
Guard.ArgumentNotNull("testCases", testCases);
Guard.ArgumentNotNull("messageSink", messageSink);
Guard.ArgumentNotNull("executionOptions", executionOptions);
var disableParallelization = false;
var maxParallelThreads = 0;
var collectionBehaviorAttribute = assemblyInfo.GetCustomAttributes(typeof(CollectionBehaviorAttribute)).SingleOrDefault();
if (collectionBehaviorAttribute != null)
{
disableParallelization = collectionBehaviorAttribute.GetNamedArgument<bool>("DisableTestParallelization");
maxParallelThreads = collectionBehaviorAttribute.GetNamedArgument<int>("MaxParallelThreads");
}
disableParallelization = executionOptions.GetValue<bool>(TestOptionsNames.Execution.DisableParallelization, disableParallelization);
var maxParallelThreadsOption = executionOptions.GetValue<int>(TestOptionsNames.Execution.MaxParallelThreads, 0);
if (maxParallelThreadsOption > 0)
maxParallelThreads = maxParallelThreadsOption;
var displayName = GetDisplayName(collectionBehaviorAttribute, disableParallelization, maxParallelThreads);
var cancellationTokenSource = new CancellationTokenSource();
var totalSummary = new RunSummary();
var scheduler = maxParallelThreads > 0 ? new MaxConcurrencyTaskScheduler(maxParallelThreads) : TaskScheduler.Current;
string currentDirectory = Directory.GetCurrentDirectory();
var ordererAttribute = assemblyInfo.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
var orderer = ordererAttribute != null ? GetTestCaseOrderer(ordererAttribute) : new DefaultTestCaseOrderer();
using (var messageBus = new MessageBus(messageSink))
{
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (messageBus.QueueMessage(new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now,
displayName, XunitTestFrameworkDiscoverer.DisplayName)))
{
IList<RunSummary> summaries;
// TODO: Contract for Run() states that null "testCases" means "run everything".
var masterStopwatch = Stopwatch.StartNew();
if (disableParallelization)
{
summaries = new List<RunSummary>();
foreach (var collectionGroup in testCases.Cast<XunitTestCase>().GroupBy(tc => tc.TestCollection))
summaries.Add(await RunTestCollectionAsync(messageBus, collectionGroup.Key, collectionGroup, orderer, cancellationTokenSource));
}
else
{
var tasks = testCases.Cast<XunitTestCase>()
.GroupBy(tc => tc.TestCollection)
.Select(collectionGroup => Task.Factory.StartNew(() => RunTestCollectionAsync(messageBus, collectionGroup.Key, collectionGroup, orderer, cancellationTokenSource),
cancellationTokenSource.Token,
TaskCreationOptions.None,
scheduler))
.ToArray();
summaries = await Task.WhenAll(tasks.Select(t => t.Unwrap()));
}
totalSummary.Time = (decimal)masterStopwatch.Elapsed.TotalSeconds;
totalSummary.Total = summaries.Sum(s => s.Total);
totalSummary.Failed = summaries.Sum(s => s.Failed);
totalSummary.Skipped = summaries.Sum(s => s.Skipped);
}
}
finally
{
messageBus.QueueMessage(new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped));
Directory.SetCurrentDirectory(currentDirectory);
}
}
}
private static void RunTestMethod(IMessageSink messageSink,
object[] constructorArguments,
IEnumerable<XunitTestCase> testCases,
RunSummary classSummary,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
foreach (XunitTestCase testCase in testCases)
{
var delegatingSink = new DelegatingMessageSink<ITestCaseFinished>(messageSink);
testCase.Run(delegatingSink, constructorArguments, aggregator, cancellationTokenSource);
delegatingSink.Finished.WaitOne();
classSummary.Total += delegatingSink.FinalMessage.TestsRun;
classSummary.Failed += delegatingSink.FinalMessage.TestsFailed;
classSummary.Skipped += delegatingSink.FinalMessage.TestsSkipped;
classSummary.Time += delegatingSink.FinalMessage.ExecutionTime;
if (cancellationTokenSource.IsCancellationRequested)
break;
}
}
private Task<RunSummary> CreateObserveTask(Task failedObservedTask, IMessageBus messageBus, CancellationTokenSource cancellationTokenSource) {
var tcs = new TaskCompletionSource<RunSummary>();
var stopwatch = Stopwatch.StartNew();
failedObservedTask.ContinueWith(t => {
stopwatch.Stop();
if (!t.IsFaulted) {
return;
}
var runSummary = new RunSummary { Total = 1, Failed = 1, Time = (decimal)stopwatch.Elapsed.TotalSeconds };
messageBus.QueueMessage(new TestFailed(new XunitTest(TestCase, DisplayName), runSummary.Time, string.Empty, t.Exception));
cancellationTokenSource.Cancel();
tcs.SetResult(runSummary);
}, TaskContinuationOptions.ExecuteSynchronously);
return tcs.Task;
}
/// <inheritdoc/>
public virtual async Task<RunSummary> RunAsync(IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
var summary = new RunSummary();
if (!messageBus.QueueMessage(new TestCaseStarting(this)))
cancellationTokenSource.Cancel();
else
{
using (var delegatingBus = new DelegatingMessageBus(messageBus,
msg =>
{
if (msg is ITestResultMessage)
{
summary.Total++;
summary.Time += ((ITestResultMessage)msg).ExecutionTime;
}
if (msg is ITestFailed)
summary.Failed++;
if (msg is ITestSkipped)
summary.Skipped++;
}))
{
await RunTestsAsync(delegatingBus, constructorArguments, aggregator, cancellationTokenSource);
}
}
if (!messageBus.QueueMessage(new TestCaseFinished(this, summary.Time, summary.Total, summary.Failed, summary.Skipped)))
cancellationTokenSource.Cancel();
return summary;
}