/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
if (dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in toDispose)
{
timer.Aggregate(() => cleanupAggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <summary>
/// Creates an instance of the test class for the given test case. Sends the <see cref="ITestClassConstructionStarting"/>
/// and <see cref="ITestClassConstructionFinished"/> messages as appropriate.
/// </summary>
/// <param name="testCase">The test case</param>
/// <param name="testClassType">The type of the test class</param>
/// <param name="constructorArguments">The constructor arguments for the test class</param>
/// <param name="displayName">The display name of the test case</param>
/// <param name="messageBus">The message bus used to send the test messages</param>
/// <param name="timer">The timer used to measure the time taken for construction</param>
/// <param name="cancellationTokenSource">The cancellation token source</param>
/// <returns></returns>
public static object CreateTestClass(this ITestCase testCase,
Type testClassType,
object[] constructorArguments,
string displayName,
IMessageBus messageBus,
ExecutionTimer timer,
CancellationTokenSource cancellationTokenSource)
{
object testClass = null;
if (!messageBus.QueueMessage(new TestClassConstructionStarting(testCase, displayName)))
cancellationTokenSource.Cancel();
try
{
if (!cancellationTokenSource.IsCancellationRequested)
timer.Aggregate(() => testClass = Activator.CreateInstance(testClassType, constructorArguments));
}
finally
{
if (!messageBus.QueueMessage(new TestClassConstructionFinished(testCase, displayName)))
cancellationTokenSource.Cancel();
}
return testClass;
}
/// <inheritdoc/>
protected override Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName);
var summary = new RunSummary {
Total = 1
};
var timer = new ExecutionTimer();
if (!MessageBus.QueueMessage(new TestStarting(test)))
{
CancellationTokenSource.Cancel();
}
else
{
try
{
timer.Aggregate(TestCase.Lambda);
if (!MessageBus.QueueMessage(new TestPassed(test, timer.Total, null)))
{
CancellationTokenSource.Cancel();
}
}
catch (Exception ex)
{
summary.Failed++;
if (!MessageBus.QueueMessage(new TestFailed(test, timer.Total, null, ex)))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, timer.Total, null)))
{
CancellationTokenSource.Cancel();
}
summary.Time = timer.Total;
return(Task.FromResult(summary));
}
public bool AfterStarting(IMessageBus messageBus, TestStarting message)
{
_timer = new ExecutionTimer();
var testMethod = message.TestMethod.Method.ToRuntimeMethod();
foreach (var beforeAfterTestCaseAttribute in _beforeAfterAttributes)
{
try
{
_timer.Aggregate(() => beforeAfterTestCaseAttribute.Before(testMethod));
_attributesStack.Push(beforeAfterTestCaseAttribute);
}
catch (Exception e)
{
_exceptions.Add(e);
}
}
return true;
}
/// <summary>
/// Initializes a new instance of the <see cref="TestInvoker{TTestCase}"/> class.
/// </summary>
/// <param name="testCase">The test case that this invocation belongs to.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="testClass">The test class that the test method belongs to.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethod">The test method that will be invoked.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="displayName">The display name for this test invocation.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public TestInvoker(TTestCase testCase,
IMessageBus messageBus,
Type testClass,
object[] constructorArguments,
MethodInfo testMethod,
object[] testMethodArguments,
string displayName,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
TestCase = testCase;
MessageBus = messageBus;
TestClass = testClass;
ConstructorArguments = constructorArguments;
TestMethod = testMethod;
TestMethodArguments = testMethodArguments;
DisplayName = displayName;
Aggregator = aggregator;
CancellationTokenSource = cancellationTokenSource;
Timer = new ExecutionTimer();
}
/// <summary>
/// Initializes a new instance of the <see cref="TestInvoker{TTestCase}"/> class.
/// </summary>
/// <param name="test">The test that this invocation belongs to.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="testClass">The test class that the test method belongs to.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethod">The test method that will be invoked.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public TestInvoker(ITest test,
IMessageBus messageBus,
Type testClass,
object[] constructorArguments,
MethodInfo testMethod,
object[] testMethodArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
Guard.ArgumentNotNull("test", test);
Guard.ArgumentValid("test", "test.TestCase must implement " + typeof(TTestCase).FullName, test.TestCase is TTestCase);
Test = test;
MessageBus = messageBus;
TestClass = testClass;
ConstructorArguments = constructorArguments;
TestMethod = testMethod;
TestMethodArguments = testMethodArguments;
Aggregator = aggregator;
CancellationTokenSource = cancellationTokenSource;
Timer = new ExecutionTimer();
}
/// <summary>
/// Disposes the test class instance. Sends the <see cref="ITestClassDisposeStarting"/> and <see cref="ITestClassDisposeFinished"/>
/// messages as appropriate.
/// </summary>
/// <param name="test">The test</param>
/// <param name="testClass">The test class instance to be disposed</param>
/// <param name="messageBus">The message bus used to send the test messages</param>
/// <param name="timer">The timer used to measure the time taken for construction</param>
/// <param name="cancellationTokenSource">The cancellation token source</param>
public static void DisposeTestClass(this ITest test,
object testClass,
IMessageBus messageBus,
ExecutionTimer timer,
CancellationTokenSource cancellationTokenSource)
{
var disposable = testClass as IDisposable;
if (disposable == null)
return;
if (!messageBus.QueueMessage(new TestClassDisposeStarting(test)))
cancellationTokenSource.Cancel();
try
{
timer.Aggregate(disposable.Dispose);
}
finally
{
if (!messageBus.QueueMessage(new TestClassDisposeFinished(test)))
cancellationTokenSource.Cancel();
}
}
protected override async Task<RunSummary> RunTestAsync()
{
var dataDiscoveryException = this.GetDataDiscoveryException();
if (dataDiscoveryException != null)
return this.RunTest_DataDiscoveryException();
var runSummary = new RunSummary();
var testRunners = this.GetTestRunners();
foreach (var testRunner in testRunners)
runSummary.Aggregate(await RunTestAsync(testRunner));
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
var cleanupAggregator = this.GetCleanupAggregator();
var toDispose = this.GetToDispose();
foreach (var disposable in toDispose)
timer.Aggregate(() => cleanupAggregator.Run(() => disposable.Dispose()));
runSummary.Time += timer.Total;
return runSummary;
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
var testRunners = new List <XunitTestRunner>();
var toDispose = new List <IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
testRunners.Add(new XunitTestRunner(TestCase, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, theoryDisplayName, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
if (!MessageBus.QueueMessage(new TestStarting(TestCase, DisplayName)))
{
CancellationTokenSource.Cancel();
}
else
{
if (!MessageBus.QueueMessage(new TestFailed(TestCase, DisplayName, 0, null, ex.Unwrap())))
{
CancellationTokenSource.Cancel();
}
}
if (!MessageBus.QueueMessage(new TestFinished(TestCase, DisplayName, 0, null)))
{
CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
{
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <inheritdoc/>
protected override async Task<RunSummary> RunTestAsync()
{
var testRunners = new List<XunitTestRunner>();
var toDispose = new List<IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType<IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
var test = new XunitTest(TestCase, DisplayName);
if (!MessageBus.QueueMessage(new TestStarting(test)))
CancellationTokenSource.Cancel();
else
{
if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, ex.Unwrap())))
CancellationTokenSource.Cancel();
}
if (!MessageBus.QueueMessage(new TestFinished(test, 0, null)))
CancellationTokenSource.Cancel();
return new RunSummary { Total = 1, Failed = 1 };
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
runSummary.Aggregate(await testRunner.RunAsync());
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
runSummary.Time += timer.Total;
return runSummary;
}
private async Task<RunSummary> InvokeStepsAsync(
ICollection<IStepDefinition> backGroundStepDefinitions, ICollection<IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType<IFilter<IStepDefinition>>();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List<Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return summary;
}
protected override async Task<RunSummary> RunTestAsync()
{
if (this.dataDiscoveryException != null)
{
this.MessageBus.Queue(
new XunitTest(this.TestCase, this.DisplayName),
test => new TestFailed(test, 0, null, this.dataDiscoveryException.Unwrap()),
this.CancellationTokenSource);
return new RunSummary { Total = 1, Failed = 1 };
}
var summary = new RunSummary();
foreach (var scenarioRunner in this.scenarioRunners)
{
summary.Aggregate(await scenarioRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in this.disposables)
{
timer.Aggregate(() => this.cleanupAggregator.Run(() => disposable.Dispose()));
}
summary.Time += timer.Total;
return summary;
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public override async Task<RunSummary> RunAsync
#pragma warning restore CS1998
( IMessageSink diagnosticMessageSink
, IMessageBus bus
, object[] constructorArguments
, ExceptionAggregator aggregator
, CancellationTokenSource cancellationTokenSource
)
{ var summary = new RunSummary();
var test = new XunitTest(this, DisplayName);
if (Inv == null)
{ var msg = "Return Type must be Invariant.";
bus.QueueMessage
( new TestFailed(test, 0, null, new Exception(msg))
);
summary.Aggregate(new RunSummary { Total = 1, Failed = 1 });
return summary;
}
var output = new TestOutputHelper();
var timer = new ExecutionTimer();
output.Initialize(bus, test);
bus.QueueMessage(new TestStarting(test));
InvariantResult result;
timer.Aggregate(() => result = Inv.Results(Config(output)).First());
var xresult = ToXunitResult(test, result, timer.Total);
bus.QueueMessage(xresult.Item1);
summary.Aggregate(xresult.Item2);
return summary;
}