/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCaseRunner"/> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public XunitTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, messageBus, aggregator, cancellationTokenSource)
{
DisplayName = displayName;
SkipReason = skipReason;
ConstructorArguments = constructorArguments;
TestClass = TestCase.TestMethod.TestClass.Class.ToRuntimeType();
TestMethod = TestCase.Method.ToRuntimeMethod();
ParameterInfo[] parameters = TestMethod.GetParameters();
Type[] parameterTypes = new Type[parameters.Length];
for (int i = 0; i < parameters.Length; i++)
{
parameterTypes[i] = parameters[i].ParameterType;
}
TestMethodArguments = Reflector.ConvertArguments(testMethodArguments, parameterTypes);
beforeAfterAttributes =
TestClass.GetTypeInfo().GetCustomAttributes(typeof(BeforeAfterTestAttribute))
.Concat(TestMethod.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Cast <BeforeAfterTestAttribute>()
.ToList();
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCaseRunner"/> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public XunitTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, messageBus, aggregator, cancellationTokenSource)
{
DisplayName = displayName;
SkipReason = skipReason;
ConstructorArguments = constructorArguments;
TestClass = GetRuntimeClass();
TestMethod = GetRuntimeMethod();
var parameterTypes = TestMethod.GetParameters().Select(p => p.ParameterType).ToArray();
TestMethodArguments = Reflector.ConvertArguments(testMethodArguments, parameterTypes);
beforeAfterAttributes =
TestClass.GetTypeInfo().GetCustomAttributes(typeof(BeforeAfterTestAttribute))
.Concat(TestMethod.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Cast <BeforeAfterTestAttribute>()
.ToList();
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCaseRunner"/> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public XunitTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, messageBus, aggregator, cancellationTokenSource)
{
DisplayName = displayName;
SkipReason = skipReason;
ConstructorArguments = constructorArguments;
TestClass = TestCase.TestMethod.TestClass.Class.ToRuntimeType();
TestMethod = TestCase.Method.ToRuntimeMethod();
var parameters = TestMethod.GetParameters();
var parameterTypes = new Type[parameters.Length];
for (int i = 0; i < parameters.Length; i++)
{
parameterTypes[i] = parameters[i].ParameterType;
}
TestMethodArguments = Reflector.ConvertArguments(testMethodArguments, parameterTypes);
}
/// <inheritdoc/>
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererType);
IEnumerable <object[]> data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method);
if (data == null)
{
Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called."));
continue;
}
foreach (var dataRow in data)
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow);
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes);
var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip");
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
dataDiscoveryException = ex;
}
}
Attribute Instantiate(CustomAttributeData attributeData)
{
var ctorArgs = GetConstructorArguments().ToArray();
var ctorArgTypes = Reflector.EmptyTypes;
if (ctorArgs.Length > 0)
{
ctorArgTypes = new Type[attributeData.ConstructorArguments.Count];
for (var i = 0; i < ctorArgTypes.Length; i++)
{
ctorArgTypes[i] = attributeData.ConstructorArguments[i].ArgumentType;
}
}
var attribute = (Attribute?)attributeData.Constructor.Invoke(Reflector.ConvertArguments(ctorArgs, ctorArgTypes));
if (attribute == null)
{
throw new ArgumentException($"Unable to create attribute of type '{attributeData.AttributeType.FullName}'", nameof(attributeData));
}
var attributeType = attribute.GetType();
for (var i = 0; i < attributeData.NamedArguments.Count; i++)
{
var namedArg = attributeData.NamedArguments[i];
var typedValue = GetTypedValue(namedArg.TypedValue);
var memberName = namedArg.MemberName;
var propInfo = attributeType.GetRuntimeProperty(memberName);
if (propInfo != null)
{
propInfo.SetValue(attribute, typedValue);
}
else
{
var fieldInfo = attributeType.GetRuntimeField(memberName);
if (fieldInfo != null)
{
fieldInfo.SetValue(attribute, typedValue);
}
else
{
throw new ArgumentException($"Could not find property or field named '{memberName}' on instance of '{Attribute.GetType().FullName}'", nameof(attributeData));
}
}
}
return(attribute);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCaseRunner"/> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public XunitTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, messageBus, aggregator, cancellationTokenSource)
{
DisplayName = displayName;
SkipReason = skipReason;
ConstructorArguments = constructorArguments;
TestClass = TestCase.TestMethod.TestClass.Class.ToRuntimeType();
TestMethod = TestCase.Method.ToRuntimeMethod();
var parameters = TestMethod.GetParameters();
var parameterTypes = new Type[parameters.Length];
for (int i = 0; i < parameters.Length; i++)
{
parameterTypes[i] = parameters[i].ParameterType;
}
testMethodArguments = TypeUtility.ResolveMethodArguments(TestMethod, testMethodArguments);
TestMethodArguments = Reflector.ConvertArguments(testMethodArguments, parameterTypes);
IEnumerable <Attribute> beforeAfterTestCollectionAttributes;
var collectionDefinition = testCase.TestMethod.TestClass.TestCollection.CollectionDefinition as IReflectionTypeInfo;
if (collectionDefinition != null)
{
beforeAfterTestCollectionAttributes = collectionDefinition.Type.GetTypeInfo().GetCustomAttributes(typeof(BeforeAfterTestAttribute));
}
else
{
beforeAfterTestCollectionAttributes = Enumerable.Empty <Attribute>();
}
beforeAfterAttributes =
beforeAfterTestCollectionAttributes
.Concat(TestClass.GetTypeInfo().GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Concat(TestMethod.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Cast <BeforeAfterTestAttribute>()
.ToList();
}
/// <inheritdoc/>
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
dataDiscoveryException = ex;
}
}
Attribute Instantiate(CustomAttributeData attributeData)
{
var ctorArgs = GetConstructorArguments().ToArray();
var ctorArgTypes = attributeData.ConstructorArguments.Select(ci => ci.ArgumentType).ToArray();
var attribute = (Attribute)Activator.CreateInstance(attributeData.AttributeType, Reflector.ConvertArguments(ctorArgs, ctorArgTypes));
var ati = attribute.GetType();
foreach (var namedArg in attributeData.NamedArguments)
{
(ati.GetRuntimeProperty(namedArg.MemberName)).SetValue(attribute, GetTypedValue(namedArg.TypedValue), index: null);
}
return(attribute);
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
var testRunners = new List <XunitTestRunner>();
var toDispose = new List <IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
testRunners.Add(new XunitTestRunner(TestCase, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, theoryDisplayName, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
if (!MessageBus.QueueMessage(new TestStarting(TestCase, DisplayName)))
{
CancellationTokenSource.Cancel();
}
else
{
if (!MessageBus.QueueMessage(new TestFailed(TestCase, DisplayName, 0, null, ex.Unwrap())))
{
CancellationTokenSource.Cancel();
}
}
if (!MessageBus.QueueMessage(new TestFinished(TestCase, DisplayName, 0, null)))
{
CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
{
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <summary>
/// Runs a single test for a given test method.
/// </summary>
/// <param name="messageBus">The message bus to send results to.</param>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="constructorArguments">The arguments to pass to the constructor.</param>
/// <param name="methodUnderTest">The method under test.</param>
/// <param name="testMethodArguments">The arguments to pass to the test method.</param>
/// <param name="displayName">The display name for the test.</param>
/// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param>
/// <param name="parentAggregator">The parent aggregator that contains the exceptions up to this point.</param>
/// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param>
protected async Task <decimal> RunTestWithArgumentsAsync(IMessageBus messageBus,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
object[] testMethodArguments,
string displayName,
List <BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator parentAggregator,
CancellationTokenSource cancellationTokenSource)
{
var executionTimeInSeconds = 0.0m;
var aggregator = new ExceptionAggregator(parentAggregator);
var output = String.Empty; // TODO: Add output facilities for v2
if (!messageBus.QueueMessage(new TestStarting(this, displayName)))
{
cancellationTokenSource.Cancel();
}
else
{
if (!String.IsNullOrEmpty(SkipReason))
{
if (!messageBus.QueueMessage(new TestSkipped(this, displayName, SkipReason)))
{
cancellationTokenSource.Cancel();
}
}
else
{
var beforeAttributesRun = new Stack <BeforeAfterTestAttribute>();
var executionTime = new ExecutionTime();
if (!aggregator.HasExceptions)
{
await aggregator.RunAsync(async() =>
{
object testClass = null;
if (!methodUnderTest.IsStatic)
{
if (!messageBus.QueueMessage(new TestClassConstructionStarting(this, displayName)))
{
cancellationTokenSource.Cancel();
}
try
{
if (!cancellationTokenSource.IsCancellationRequested)
{
executionTime.Aggregate(() => testClass = Activator.CreateInstance(classUnderTest, constructorArguments));
}
}
finally
{
if (!messageBus.QueueMessage(new TestClassConstructionFinished(this, displayName)))
{
cancellationTokenSource.Cancel();
}
}
}
if (!cancellationTokenSource.IsCancellationRequested)
{
await aggregator.RunAsync(async() =>
{
foreach (var beforeAfterAttribute in beforeAfterAttributes)
{
var attributeName = beforeAfterAttribute.GetType().Name;
if (!messageBus.QueueMessage(new BeforeTestStarting(this, displayName, attributeName)))
{
cancellationTokenSource.Cancel();
}
else
{
try
{
executionTime.Aggregate(() => beforeAfterAttribute.Before(methodUnderTest));
beforeAttributesRun.Push(beforeAfterAttribute);
}
finally
{
if (!messageBus.QueueMessage(new BeforeTestFinished(this, displayName, attributeName)))
{
cancellationTokenSource.Cancel();
}
}
}
if (cancellationTokenSource.IsCancellationRequested)
{
return;
}
}
if (!cancellationTokenSource.IsCancellationRequested)
{
var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray();
var oldSyncContext = SynchronizationContext.Current;
try
{
var asyncSyncContext = new AsyncTestSyncContext();
SetSynchronizationContext(asyncSyncContext);
await aggregator.RunAsync(async() =>
{
await executionTime.AggregateAsync(async() =>
{
var result = methodUnderTest.Invoke(testClass, Reflector.ConvertArguments(testMethodArguments, parameterTypes));
var task = result as Task;
if (task != null)
{
await task;
}
else
{
var ex = await asyncSyncContext.WaitForCompletionAsync();
if (ex != null)
{
aggregator.Add(ex);
}
}
});
});
}
finally
{
SetSynchronizationContext(oldSyncContext);
}
}
});
foreach (var beforeAfterAttribute in beforeAttributesRun)
{
var attributeName = beforeAfterAttribute.GetType().Name;
if (!messageBus.QueueMessage(new AfterTestStarting(this, displayName, attributeName)))
{
cancellationTokenSource.Cancel();
}
aggregator.Run(() => executionTime.Aggregate(() => beforeAfterAttribute.After(methodUnderTest)));
if (!messageBus.QueueMessage(new AfterTestFinished(this, displayName, attributeName)))
{
cancellationTokenSource.Cancel();
}
}
}
aggregator.Run(() =>
{
var disposable = testClass as IDisposable;
if (disposable != null)
{
if (!messageBus.QueueMessage(new TestClassDisposeStarting(this, displayName)))
{
cancellationTokenSource.Cancel();
}
try
{
executionTime.Aggregate(disposable.Dispose);
}
finally
{
if (!messageBus.QueueMessage(new TestClassDisposeFinished(this, displayName)))
{
cancellationTokenSource.Cancel();
}
}
}
});
});
}
if (!cancellationTokenSource.IsCancellationRequested)
{
executionTimeInSeconds = (decimal)executionTime.Total.TotalSeconds;
var exception = aggregator.ToException();
var testResult = exception == null ? (TestResultMessage) new TestPassed(this, displayName, executionTimeInSeconds, output)
: new TestFailed(this, displayName, executionTimeInSeconds, output, exception);
if (!messageBus.QueueMessage(testResult))
{
cancellationTokenSource.Cancel();
}
}
}
}
if (!messageBus.QueueMessage(new TestFinished(this, displayName, executionTimeInSeconds, output)))
{
cancellationTokenSource.Cancel();
}
return(executionTimeInSeconds);
}
Attribute Instantiate(CustomAttributeData attributeData)
{
var ctorArgs = GetConstructorArguments().ToArray();
Type[] ctorArgTypes = Reflector.EmptyTypes;
if (ctorArgs.Length > 0)
{
ctorArgTypes = new Type[attributeData.ConstructorArguments.Count];
for (int i = 0; i < ctorArgTypes.Length; i++)
{
ctorArgTypes[i] = attributeData.ConstructorArguments[i].ArgumentType;
}
}
var attribute = (Attribute)Activator.CreateInstance(attributeData.AttributeType, Reflector.ConvertArguments(ctorArgs, ctorArgTypes));
var ati = attribute.GetType();
for (int i = 0; i < attributeData.NamedArguments.Count; i++)
{
var namedArg = attributeData.NamedArguments[i];
(ati.GetRuntimeProperty(namedArg.MemberName)).SetValue(attribute, GetTypedValue(namedArg.TypedValue), index: null);
}
return(attribute);
}
Attribute Instantiate(CustomAttributeData attributeData)
{
var ctorArgs = GetConstructorArguments().ToArray();
var ctorArgTypes = attributeData.Constructor.GetParameters().Select(p => p.ParameterType).ToArray();
var attribute = (Attribute)Activator.CreateInstance(attributeData.Constructor.ReflectedType, Reflector.ConvertArguments(ctorArgs, ctorArgTypes));
foreach (var namedArg in attributeData.NamedArguments)
{
((PropertyInfo)namedArg.MemberInfo).SetValue(attribute, namedArg.TypedValue.Value, index: null);
}
return(attribute);
}
