/// <inheritdoc/>
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
// Special case Skip, because we want a single Skip (not one per data item), and a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
if (factAttribute.GetNamedArgument <string>("Skip") != null)
{
return new[] { new XunitTestCase(defaultMethodDisplay, testMethod) }
}
;
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var results = new List <XunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute);
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) }
}
;
// GetData may return null, but that's okay; we'll let the NullRef happen and then catch it
// down below so that we get the composite test case.
foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method))
{
// Attempt to serialize the test case, since we need a way to uniquely identify a test
// and serialization is the best way to do that. If it's not serializable, this will
// throw and we will fall back to a single theory test case that gets its data
// at runtime.
var testCase = new XunitTestCase(defaultMethodDisplay, testMethod, dataRow);
SerializationHelper.Serialize(testCase);
results.Add(testCase);
}
}
// REVIEW: Could we re-write LambdaTestCase to just be for exceptions?
if (results.Count == 0)
{
results.Add(new LambdaTestCase(defaultMethodDisplay, testMethod,
() => { throw new InvalidOperationException(String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)); }));
}
return(results);
}
catch { }
}
return(new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) });
}
}
/// <inheritdoc/>
public IEnumerable <IXunitTestCase> Discover(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo testClass, IMethodInfo testMethod, IAttributeInfo factAttribute)
{
// Special case Skip, because we want a single Skip (not one per data item), and a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
if (factAttribute.GetNamedArgument <string>("Skip") != null)
{
return new[] { new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute) }
}
;
try
{
using (var memoryStream = new MemoryStream())
{
var results = new List <XunitTestCase>();
var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
// GetData may return null, but that's okay; we'll let the NullRef happen and then catch it
// down below so that we get the composite test case.
foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
{
// Attempt to serialize the test case, since we need a way to uniquely identify a test
// and serialization is the best way to do that. If it's not serializable, this will
// throw and we will fall back to a single theory test case that gets its data
// at runtime.
var testCase = new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute, dataRow);
SerializationHelper.Serialize(testCase);
results.Add(testCase);
}
}
// REVIEW: Could we re-write LambdaTestCase to just be for exceptions?
if (results.Count == 0)
{
results.Add(new LambdaTestCase(testCollection, assembly, testClass, testMethod, factAttribute, () => { throw new InvalidOperationException("No data found for " + testClass.Name + "." + testMethod.Name); }));
}
return(results);
}
}
catch
{
return(new XunitTestCase[] { new XunitTheoryTestCase(testCollection, assembly, testClass, testMethod, factAttribute) });
}
}
}
/// <inheritdoc/>
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererType);
IEnumerable <object[]> data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method);
if (data == null)
{
Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called."));
continue;
}
foreach (var dataRow in data)
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow);
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes);
var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip");
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
dataDiscoveryException = ex;
}
}
/// <inheritdoc/>
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
dataDiscoveryException = ex;
}
}
/// <summary>
/// Discover test cases from a test method.
/// </summary>
/// <remarks>
/// This method performs the following steps:
/// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>;
/// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>;
/// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it;
/// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason.
/// </remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
// Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
var skipReason = theoryAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) }
}
;
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var results = new List <IXunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute);
skipReason = dataAttribute.GetNamedArgument <string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }
}
;
// GetData may return null, but that's okay; we'll let the NullRef happen and then catch it
// down below so that we get the composite test case.
foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method))
{
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
if (!SerializationHelper.IsSerializable(dataRow))
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case."));
return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) });
}
var testCase =
skipReason != null
? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason)
: CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
results.Add(testCase);
}
}
if (results.Count == 0)
{
results.Add(new ExecutionErrorTestCase(diagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"));
}
return(results);
}
catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}"));
}
}
return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) });
}
}
#pragma warning restore CS0618
/// <summary>
/// Discover test cases from a test method.
/// </summary>
/// <remarks>
/// This method performs the following steps:
/// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>;
/// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>;
/// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it;
/// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason.
/// </remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
// Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
var skipReason = theoryAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return(CreateTestCasesForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason));
}
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var results = new List <IXunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
IDataDiscoverer discoverer;
try
{
discoverer = ExtensibilityPointFactory.GetDataDiscoverer(DiagnosticMessageSink, discovererAttribute);
}
catch (InvalidCastException)
{
var reflectionAttribute = dataAttribute as IReflectionAttributeInfo;
if (reflectionAttribute != null)
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer."
)
);
}
else
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer."
)
);
}
continue;
}
if (discoverer == null)
{
var reflectionAttribute = dataAttribute as IReflectionAttributeInfo;
if (reflectionAttribute != null)
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist."
)
);
}
else
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist."
)
);
}
continue;
}
skipReason = dataAttribute.GetNamedArgument <string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
var data = discoverer.GetData(dataAttribute, testMethod.Method);
if (data == null)
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called."
)
);
continue;
}
foreach (var dataRow in data)
{
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
if (!SerializationHelper.IsSerializable(dataRow))
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case."));
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
var testCases =
skipReason != null
? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason)
: CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
results.AddRange(testCases);
}
}
if (results.Count == 0)
{
results.Add(new ExecutionErrorTestCase(DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"));
}
return(results);
}
catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}"));
}
}
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
/// <summary>
/// Discover test cases from a test method.
/// </summary>
/// <remarks>
/// This method performs the following steps:
/// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCasesForSkippedTheory"/>;
/// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCasesForTheory"/>;
/// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it;
/// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCasesForDataRow"/> or <see cref="CreateTestCasesForSkippedDataRow"/> if the data attribute has a skip reason.
/// </remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual async ValueTask <IReadOnlyCollection <IXunitTestCase> > Discover(
_ITestFrameworkDiscoveryOptions discoveryOptions,
_ITestMethod testMethod,
_IAttributeInfo theoryAttribute)
{
Guard.ArgumentNotNull(discoveryOptions);
Guard.ArgumentNotNull(testMethod);
Guard.ArgumentNotNull(theoryAttribute);
// Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
var skipReason = theoryAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return(await CreateTestCasesForSkippedTheory(discoveryOptions, testMethod, theoryAttribute, skipReason));
}
var preEnumerate =
discoveryOptions.PreEnumerateTheoriesOrDefault() &&
!theoryAttribute.GetNamedArgument <bool>(nameof(TheoryAttribute.DisableDiscoveryEnumeration));
if (preEnumerate)
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var results = new List <IXunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
IDataDiscoverer?discoverer;
try
{
discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute);
}
catch (InvalidCastException)
{
if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute)
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer."
)
);
}
else
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer."
)
);
}
continue;
}
if (discoverer == null)
{
if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute)
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist."
)
);
}
else
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist."
)
);
}
continue;
}
skipReason = dataAttribute.GetNamedArgument <string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
var data = await discoverer.GetData(dataAttribute, testMethod.Method);
if (data == null)
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called."
)
);
continue;
}
foreach (var dataRow in data)
{
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
// Also, if we can, we should attempt to resolve it to its parameter type right now, because
// the incoming data might be serializable but the actual parameter value that it gets converted
// to might not be, and serialization uses the resolved argument and not the input argument.
var resolvedData = dataRow.GetData();
var dataRowSkipReason = skipReason ?? dataRow.Skip;
if (testMethod.Method is _IReflectionMethodInfo reflectionMethodInfo)
{
resolvedData = reflectionMethodInfo.MethodInfo.ResolveMethodArguments(resolvedData);
}
if (!resolvedData.All(d => SerializationHelper.IsSerializable(d)))
{
var typeNames =
resolvedData
.Select(x => x?.GetType().FullName)
.WhereNotNull()
.Select(x => $"'{x}'")
.ToList();
TestContext.Current?.SendDiagnosticMessage(
"Non-serializable data (one or more of: {0}) found for '{1}.{2}'; falling back to single test case.",
string.Join(", ", typeNames),
testMethod.TestClass.Class.Name,
testMethod.Method.Name
);
return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
try
{
var testCases =
dataRowSkipReason != null
? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData, dataRowSkipReason)
: CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData);
results.AddRange(await testCases);
}
catch (Exception ex)
{
TestContext.Current?.SendDiagnosticMessage(
"Error creating theory test case for for '{0}.{1}'; falling back to single test case. Exception message: '{2}'",
testMethod.TestClass.Class.Name,
testMethod.Method.Name,
ex.Message
);
return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
}
}
if (results.Count == 0)
{
results.Add(
new ExecutionErrorTestCase(
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"
)
);
}
return(results);
}
catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase
{
TestContext.Current?.SendDiagnosticMessage(
"Exception thrown during theory discovery on '{0}.{1}'; falling back to single test case.{2}{3}",
testMethod.TestClass.Class.Name,
testMethod.Method.Name,
Environment.NewLine,
ex
);
}
}
return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
var testRunners = new List <XunitTestRunner>();
var toDispose = new List <IDisposable>();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes);
testRunners.Add(new XunitTestRunner(TestCase, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, theoryDisplayName, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
if (!MessageBus.QueueMessage(new TestStarting(TestCase, DisplayName)))
{
CancellationTokenSource.Cancel();
}
else
{
if (!MessageBus.QueueMessage(new TestFailed(TestCase, DisplayName, 0, null, ex.Unwrap())))
{
CancellationTokenSource.Cancel();
}
}
if (!MessageBus.QueueMessage(new TestFinished(TestCase, DisplayName, 0, null)))
{
CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
var aggregator = new ExceptionAggregator();
// REVIEW: What should be done with these leftover errors?
foreach (var disposable in toDispose)
{
timer.Aggregate(() => aggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <inheritdoc/>
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
// Special case Skip, because we want a single Skip (not one per data item), and a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
if (factAttribute.GetNamedArgument <string>("Skip") != null)
{
return new[] { new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }
}
;
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var results = new List <XunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute);
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }
}
;
// GetData may return null, but that's okay; we'll let the NullRef happen and then catch it
// down below so that we get the composite test case.
foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method))
{
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
var testCase = new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod, dataRow);
if (!SerializationHelper.IsSerializable(dataRow))
{
return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }
}
;
results.Add(testCase);
}
}
if (results.Count == 0)
{
results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod,
string.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)));
}
return(results);
}
catch { } // If something goes wrong, fall through to return just the XunitTestCase
}
return(new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) });
}
}
/// <inheritdoc />
protected override async Task RunTestsOnMethodAsync(IMessageBus messageBus,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
List <BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var executionTime = 0M;
try
{
var testMethod = Reflector.Wrap(methodUnderTest);
var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType);
foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
{
var methodToRun = methodUnderTest;
ITypeInfo[] resolvedTypes = null;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = ResolveGenericTypes(testMethod, dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
executionTime +=
await RunTestWithArgumentsAsync(messageBus,
classUnderTest,
constructorArguments,
methodToRun,
dataRow,
GetDisplayNameWithArguments(DisplayName, dataRow, resolvedTypes),
beforeAfterAttributes,
aggregator,
cancellationTokenSource);
if (cancellationTokenSource.IsCancellationRequested)
{
return;
}
}
}
}
catch (Exception ex)
{
if (!messageBus.QueueMessage(new TestStarting(this, DisplayName)))
{
cancellationTokenSource.Cancel();
}
else
{
if (!messageBus.QueueMessage(new TestFailed(this, DisplayName, executionTime, null, ex.Unwrap())))
{
cancellationTokenSource.Cancel();
}
}
if (!messageBus.QueueMessage(new TestFinished(this, DisplayName, executionTime, null)))
{
cancellationTokenSource.Cancel();
}
}
}
