/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument <string>("Skip") != null) { return new[] { new XunitTestCase(defaultMethodDisplay, testMethod) } } ; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var results = new List <XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(defaultMethodDisplay, testMethod, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) { results.Add(new LambdaTestCase(defaultMethodDisplay, testMethod, () => { throw new InvalidOperationException(String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)); })); } return(results); } catch { } } return(new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) }); } }
/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo testClass, IMethodInfo testMethod, IAttributeInfo factAttribute) { // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument <string>("Skip") != null) { return new[] { new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute) } } ; try { using (var memoryStream = new MemoryStream()) { var results = new List <XunitTestCase>(); var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) { results.Add(new LambdaTestCase(testCollection, assembly, testClass, testMethod, factAttribute, () => { throw new InvalidOperationException("No data found for " + testClass.Name + "." + testMethod.Name); })); } return(results); } } catch { return(new XunitTestCase[] { new XunitTheoryTestCase(testCollection, assembly, testClass, testMethod, factAttribute) }); } } }
/// <inheritdoc/> protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererType); IEnumerable <object[]> data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method); if (data == null) { Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called.")); continue; } foreach (var dataRow in data) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow); if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes); var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip"); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync dataDiscoveryException = ex; } }
/// <inheritdoc/> protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync dataDiscoveryException = ex; } }
/// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) } } ; if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List <IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); skipReason = dataAttribute.GetNamedArgument <string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. if (!SerializationHelper.IsSerializable(dataRow)) { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.")); return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }); } var testCase = skipReason != null ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) : CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); results.Add(testCase); } } if (results.Count == 0) { results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } } return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }); } }
#pragma warning restore CS0618 /// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return(CreateTestCasesForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason)); } if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List <IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); IDataDiscoverer discoverer; try { discoverer = ExtensibilityPointFactory.GetDataDiscoverer(DiagnosticMessageSink, discovererAttribute); } catch (InvalidCastException) { var reflectionAttribute = dataAttribute as IReflectionAttributeInfo; if (reflectionAttribute != null) { results.Add( new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } else { results.Add( new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } continue; } if (discoverer == null) { var reflectionAttribute = dataAttribute as IReflectionAttributeInfo; if (reflectionAttribute != null) { results.Add( new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } else { results.Add( new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } continue; } skipReason = dataAttribute.GetNamedArgument <string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var data = discoverer.GetData(dataAttribute, testMethod.Method); if (data == null) { results.Add( new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called." ) ); continue; } foreach (var dataRow in data) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. if (!SerializationHelper.IsSerializable(dataRow)) { DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.")); return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var testCases = skipReason != null ? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) : CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); results.AddRange(testCases); } } if (results.Count == 0) { results.Add(new ExecutionErrorTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } } return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); }
/// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCasesForSkippedTheory"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCasesForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCasesForDataRow"/> or <see cref="CreateTestCasesForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual async ValueTask <IReadOnlyCollection <IXunitTestCase> > Discover( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { Guard.ArgumentNotNull(discoveryOptions); Guard.ArgumentNotNull(testMethod); Guard.ArgumentNotNull(theoryAttribute); // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return(await CreateTestCasesForSkippedTheory(discoveryOptions, testMethod, theoryAttribute, skipReason)); } var preEnumerate = discoveryOptions.PreEnumerateTheoriesOrDefault() && !theoryAttribute.GetNamedArgument <bool>(nameof(TheoryAttribute.DisableDiscoveryEnumeration)); if (preEnumerate) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List <IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); IDataDiscoverer?discoverer; try { discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); } catch (InvalidCastException) { if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } else { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } continue; } if (discoverer == null) { if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } else { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } continue; } skipReason = dataAttribute.GetNamedArgument <string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var data = await discoverer.GetData(dataAttribute, testMethod.Method); if (data == null) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called." ) ); continue; } foreach (var dataRow in data) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. // Also, if we can, we should attempt to resolve it to its parameter type right now, because // the incoming data might be serializable but the actual parameter value that it gets converted // to might not be, and serialization uses the resolved argument and not the input argument. var resolvedData = dataRow.GetData(); var dataRowSkipReason = skipReason ?? dataRow.Skip; if (testMethod.Method is _IReflectionMethodInfo reflectionMethodInfo) { resolvedData = reflectionMethodInfo.MethodInfo.ResolveMethodArguments(resolvedData); } if (!resolvedData.All(d => SerializationHelper.IsSerializable(d))) { var typeNames = resolvedData .Select(x => x?.GetType().FullName) .WhereNotNull() .Select(x => $"'{x}'") .ToList(); TestContext.Current?.SendDiagnosticMessage( "Non-serializable data (one or more of: {0}) found for '{1}.{2}'; falling back to single test case.", string.Join(", ", typeNames), testMethod.TestClass.Class.Name, testMethod.Method.Name ); return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } try { var testCases = dataRowSkipReason != null ? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData, dataRowSkipReason) : CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData); results.AddRange(await testCases); } catch (Exception ex) { TestContext.Current?.SendDiagnosticMessage( "Error creating theory test case for for '{0}.{1}'; falling back to single test case. Exception message: '{2}'", testMethod.TestClass.Class.Name, testMethod.Method.Name, ex.Message ); return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } } } if (results.Count == 0) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}" ) ); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { TestContext.Current?.SendDiagnosticMessage( "Exception thrown during theory discovery on '{0}.{1}'; falling back to single test case.{2}{3}", testMethod.TestClass.Class.Name, testMethod.Method.Name, Environment.NewLine, ex ); } } return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); }
/// <inheritdoc/> protected override async Task <RunSummary> RunTestAsync() { var testRunners = new List <XunitTestRunner>(); var toDispose = new List <IDisposable>(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); testRunners.Add(new XunitTestRunner(TestCase, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, theoryDisplayName, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { if (!MessageBus.QueueMessage(new TestStarting(TestCase, DisplayName))) { CancellationTokenSource.Cancel(); } else { if (!MessageBus.QueueMessage(new TestFailed(TestCase, DisplayName, 0, null, ex.Unwrap()))) { CancellationTokenSource.Cancel(); } } if (!MessageBus.QueueMessage(new TestFinished(TestCase, DisplayName, 0, null))) { CancellationTokenSource.Cancel(); } return(new RunSummary { Total = 1, Failed = 1 }); } var runSummary = new RunSummary(); foreach (var testRunner in testRunners) { runSummary.Aggregate(await testRunner.RunAsync()); } var timer = new ExecutionTimer(); var aggregator = new ExceptionAggregator(); // REVIEW: What should be done with these leftover errors? foreach (var disposable in toDispose) { timer.Aggregate(() => aggregator.Run(() => disposable.Dispose())); } runSummary.Time += timer.Total; return(runSummary); }
/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument <string>("Skip") != null) { return new[] { new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) } } ; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var results = new List <XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. var testCase = new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod, dataRow); if (!SerializationHelper.IsSerializable(dataRow)) { return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) } } ; results.Add(testCase); } } if (results.Count == 0) { results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod, string.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name))); } return(results); } catch { } // If something goes wrong, fall through to return just the XunitTestCase } return(new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }); } }
/// <inheritdoc /> protected override async Task RunTestsOnMethodAsync(IMessageBus messageBus, Type classUnderTest, object[] constructorArguments, MethodInfo methodUnderTest, List <BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { var executionTime = 0M; try { var testMethod = Reflector.Wrap(methodUnderTest); var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod)) { var methodToRun = methodUnderTest; ITypeInfo[] resolvedTypes = null; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(testMethod, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } executionTime += await RunTestWithArgumentsAsync(messageBus, classUnderTest, constructorArguments, methodToRun, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow, resolvedTypes), beforeAfterAttributes, aggregator, cancellationTokenSource); if (cancellationTokenSource.IsCancellationRequested) { return; } } } } catch (Exception ex) { if (!messageBus.QueueMessage(new TestStarting(this, DisplayName))) { cancellationTokenSource.Cancel(); } else { if (!messageBus.QueueMessage(new TestFailed(this, DisplayName, executionTime, null, ex.Unwrap()))) { cancellationTokenSource.Cancel(); } } if (!messageBus.QueueMessage(new TestFinished(this, DisplayName, executionTime, null))) { cancellationTokenSource.Cancel(); } } }