ExecutionErrorTestCase ErrorTestCase( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, string message) => new ExecutionErrorTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, message );
/// <summary> /// Creates test cases for the entire theory. This is used when one or more of the theory data items /// are not serializable, or if the user has requested to skip theory pre-enumeration. By default, /// returns a single instance of <see cref="XunitDelayEnumeratedTheoryTestCase"/>, which performs the data discovery /// at runtime. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>The test case</returns> protected virtual ValueTask <IReadOnlyCollection <IXunitTestCase> > CreateTestCasesForTheory( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { var testCase = new XunitDelayEnumeratedTheoryTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod ); return(new(new[] { testCase })); }
/// <summary> /// Creates test cases for the entire theory. This is used when one or more of the theory data items /// are not serializable, or if the user has requested to skip theory pre-enumeration. By default, /// returns a single instance of <see cref="XunitDelayEnumeratedTheoryTestCase"/>, which performs the data discovery /// at runtime. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>The test case</returns> protected virtual IEnumerable <IXunitTestCase> CreateTestCasesForTheory( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { var testCase = new XunitDelayEnumeratedTheoryTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod ); return(new[] { testCase }); }
/// <summary> /// Creates test cases for a skipped theory. By default, returns a single instance of <see cref="XunitTestCase"/> /// (which inherently discovers the skip reason via the fact attribute). /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="skipReason">The skip reason that decorates <paramref name="theoryAttribute"/>.</param> /// <returns>The test cases</returns> protected virtual ValueTask <IReadOnlyCollection <IXunitTestCase> > CreateTestCasesForSkippedTheory( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, string skipReason) { // TODO: Skip reason should be passed down into the test case var testCase = new XunitTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod ); return(new(new[] { testCase })); }
/// <summary> /// Creates a single <see cref="XunitTestCase"/> for the given test method. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method.</param> /// <param name="factAttribute">The attribute that decorates the test method.</param> protected virtual IXunitTestCase CreateTestCase( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo factAttribute) { Guard.ArgumentNotNull(discoveryOptions); Guard.ArgumentNotNull(testMethod); Guard.ArgumentNotNull(factAttribute); return(new XunitTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod )); }
/// <summary> /// Creates test cases for a skipped theory. By default, returns a single instance of <see cref="XunitTestCase"/> /// (which inherently discovers the skip reason via the fact attribute). /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="skipReason">The skip reason that decorates <paramref name="theoryAttribute"/>.</param> /// <returns>The test cases</returns> protected virtual IEnumerable <IXunitTestCase> CreateTestCasesForSkip( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, string skipReason) { // TODO: Skip reason should be passed down into the test case var testCase = new XunitTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod ); return(new[] { testCase }); }
protected override ValueTask <IReadOnlyCollection <IXunitTestCase> > CreateTestCasesForTheory( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { var cultures = GetCultures(theoryAttribute); var result = cultures.Select( culture => new CulturedXunitTheoryTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, culture ) ).CastOrToReadOnlyCollection(); return(new(result)); }
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { var cultures = GetCultures(theoryAttribute); return(cultures.Select( culture => new CulturedXunitTheoryTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, culture ) ).ToList()); }
/// <summary> /// Creates test cases for a single row of skipped data. By default, returns a single instance of <see cref="XunitSkippedDataRowTestCase"/> /// with the data row inside of it. /// </summary> /// <remarks>If this method is overridden, the implementation will have to override <see cref="TestMethodTestCase.SkipReason"/> otherwise /// the default behavior will look at the <see cref="TheoryAttribute"/> and the test case will not be skipped.</remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="dataRow">The row of data for this test case.</param> /// <param name="skipReason">The reason this test case is to be skipped</param> /// <returns>The test cases</returns> protected virtual IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, object?[] dataRow, string skipReason) { var testCase = new XunitSkippedDataRowTestCase( DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, dataRow , skipReason); return(new[] { testCase }); }
/// <summary> /// Creates test cases for a single row of data. By default, returns a single instance of <see cref="XunitTestCase"/> /// with the data row inside of it. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="displayName">The optional display name for the test</param> /// <param name="traits">The traits associated with the test case.</param> /// <param name="dataRow">The row of data for this test case.</param> /// <returns>The test cases</returns> protected virtual ValueTask <IReadOnlyCollection <IXunitTestCase> > CreateTestCasesForDataRow( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, string?displayName, Dictionary <string, List <string> >?traits, object?[] dataRow) { var testCase = new XunitPreEnumeratedTheoryTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, dataRow, traits: traits, displayName: displayName ); return(new(new[] { testCase })); }
protected override ValueTask <IReadOnlyCollection <IXunitTestCase> > CreateTestCasesForDataRow( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute, string?displayName, Dictionary <string, List <string> >?traits, object?[] dataRow) { var cultures = GetCultures(theoryAttribute); var result = cultures.Select( culture => new CulturedXunitTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, culture, dataRow, traits, displayName) ).CastOrToReadOnlyCollection(); return(new(result)); }
public ValueTask <IReadOnlyCollection <IXunitTestCase> > Discover( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo factAttribute) { var ctorArgs = factAttribute.GetConstructorArguments().ToArray(); var cultures = Reflector.ConvertArguments(ctorArgs, new[] { typeof(string[]) }).Cast <string[]>().Single(); if (cultures == null || cultures.Length == 0) { cultures = new[] { "en-US", "fr-FR" } } ; var methodDisplay = discoveryOptions.MethodDisplayOrDefault(); var methodDisplayOptions = discoveryOptions.MethodDisplayOptionsOrDefault(); var result = cultures .Select(culture => CreateTestCase(testMethod, culture, methodDisplay, methodDisplayOptions)) .CastOrToReadOnlyCollection(); return(new(result)); } CulturedXunitTestCase CreateTestCase( _ITestMethod testMethod, string culture, TestMethodDisplay methodDisplay, TestMethodDisplayOptions methodDisplayOptions) { return(new CulturedXunitTestCase( methodDisplay, methodDisplayOptions, testMethod, culture )); } }
public IEnumerable <IXunitTestCase> Discover( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo factAttribute) { var ctorArgs = factAttribute.GetConstructorArguments().ToArray(); var cultures = Reflector.ConvertArguments(ctorArgs, new[] { typeof(string[]) }).Cast <string[]>().Single(); if (cultures == null || cultures.Length == 0) { cultures = new[] { "en-US", "fr-FR" } } ; var methodDisplay = discoveryOptions.MethodDisplayOrDefault(); var methodDisplayOptions = discoveryOptions.MethodDisplayOptionsOrDefault(); return (cultures .Select(culture => CreateTestCase(testMethod, culture, methodDisplay, methodDisplayOptions)) .ToList()); } CulturedXunitTestCase CreateTestCase( _ITestMethod testMethod, string culture, TestMethodDisplay methodDisplay, TestMethodDisplayOptions methodDisplayOptions) { return(new CulturedXunitTestCase( diagnosticMessageSink, methodDisplay, methodDisplayOptions, testMethod, culture )); } }
/// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCasesForSkippedTheory"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCasesForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCasesForDataRow"/> or <see cref="CreateTestCasesForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual async ValueTask <IReadOnlyCollection <IXunitTestCase> > Discover( _ITestFrameworkDiscoveryOptions discoveryOptions, _ITestMethod testMethod, _IAttributeInfo theoryAttribute) { Guard.ArgumentNotNull(discoveryOptions); Guard.ArgumentNotNull(testMethod); Guard.ArgumentNotNull(theoryAttribute); // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return(await CreateTestCasesForSkippedTheory(discoveryOptions, testMethod, theoryAttribute, skipReason)); } var preEnumerate = discoveryOptions.PreEnumerateTheoriesOrDefault() && !theoryAttribute.GetNamedArgument <bool>(nameof(TheoryAttribute.DisableDiscoveryEnumeration)); if (preEnumerate) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List <IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); IDataDiscoverer?discoverer; try { discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); } catch (InvalidCastException) { if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } else { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer." ) ); } continue; } if (discoverer == null) { if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } else { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist." ) ); } continue; } skipReason = dataAttribute.GetNamedArgument <string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var data = await discoverer.GetData(dataAttribute, testMethod.Method); if (data == null) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called." ) ); continue; } foreach (var dataRow in data) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. // Also, if we can, we should attempt to resolve it to its parameter type right now, because // the incoming data might be serializable but the actual parameter value that it gets converted // to might not be, and serialization uses the resolved argument and not the input argument. var resolvedData = dataRow.GetData(); var dataRowSkipReason = skipReason ?? dataRow.Skip; if (testMethod.Method is _IReflectionMethodInfo reflectionMethodInfo) { resolvedData = reflectionMethodInfo.MethodInfo.ResolveMethodArguments(resolvedData); } if (!resolvedData.All(d => SerializationHelper.IsSerializable(d))) { var typeNames = resolvedData .Select(x => x?.GetType().FullName) .WhereNotNull() .Select(x => $"'{x}'") .ToList(); TestContext.Current?.SendDiagnosticMessage( "Non-serializable data (one or more of: {0}) found for '{1}.{2}'; falling back to single test case.", string.Join(", ", typeNames), testMethod.TestClass.Class.Name, testMethod.Method.Name ); return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } try { var testCases = dataRowSkipReason != null ? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData, dataRowSkipReason) : CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow.TestDisplayName, dataRow.Traits, resolvedData); results.AddRange(await testCases); } catch (Exception ex) { TestContext.Current?.SendDiagnosticMessage( "Error creating theory test case for for '{0}.{1}'; falling back to single test case. Exception message: '{2}'", testMethod.TestClass.Class.Name, testMethod.Method.Name, ex.Message ); return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } } } if (results.Count == 0) { results.Add( new ExecutionErrorTestCase( discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}" ) ); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { TestContext.Current?.SendDiagnosticMessage( "Exception thrown during theory discovery on '{0}.{1}'; falling back to single test case.{2}{3}", testMethod.TestClass.Class.Name, testMethod.Method.Name, Environment.NewLine, ex ); } } return(await CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); }