void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument<string>("DisplayName") ?? type.Name + "." + method.Name; ITypeInfo[] resolvedTypes = null; if (arguments != null && method.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(method, arguments); method = method.MakeGenericMethod(resolvedTypes); } Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments, resolvedTypes); SkipReason = factAttribute.GetNamedArgument<string>("Skip"); Traits = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase); TestCollection = testCollection; foreach (var traitAttribute in Method.GetCustomAttributes(typeof(ITraitAttribute)) .Concat(Class.GetCustomAttributes(typeof(ITraitAttribute)))) { var discovererAttribute = traitAttribute.GetCustomAttributes(typeof(TraitDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetTraitDiscoverer(discovererAttribute); if (discoverer != null) foreach (var keyValuePair in discoverer.GetTraits(traitAttribute)) Traits.Add(keyValuePair.Key, keyValuePair.Value); } uniqueID = new Lazy<string>(GetUniqueID, true); }
protected override void OnAssemblyStarting() { collectionBehaviorAttribute = AssemblyInfo.GetCustomAttributes(typeof(CollectionBehaviorAttribute)).SingleOrDefault(); if (collectionBehaviorAttribute != null) { disableParallelization = collectionBehaviorAttribute.GetNamedArgument <bool>("DisableTestParallelization"); maxParallelThreads = collectionBehaviorAttribute.GetNamedArgument <int>("MaxParallelThreads"); } disableParallelization = ExecutionOptions.GetValue <bool>(TestOptionsNames.Execution.DisableParallelization, disableParallelization); var maxParallelThreadsOption = ExecutionOptions.GetValue <int>(TestOptionsNames.Execution.MaxParallelThreads, 0); if (maxParallelThreadsOption > 0) { maxParallelThreads = maxParallelThreadsOption; } scheduler = GetTaskScheduler(maxParallelThreads); var ordererAttribute = AssemblyInfo.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (ordererAttribute != null) { TestCaseOrderer = GetXunitTestCaseOrderer(ordererAttribute); } }
void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument <string>("DisplayName") ?? type.Name + "." + method.Name; ITypeInfo[] resolvedTypes = null; if (arguments != null && method.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(method, arguments); method = method.MakeGenericMethod(resolvedTypes); } Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments, resolvedTypes); SkipReason = factAttribute.GetNamedArgument <string>("Skip"); Traits = new Dictionary <string, List <string> >(StringComparer.OrdinalIgnoreCase); TestCollection = testCollection; foreach (IAttributeInfo traitAttribute in Method.GetCustomAttributes(typeof(TraitAttribute)) .Concat(Class.GetCustomAttributes(typeof(TraitAttribute)))) { var ctorArgs = traitAttribute.GetConstructorArguments().ToList(); Traits.Add((string)ctorArgs[0], (string)ctorArgs[1]); } uniqueID = new Lazy <string>(GetUniqueID, true); }
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .ToDictionary( a => a.GetNamedArgument <string>(nameof(BenchmarkVariationAttribute.VariationName)), a => a.GetNamedArgument <object[]>(nameof(BenchmarkVariationAttribute.Data))); if (!variations.Any()) { variations.Add("Default", new object[0]); } var tests = new List <IXunitTestCase>(); foreach (var variation in variations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Key, _diagnosticMessageSink, testMethod, variation.Value)); } return(tests); }
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .ToDictionary( a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)), a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data))); if (!variations.Any()) { variations.Add("Default", new object[0]); } var tests = new List<IXunitTestCase>(); foreach (var variation in variations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Key, _diagnosticMessageSink, testMethod, variation.Value)); } return tests; }
/// <inheritdoc /> public IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { int maxRetries = factAttribute .GetNamedArgument <int>(nameof(OrderedFactAttribute.MaxRetries)); int delaySeconds = factAttribute .GetNamedArgument <int>(nameof(OrderedFactAttribute.DelaySeconds)); string exceptionTypeFullName = factAttribute .GetNamedArgument <string>(nameof(OrderedFactAttribute.ExceptionTypeFullName)); var retryTestCase = new RetryTestCase( _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries, delaySeconds, exceptionTypeFullName ); yield return(retryTestCase); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { IXunitTestCase testCase; if (testMethod.Method.GetParameters().Any()) { testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[RetryFact] methods are not allowed to have parameters. Did you mean to use [RetryTheory]?"); } else if (testMethod.Method.IsGenericMethodDefinition) { testCase = new ExecutionErrorTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "[RetryFact] methods are not allowed to be generic."); } else { var maxRetries = factAttribute.GetNamedArgument <int>(nameof(RetryFactAttribute.MaxRetries)); var delayBetweenRetriesMs = factAttribute.GetNamedArgument <int>(nameof(RetryFactAttribute.DelayBetweenRetriesMs)); testCase = new RetryTestCase(_messageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs); } return(new[] { testCase }); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (factAttribute == null) { throw new ArgumentNullException(nameof(factAttribute)); } var dataNumber = factAttribute.GetNamedArgument <int>(nameof(GenerateDoubleInlineDataAttribute.DataNumber)); if (dataNumber < 1) { dataNumber = 1; } var maxValue = factAttribute.GetNamedArgument <double>(nameof(GenerateDoubleInlineDataAttribute.MaxValue)); if (maxValue < 1) { maxValue = 1; } var minvalue = factAttribute.GetNamedArgument <double>(nameof(GenerateDoubleInlineDataAttribute.MinValue)); if (minvalue < 1) { minvalue = 1; } yield return(new GeneratorDoubleCase(mDiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, dataNumber, maxValue, minvalue)); }
void Initialize() { if (initialized) { return; } collectionBehaviorAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(CollectionBehaviorAttribute)).SingleOrDefault(); if (collectionBehaviorAttribute != null) { disableParallelization = collectionBehaviorAttribute.GetNamedArgument <bool>("DisableTestParallelization"); maxParallelThreads = collectionBehaviorAttribute.GetNamedArgument <int>("MaxParallelThreads"); } disableParallelization = ExecutionOptions.GetValue <bool>(TestOptionsNames.Execution.DisableParallelization, disableParallelization); var maxParallelThreadsOption = ExecutionOptions.GetValue <int>(TestOptionsNames.Execution.MaxParallelThreads, 0); if (maxParallelThreadsOption > 0) { maxParallelThreads = maxParallelThreadsOption; } scheduler = GetTaskScheduler(maxParallelThreads); var ordererAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (ordererAttribute != null) { TestCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(ordererAttribute); } initialized = true; }
void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument<string>("DisplayName") ?? type.Name + "." + method.Name; ITypeInfo[] resolvedTypes = null; if (arguments != null && method.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(method, arguments); method = method.MakeGenericMethod(resolvedTypes); } Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments, resolvedTypes); SkipReason = factAttribute.GetNamedArgument<string>("Skip"); Traits = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase); TestCollection = testCollection; foreach (IAttributeInfo traitAttribute in Method.GetCustomAttributes(typeof(TraitAttribute)) .Concat(Class.GetCustomAttributes(typeof(TraitAttribute)))) { var ctorArgs = traitAttribute.GetConstructorArguments().ToList(); Traits.Add((string)ctorArgs[0], (string)ctorArgs[1]); } uniqueID = new Lazy<string>(GetUniqueID, true); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var maxRetries = factAttribute.GetNamedArgument <int>("RetryOnFailureCount"); var exceptionToIgnoreTest = factAttribute.GetNamedArgument <Type>("ExceptionTypeToIgnore"); yield return(new FlakySystemTestCases(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries, exceptionToIgnoreTest)); }
/// <inheritdoc/> public override IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute ) { var baseCases = base.Discover(discoveryOptions, testMethod, factAttribute); if (!String.IsNullOrEmpty(factAttribute.GetNamedArgument <string>("Skip"))) { // No need to change skipped tests. return(baseCases); } var platforms = factAttribute.GetNamedArgument <Platform>("Platforms"); if ((platforms & Platform) != 0) { // No need to change tests that should run on the current platform. return(baseCases); } // Base implementation always returns a single test case. var baseCase = baseCases.FirstOrDefault(); Contract.Assert(baseCase != null); if (baseCase is ExecutionErrorTestCase) { // No need to change an erroneous test. return(baseCases); } if (!String.IsNullOrEmpty(baseCase.SkipReason)) { // No need to change a skipped test. Covered to protect against changes in the base class. return(baseCases); } // Replace test with its skipped equivalent. var platformJustification = factAttribute.GetNamedArgument <string>( "PlatformJustification" ); var skipReason = String.Format( platformJustification, platforms.ToString().Replace(", ", " | "), Platform ); var testCase = new SkippedXunitTestCase( _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), skipReason, baseCase.TestMethod, baseCase.TestMethodArguments ); return(new[] { testCase }); }
/// <inheritdoc/> public override IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { var baseCases = base.Discover(discoveryOptions, testMethod, theoryAttribute); if (!String.IsNullOrEmpty(theoryAttribute.GetNamedArgument <string>("Skip"))) { // No need to change skipped tests. return(baseCases); } var platforms = theoryAttribute.GetNamedArgument <Platform>("Platforms"); if ((platforms & Platform) != 0) { // No need to change tests that should run on the current platform. return(baseCases); } // Update the individual test cases as needed: Skip test cases that would otherwise run. var testCases = new List <IXunitTestCase>(); var platformJustification = theoryAttribute.GetNamedArgument <string>("PlatformJustification"); var skipReason = String.Format(platformJustification, platforms.ToString().Replace(", ", " | "), Platform); foreach (var baseCase in baseCases) { if (baseCase is ExecutionErrorTestCase) { // No need to change an erroneous test. Covered to protect against changes in the base class. testCases.Add(baseCase); continue; } if (!String.IsNullOrEmpty(baseCase.SkipReason)) { // No need to change a skipped test. Likely to hit this case only after xUnit.net has been updated // to 2.2.0+, where [Data] also has a Skip property. testCases.Add(baseCase); continue; } var testCase = new SkippedXunitTestCase( _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), skipReason, baseCase.TestMethod, baseCase.TestMethodArguments); testCases.Add(testCase); } return(testCases); }
/// <inheritdoc /> public IEnumerable <KeyValuePair <string, string> > GetTraits(IAttributeInfo traitAttribute) { bool requiresAdmin = traitAttribute.GetNamedArgument <bool>(nameof(FactIfSupportedAttribute.RequiresAdmin)); bool requiresJournalScan = traitAttribute.GetNamedArgument <bool>(nameof(FactIfSupportedAttribute.RequiresJournalScan)); bool requiresSymlinkPermission = traitAttribute.GetNamedArgument <bool>(nameof(FactIfSupportedAttribute.RequiresSymlinkPermission)); if (requiresAdmin || requiresJournalScan || requiresSymlinkPermission) { yield return(new KeyValuePair <string, string>("Category", RequiresAdmin)); } }
private static STATestCase WrapTestCase(IXunitTestCase testCase, IAttributeInfo factAttribute) { var usePackUri = factAttribute.GetNamedArgument <bool>(STATestCase.UsePackUriPropertyName); var usePackUriApplication = factAttribute.GetNamedArgument <bool>(STATestCase.UsePackUriApplicationPropertyName); var staTestCase = new STATestCase(testCase) { UsePackUri = usePackUri, UsePackUriApplication = usePackUriApplication, }; return(staTestCase); }
public virtual IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .Select(a => new { Name = a.GetNamedArgument <string>(nameof(BenchmarkVariationAttribute.VariationName)), TestMethodArguments = a.GetNamedArgument <object[]>(nameof(BenchmarkVariationAttribute.Data)), Framework = a.GetNamedArgument <string>(nameof(BenchmarkVariationAttribute.Framework)) }) .ToList(); if (!variations.Any()) { variations.Add(new { Name = "Default", TestMethodArguments = new object[0], Framework = (string)null }); } var tests = new List <IXunitTestCase>(); foreach (var variation in variations) { if (BenchmarkConfig.Instance.RunIterations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Framework, variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } else { tests.Add(new NonCollectingBenchmarkTestCase( variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } } return(tests); }
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { var maxRetries = theoryAttribute.GetNamedArgument <int>(nameof(RetryTheoryAttribute.MaxRetries)); var delayBetweenRetriesMs = theoryAttribute.GetNamedArgument <int>(nameof(RetryTheoryAttribute.DelayBetweenRetriesMs)); return(new[] { new RetryTheoryDiscoveryAtRuntimeCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries, delayBetweenRetriesMs) }); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var maxRetries = factAttribute.GetNamedArgument <int>("MaxRetries"); if (maxRetries < 1) { maxRetries = 3; } var exponentialBackoffMs = Math.Max(0, factAttribute.GetNamedArgument <int>("ExponentialBackoffMs")); yield return(new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries, exponentialBackoffMs)); }
public virtual IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .Select(a => new { Name = a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)), TestMethodArguments = a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data)), Framework = a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.Framework)) }) .ToList(); if (!variations.Any()) { variations.Add(new { Name = "Default", TestMethodArguments = new object[0], Framework = (string)null }); } var tests = new List<IXunitTestCase>(); foreach (var variation in variations) { if (BenchmarkConfig.Instance.RunIterations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Framework, variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } else { tests.Add(new NonCollectingBenchmarkTestCase( variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } } return tests; }
public TestParameters(ITestMethod testMethod, IAttributeInfo factAttribute) { SkipReason = factAttribute.GetNamedArgument <string>(nameof(TestAttribute.Skip)); ThreadType = factAttribute.GetNamedArgument <ThreadType>(nameof(TestAttribute.ThreadType)); ShowWindow = factAttribute.GetNamedArgument <bool>(nameof(TestAttribute.ShowWindow)); if (ThreadType == ThreadType.Default) { var classThreadTypeAttribute = GetClassAttribute <ThreadTypeAttribute>(testMethod); if (classThreadTypeAttribute != null) { ThreadType = classThreadTypeAttribute.GetNamedArgument <ThreadType>(nameof(TestAttribute.ThreadType)); } } }
/// <inheritdoc cref="IXunitTestCaseDiscoverer"/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo expectThatAttribute) { // Check if expectation should be skipped if (expectThatAttribute.GetNamedArgument <string>(nameof(ExpectThatAttribute.Skip)) != null) { return(CreateSkipTestCases(discoveryOptions, testMethod)); } try { var scenarioAttributes = testMethod.GetScenarioAttributes(DiagnosticMessageSink); if (!scenarioAttributes.Any()) { return(CreateNoScenarioTestCases(discoveryOptions, testMethod)); } if (IsPreEnumerationSupported(discoveryOptions)) { return(CreatePreEnumeratedTestCases(discoveryOptions, testMethod, scenarioAttributes)); } else { return(CreateSingleTestCaseForAllScenarios(discoveryOptions, testMethod)); } } catch (Exception e) { DiagnosticMessageSink.OnMessage(new PrintableDiagnosticMessage($"Exception thrown during scenario expectation discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'.{Environment.NewLine}{e}")); throw; } }
public IEnumerable <T> OrderTestCases <T>(IEnumerable <T> testCases) where T : ITestCase { var sortedTestCases = new SortedDictionary <int, List <T> >(new PriorityComparer()); foreach (var testCase in testCases) { IAttributeInfo attribute = testCase.TestMethod.Method.GetCustomAttributes((typeof(TestPriorityAttribute))).FirstOrDefault(); int priority = attribute != null?attribute.GetNamedArgument <int>("Priority") : 0; if (!sortedTestCases.TryGetValue(priority, out var list)) { sortedTestCases.Add(priority, (list = new List <T>())); } list.Add(testCase); } foreach (var list in sortedTestCases.Values) { list.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.TestMethod.Method.Name, y.TestMethod.Method.Name)); foreach (var testCase in list) { yield return(testCase); } } }
private static IMethodInfo ResolveWarmupMethod(ITestMethod testMethod, IAttributeInfo factAttribute) { var warmupMethodName = factAttribute.GetNamedArgument <string>(nameof(StressAttribute.WarmupMethodName)); var warmupMethod = testMethod.TestClass.Class.GetMethod(warmupMethodName, includePrivateMethod: false); return(warmupMethod); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); return factAttribute.GetNamedArgument<string>("Skip") != null ? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) } : new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }; }
/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument <string>("Skip") != null) { return new[] { new XunitTestCase(defaultMethodDisplay, testMethod) } } ; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var results = new List <XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(defaultMethodDisplay, testMethod, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) { results.Add(new LambdaTestCase(defaultMethodDisplay, testMethod, () => { throw new InvalidOperationException(String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)); })); } return(results); } catch { } } return(new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) }); } }
public CustomFactDiscoverer(IMessageSink diagnosticMessageSink, IXunitTestCase inner, IAttributeInfo fact) : base(diagnosticMessageSink, TestMethodDisplay.ClassAndMethod, inner.TestMethod, inner.TestMethodArguments) { this.nameOverride = fact.GetNamedArgument <string>("DisplayName"); }
/// <summary> /// Gets the timeout for the test case. By default, pulls the skip reason from the /// <see cref="FactAttribute.Timeout"/> property. /// </summary> /// <param name="factAttribute">The fact attribute the decorated the test case.</param> /// <returns>The timeout in milliseconds, if set; 0, if unset.</returns> protected virtual int GetTimeout(IAttributeInfo factAttribute) { if (factAttribute is null) { throw new ArgumentNullException(nameof(factAttribute)); } return(factAttribute.GetNamedArgument <int>("Timeout")); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); return(factAttribute.GetNamedArgument <string>("Skip") != null ? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) } : new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }); }
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo testAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { DisableRetry = testAttribute == null ? true : testAttribute.GetNamedArgument<bool>("DisableRetry"); }
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo testAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { DisableRetry = testAttribute == null ? true : testAttribute.GetNamedArgument <bool>("DisableRetry"); }
/// <summary> /// Gets the skip reason for the test case. By default, pulls the skip reason from the /// <see cref="FactAttribute.Skip"/> property. /// </summary> /// <param name="factAttribute">The fact attribute the decorated the test case.</param> /// <returns>The skip reason, if skipped; <c>null</c>, otherwise.</returns> protected virtual string GetSkipReason(IAttributeInfo factAttribute) { if (factAttribute is null) { throw new ArgumentNullException(nameof(factAttribute)); } return(factAttribute.GetNamedArgument <string>("SkipReason")); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var appName = factAttribute.GetNamedArgument <string>("AppName"); var appAssembly = factAttribute.GetNamedArgument <string>("AppAssembly"); var appFolderPath = TestApplicationRunner.GetApplicationOutputFolderPath(appName); MessageSink.OnMessage(new DiagnosticMessage("Discovering tests case in {0} for application {1}", appFolderPath, appName)); var results = new List <IXunitTestCase>(); if (!System.IO.Directory.Exists(appFolderPath)) { results.Add( new ExecutionErrorTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.None, testMethod, $"Application folder path '{appFolderPath}' does not exist: try compiling application '{appName}' first.")); return(results); } foreach (string folder in System.IO.Directory.GetDirectories(appFolderPath)) { results.Add( new XunitTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, testMethod, new object[] { appName, System.IO.Path.GetFileName(folder), appAssembly })); } if (results.Count == 0) { results.Add( new ExecutionErrorTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.None, testMethod, $"Application '{appName}' does not have any test cases: try compiling the application '{appName}' first.")); } return(results); }
public override IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var minimumVersion = factAttribute.GetNamedArgument <string>(nameof(PgVersionTargetedFact.MinimumVersion)); var maximumVersion = factAttribute.GetNamedArgument <string>(nameof(PgVersionTargetedFact.MaximumVersion)); if (minimumVersion != null && Version.TryParse(minimumVersion, out var minVersion) && Version < minVersion) { yield return(new TestCaseSkippedDueToVersion($"Minimum required PG version {minimumVersion} is higher than {Version}", DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod)); } if (maximumVersion != null && Version.TryParse(maximumVersion, out var maxVersion) && Version > maxVersion) { yield return(new TestCaseSkippedDueToVersion($"Maximum allowed PG version {maximumVersion} is higher than {Version}", DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod)); } yield return(base.CreateTestCase(discoveryOptions, testMethod, factAttribute)); }
/// <summary> /// Gets the name of the category from the <see cref="CategoryTraitAttribute" />. /// </summary> /// <param name="traitAttribute"> /// The <see cref="ITraitAttribute" /> containing the category values. /// </param> /// <returns> /// The trait values extracted from the <see cref="ITraitAttribute" />. /// </returns> public IEnumerable <KeyValuePair <string, string> > GetTraits( IAttributeInfo traitAttribute) { var categoryName = traitAttribute.GetNamedArgument <string>("Category"); yield return(new KeyValuePair <string, string>("Category", categoryName)); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries"); if (maxRetries < 1) maxRetries = 3; yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries); }
/// <inheritdoc /> public IEnumerable <KeyValuePair <string, string> > GetTraits(IAttributeInfo trait) { yield return(new KeyValuePair <string, string>("Category", "Story")); var traitKey = trait.GetNamedArgument <string>("Key"); if (!string.IsNullOrEmpty(traitKey)) { yield return(new KeyValuePair <string, string>("Story", traitKey)); } var traitReference = trait.GetNamedArgument <string>("Reference"); if (!string.IsNullOrEmpty(traitReference)) { yield return(new KeyValuePair <string, string>("Story", traitReference)); } }
public IEnumerable <KeyValuePair <string, string> > GetTraits(IAttributeInfo traitAttribute) { var categoryValue = traitAttribute.GetNamedArgument <string>(nameof(CategoryTraitAttribute.Category)); if (!String.IsNullOrWhiteSpace(categoryValue)) { yield return(new KeyValuePair <string, string>("Category", categoryValue)); } }
public KuduXunitTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay testMethodDisplay, ITestMethod testMethod, object[] testMethodArguments, IAttributeInfo testAttribute) : base(diagnosticMessageSink, testMethodDisplay, testMethod, testMethodArguments) { DisableRetry = testAttribute == null ? true : testAttribute.GetNamedArgument<bool>("DisableRetry"); }
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .ToDictionary( a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)), a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data))); if (!variations.Any()) { variations.Add("Default", new object[0]); } var tests = new List<IXunitTestCase>(); foreach (var variation in variations) { if (BenchmarkConfig.Instance.RunIterations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Key, _diagnosticMessageSink, testMethod, variation.Value)); } else { // TODO running a single iteration is slow under DNX (see #2574) // disabling so that we don't add 10min to build.cmd #if !DNX451 && !DNXCORE50 var args = new[] { new MetricCollector() } .Concat(variation.Value) .ToArray(); tests.Add(new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, args)); #endif } } return tests; }
/// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/>. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument<string>("Skip"); if (skipReason != null) return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) }; if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List<IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. if (!SerializationHelper.IsSerializable(dataRow)) return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; var testCase = CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); results.Add(testCase); } } if (results.Count == 0) results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); return results; } catch { } // If something goes wrong, fall through to return just the XunitTestCase } return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; }
/// <inheritdoc/> public IEnumerable<IXunitTestCase> Discover(ITestMethod testMethod, IAttributeInfo factAttribute) { // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument<string>("Skip") != null) return new[] { new XunitTestCase(testMethod) }; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); try { using (var memoryStream = new MemoryStream()) { var results = new List<XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) return new XunitTestCase[] { new XunitTheoryTestCase(testMethod) }; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(testMethod, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) results.Add(new LambdaTestCase(testMethod, () => { throw new InvalidOperationException(String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)); })); return results; } } catch { return new XunitTestCase[] { new XunitTheoryTestCase(testMethod) }; } }
/// <inheritdoc/> public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument<string>("Skip") != null) return new[] { new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var results = new List<XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } if (results.Count == 0) results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod, String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name))); return results; } catch { } // If there are serialization issues with the theory data, fall through to return just the XunitTestCase } return new XunitTestCase[] { new XunitTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testMethod) }; }
/// <inheritdoc/> public IEnumerable<XunitTestCase> Discover(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo testClass, IMethodInfo testMethod, IAttributeInfo factAttribute) { // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument<string>("Skip") != null) return new[] { new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute) }; try { using (var memoryStream = new MemoryStream()) { List<XunitTestCase> results = new List<XunitTestCase>(); var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType); // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(testCollection, assembly, testClass, testMethod, factAttribute, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) results.Add(new LambdaTestCase(testCollection, assembly, testClass, testMethod, factAttribute, () => { throw new InvalidOperationException("No data found for " + testClass.Name + "." + testMethod.Name); })); return results; } } catch { return new XunitTestCase[] { new XunitTheoryTestCase(testCollection, assembly, testClass, testMethod, factAttribute) }; } }
public override IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo benchmarkAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); // // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). // if (benchmarkAttribute.GetNamedArgument<string>("Skip") != null) { yield return new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod); yield break; } // // Use the TheoryDiscoverer to enumerate the cases. We can't do this, because // xUnit doesn't expose everything we need (for example, the ability to ask if an // object is xUnit-serializable). // foreach (var theoryCase in base.Discover(discoveryOptions, testMethod, benchmarkAttribute)) { if (theoryCase is XunitTheoryTestCase) { // // TheoryDiscoverer returns one of these if it cannot enumerate the cases now. // We'll return a BenchmarkTestCase with no data associated. // yield return new BenchmarkTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod, benchmarkAttribute); } else { // // This is a test case with data // yield return new BenchmarkTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod, benchmarkAttribute, theoryCase.TestMethodArguments); } } }
public TestParameters(IAttributeInfo factAttribute) { SkipReason = factAttribute.GetNamedArgument<string>("Skip"); ThreadType = factAttribute.GetNamedArgument<ThreadType>("ThreadType"); }
/// <summary> /// Gets the skip reason for the test case. By default, pulls the skip reason from the /// <see cref="FactAttribute.Skip"/> property. /// </summary> /// <param name="factAttribute">The fact attribute the decorated the test case.</param> /// <returns>The skip reason, if skipped; <c>null</c>, otherwise.</returns> protected virtual string GetSkipReason(IAttributeInfo factAttribute) => factAttribute.GetNamedArgument<string>("Skip");
public TestForTypesParameters(IAttributeInfo factAttribute) : base(factAttribute) { Types = factAttribute.GetNamedArgument<Type[]>("Types"); }
void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument<string>("DisplayName") ?? type.Name + "." + method.Name; Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments); SkipReason = factAttribute.GetNamedArgument<string>("Skip"); Traits = new Dictionary<string, string>(); TestCollection = testCollection; foreach (IAttributeInfo traitAttribute in Method.GetCustomAttributes(typeof(TraitAttribute))) { var ctorArgs = traitAttribute.GetConstructorArguments().ToList(); Traits.Add((string)ctorArgs[0], (string)ctorArgs[1]); } uniqueID = new Lazy<string>(GetUniqueID, true); }
/// <inheritdoc/> public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return !dataAttribute.GetNamedArgument<bool>("DisableDiscoveryEnumeration"); }
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute) { var categories = traitAttribute.GetNamedArgument<string[]>(nameof(CategoryAttribute.Categories)); return categories.Select(category => new KeyValuePair<string, string>(category, null)); }
/// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument<string>("Skip"); if (skipReason != null) return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) }; if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List<IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); skipReason = dataAttribute.GetNamedArgument<string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; IEnumerable<object[]> data = discoverer.GetData(dataAttribute, testMethod.Method); if (data == null) { results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called.")); continue; } foreach (var dataRow in data) { // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. if (!SerializationHelper.IsSerializable(dataRow)) { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.")); return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; } var testCase = skipReason != null ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) : CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); results.Add(testCase); } } if (results.Count == 0) results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); return results; } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } } return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; }