public Type GetTestFrameworkType(IAttributeInfo attribute) { if (BenchmarkConfiguration.RunningAsPerfTest) return typeof(BenchmarkTestFramework); else return typeof(XunitTestFramework); }
public IEnumerable<IXunitTestCase> Discover (ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault (); if (testMethod.Method.GetParameters ().Any ()) { return new IXunitTestCase[] { new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, "[VsixFact] methods are not allowed to have parameters.") }; } else { var vsVersions = VsVersions.GetFinalVersions(testMethod.GetComputedProperty<string[]>(factAttribute, SpecialNames.VsixAttribute.VisualStudioVersions)); // Process VS-specific traits. var suffix = testMethod.GetComputedArgument<string>(factAttribute, SpecialNames.VsixAttribute.RootSuffix) ?? "Exp"; var newInstance = testMethod.GetComputedArgument<bool?>(factAttribute, SpecialNames.VsixAttribute.NewIdeInstance); var timeout = testMethod.GetComputedArgument<int?>(factAttribute, SpecialNames.VsixAttribute.TimeoutSeconds).GetValueOrDefault(XunitExtensions.DefaultTimeout); var testCases = new List<IXunitTestCase>(); // Add invalid VS versions. testCases.AddRange (vsVersions .Where (v => !VsVersions.InstalledVersions.Contains (v)) .Select (v => new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, string.Format ("Cannot execute test for specified {0}={1} because there is no VSSDK installed for that version.", SpecialNames.VsixAttribute.VisualStudioVersions, v)))); testCases.AddRange (vsVersions .Where (v => VsVersions.InstalledVersions.Contains (v)) .Select (v => new VsixTestCase (messageSink, defaultMethodDisplay, testMethod, v, suffix, newInstance, timeout))); return testCases; } }
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo testAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { }
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .ToDictionary( a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)), a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data))); if (!variations.Any()) { variations.Add("Default", new object[0]); } var tests = new List<IXunitTestCase>(); foreach (var variation in variations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Key, _diagnosticMessageSink, testMethod, variation.Value)); } return tests; }
public override IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod(); string conditionMemberName = factAttribute.GetConstructorArguments().FirstOrDefault() as string; MethodInfo conditionMethodInfo; if (conditionMemberName == null || (conditionMethodInfo = LookupConditionalMethod(testMethodInfo.DeclaringType, conditionMemberName)) == null) { return new[] { new ExecutionErrorTestCase( _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, GetFailedLookupString(conditionMemberName)) }; } IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute); if ((bool)conditionMethodInfo.Invoke(null, null)) { return testCases; } else { string skippedReason = "\"" + conditionMemberName + "\" returned false."; return testCases.Select(tc => new SkippedTestCase(tc, skippedReason)); } }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var skipReason = EvaluateSkipConditions(testMethod); var isTheory = false; IXunitTestCaseDiscoverer innerDiscoverer; if (testMethod.Method.GetCustomAttributes(typeof(TheoryAttribute)).Any()) { isTheory = true; innerDiscoverer = new TheoryDiscoverer(_diagnosticMessageSink); } else { innerDiscoverer = new FactDiscoverer(_diagnosticMessageSink); } var testCases = innerDiscoverer .Discover(discoveryOptions, testMethod, factAttribute) .Select(testCase => new SkipReasonTestCase(isTheory, skipReason, testCase)); return testCases; }
public CulturedXunitTestCase(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, string culture) : base(testCollection, assembly, type, method, factAttribute) { this.culture = culture; Traits.Add("Culture", culture); }
public UnresolvedCustomAttributeData(IAttributeInfo adapter) { if (adapter == null) throw new ArgumentNullException("adapter"); this.adapter = adapter; }
void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument<string>("DisplayName") ?? type.Name + "." + method.Name; ITypeInfo[] resolvedTypes = null; if (arguments != null && method.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(method, arguments); method = method.MakeGenericMethod(resolvedTypes); } Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments, resolvedTypes); SkipReason = factAttribute.GetNamedArgument<string>("Skip"); Traits = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase); TestCollection = testCollection; foreach (IAttributeInfo traitAttribute in Method.GetCustomAttributes(typeof(TraitAttribute)) .Concat(Class.GetCustomAttributes(typeof(TraitAttribute)))) { var ctorArgs = traitAttribute.GetConstructorArguments().ToList(); Traits.Add((string)ctorArgs[0], (string)ctorArgs[1]); } uniqueID = new Lazy<string>(GetUniqueID, true); }
public IEnumerable<IXunitTestCase> Discover ( ITestFrameworkDiscoveryOptions discoveryOptions , ITestMethod testMethod , IAttributeInfo factAttribute ) { var inv = InvariantTestCase.InvariantFromMethod(testMethod); return inv == null ? new[] { new InvariantTestCase ( MessageSink , TestMethodDisplay.Method , testMethod , null ) } : inv.AsSeq().Select ( i => new InvariantTestCase ( MessageSink , TestMethodDisplay.Method , testMethod , i ) ); }
/// <summary> /// Gets the trait values from the Category attribute. /// </summary> /// <param name="traitAttribute">The trait attribute containing the trait values.</param> /// <returns>The trait values.</returns> public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute) { TargetFrameworkMonikers platform = (TargetFrameworkMonikers)traitAttribute.GetConstructorArguments().First(); if (platform.HasFlag(TargetFrameworkMonikers.Net45)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet45Test); if (platform.HasFlag(TargetFrameworkMonikers.Net451)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet451Test); if (platform.HasFlag(TargetFrameworkMonikers.Net452)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet452Test); if (platform.HasFlag(TargetFrameworkMonikers.Net46)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet46Test); if (platform.HasFlag(TargetFrameworkMonikers.Net461)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet461Test); if (platform.HasFlag(TargetFrameworkMonikers.Net462)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet462Test); if (platform.HasFlag(TargetFrameworkMonikers.Net463)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNet463Test); if (platform.HasFlag(TargetFrameworkMonikers.Netcore50)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNetcore50Test); if (platform.HasFlag(TargetFrameworkMonikers.Netcore50aot)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNetcore50aotTest); if (platform.HasFlag(TargetFrameworkMonikers.Netcoreapp1_0)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNetcoreapp1_0Test); if (platform.HasFlag(TargetFrameworkMonikers.Netcoreapp1_1)) yield return new KeyValuePair<string, string>(XunitConstants.Category, XunitConstants.NonNetcoreapp1_1Test); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); return factAttribute.GetNamedArgument<string>("Skip") != null ? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) } : new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }; }
public override IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { string[] conditionMemberNames = factAttribute.GetConstructorArguments().FirstOrDefault() as string[]; IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute); return ConditionalTestDiscoverer.Discover(discoveryOptions, _diagnosticMessageSink, testMethod, testCases, conditionMemberNames); }
void Initialize(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo factAttribute, object[] arguments) { string displayNameBase = factAttribute.GetNamedArgument<string>("DisplayName") ?? type.Name + "." + method.Name; ITypeInfo[] resolvedTypes = null; if (arguments != null && method.IsGenericMethodDefinition) { resolvedTypes = ResolveGenericTypes(method, arguments); method = method.MakeGenericMethod(resolvedTypes); } Assembly = assembly; Class = type; Method = method; Arguments = arguments; DisplayName = GetDisplayNameWithArguments(displayNameBase, arguments, resolvedTypes); SkipReason = factAttribute.GetNamedArgument<string>("Skip"); Traits = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase); TestCollection = testCollection; foreach (var traitAttribute in Method.GetCustomAttributes(typeof(ITraitAttribute)) .Concat(Class.GetCustomAttributes(typeof(ITraitAttribute)))) { var discovererAttribute = traitAttribute.GetCustomAttributes(typeof(TraitDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetTraitDiscoverer(discovererAttribute); if (discoverer != null) foreach (var keyValuePair in discoverer.GetTraits(traitAttribute)) Traits.Add(keyValuePair.Key, keyValuePair.Value); } uniqueID = new Lazy<string>(GetUniqueID, true); }
/// <inheritdoc/> public virtual IEnumerable<object[]> GetData(IAttributeInfo dataAttribute, IMethodInfo testMethod) { var reflectionDataAttribute = dataAttribute as IReflectionAttributeInfo; var reflectionTestMethod = testMethod as IReflectionMethodInfo; if (reflectionDataAttribute != null && reflectionTestMethod != null) { var attribute = (DataAttribute)reflectionDataAttribute.Attribute; try { return attribute.GetData(reflectionTestMethod.MethodInfo); } catch (ArgumentException) { // If we couldn't find the data on the base type, check if it is in current type. // This allows base classes to specify data that exists on a sub type, but not on the base type. var memberDataAttribute = attribute as MemberDataAttribute; var reflectionTestMethodType = reflectionTestMethod.Type as IReflectionTypeInfo; if (memberDataAttribute != null && memberDataAttribute.MemberType == null) { memberDataAttribute.MemberType = reflectionTestMethodType.Type; } return attribute.GetData(reflectionTestMethod.MethodInfo); } } return null; }
/// <summary> /// Always returns 'false', indicating that discovery of tests is /// not supported. /// </summary> /// <param name="dataAttribute">The attribute</param> /// <param name="testMethod">The method being discovered</param> /// <returns>false</returns> public override bool SupportsDiscoveryEnumeration( IAttributeInfo dataAttribute, IMethodInfo testMethod) { // The data return by AutoDataAttribute is (like AutoFixture itself) typically // not 'stable'. In other words, the data often changes (e.g. string guids), and // therefore pre-discovery of tests decorated with this attribute is not possible. return false; }
public KuduXunitTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay testMethodDisplay, ITestMethod testMethod, object[] testMethodArguments, IAttributeInfo testAttribute) : base(diagnosticMessageSink, testMethodDisplay, testMethod, testMethodArguments) { }
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true); //in mixed mode we do not want to run any api tests for plugins when running against a snapshot //because the client is "hot" var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection); return TestClient.Configuration.RunIntegrationTests && RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType); }
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo testAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { DisableRetry = testAttribute == null ? true : testAttribute.GetNamedArgument<bool>("DisableRetry"); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries"); if (maxRetries < 1) maxRetries = 3; yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries); }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { Guard.AgainstNullArgument("discoveryOptions", discoveryOptions); yield return new ScenarioOutline( this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (testMethod.Method.ReturnType.Name == "System.Void" && testMethod.Method.GetCustomAttributes(typeof(AsyncStateMachineAttribute)).Any()) { return new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "Async void methods are not supported."); } return new LegacyUITestCase(UITestCase.SyncContextType.WPF, diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
public IEnumerable<PerformanceMetricInfo> GetMetrics(IAttributeInfo metricAttribute) { if (_profileSource != -1) { var interval = (int)(metricAttribute.GetConstructorArguments().FirstOrDefault() ?? DefaultInterval); yield return new InstructionsRetiredMetric(interval, _profileSource); } }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute ) { yield return new FarmDependentTestCase(_diagnosticSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
/// <summary> /// Gets the trait values from the trait attribute. /// </summary> /// <param name="traitAttribute">The trait attribute containing the trait values.</param> /// <returns>The trait values</returns> public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute) { IEnumerator<object> enumerator = traitAttribute.GetConstructorArguments().GetEnumerator(); while (enumerator.MoveNext()) { yield return new KeyValuePair<string, string>(enumerator.Current.ToString(), ""); } }
public IEnumerable<KeyValuePair<string, string>> GetTraits(IAttributeInfo traitAttribute) { var array = (CompilerFeature[])traitAttribute.GetConstructorArguments().Single(); foreach (var feature in array) { var value = feature.ToString(); yield return new KeyValuePair<string, string>("Compiler", value); } }
protected override String GetSkipReason(IAttributeInfo factAttribute) { if (!FarmJoined) { return "This test needs to run on a server joined to a SharePoint 2013 farm."; } return base.GetSkipReason(factAttribute); }
public KuduXunitTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay testMethodDisplay, ITestMethod testMethod, object[] testMethodArguments, IAttributeInfo testAttribute) : base(diagnosticMessageSink, testMethodDisplay, testMethod, testMethodArguments) { DisableRetry = testAttribute == null ? true : testAttribute.GetNamedArgument<bool>("DisableRetry"); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (Helper.Organization == null) { return Enumerable.Empty<IXunitTestCase>(); } return new[] { new XunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) }; }
/// <summary> /// Discover test cases from a test method. By default, inspects the test method's argument list /// to ensure it's empty, and if not, returns a single <see cref="ExecutionErrorTestCase"/>; /// otherwise, it returns the result of calling <see cref="CreateTestCase"/>. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="factAttribute">The fact attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var testCase = testMethod.Method.GetParameters().Any() ? new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to have parameters. Did you mean to use [Theory]?") : CreateTestCase(discoveryOptions, testMethod, factAttribute); return new[] { testCase }; }
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. public MefFactTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions testMethodDisplayOptions, ITestMethod testMethod, IAttributeInfo factAttributeInfo) : base(diagnosticMessageSink, defaultMethodDisplay, testMethodDisplayOptions, testMethod) { var factAttribute = MefFactAttribute.Instantiate(factAttributeInfo); this.SkipReason = factAttribute.Skip; this.parts = factAttribute.Parts; this.assemblies = factAttribute.Assemblies; this.compositionVersions = factAttribute.CompositionVersions; this.noCompatGoal = factAttribute.NoCompatGoal; this.invalidConfiguration = factAttribute.InvalidConfiguration; if (this.Traits.ContainsKey(Tests.Traits.SkipOnMono) && TestUtilities.IsOnMono) { this.SkipReason = this.SkipReason ?? "Test marked as skipped on Mono runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnMono]); } if (this.Traits.ContainsKey(Tests.Traits.SkipOnCoreCLR) && TestUtilities.IsOnCoreCLR) { this.SkipReason = this.SkipReason ?? "Test marked as skipped on CoreCLR runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnCoreCLR]); } }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (testMethod.Method.ReturnType.Name == "System.Void" && testMethod.Method.GetCustomAttributes(typeof(AsyncStateMachineAttribute)).Any()) { return(new ExecutionErrorTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "Async void methods are not supported.")); } return(new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { yield return(new SkippableFactTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)); }
/// <summary> /// Gets a test collection orderer, as specified in a reflected <see cref="TestCollectionOrdererAttribute"/>. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="testCollectionOrdererAttribute">The test collection orderer attribute.</param> /// <returns>The test collection orderer, if the type is loadable; <c>null</c>, otherwise.</returns> public static ITestCollectionOrderer GetTestCollectionOrderer(IMessageSink diagnosticMessageSink, IAttributeInfo testCollectionOrdererAttribute) { var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast <string>().ToList(); var ordererType = SerializationHelper.GetType(args[1], args[0]); if (ordererType == null) { return(null); } return(GetTestCollectionOrderer(diagnosticMessageSink, ordererType)); }
/// <inheritdoc/> public virtual IEnumerable <KeyValuePair <string, string> > GetTraits(IAttributeInfo traitAttribute) { var ctorArgs = traitAttribute.GetConstructorArguments().Cast <string>().ToList(); yield return(new KeyValuePair <string, string>(ctorArgs[0], ctorArgs[1])); }
/// <summary> /// Gets a test framework discoverer, as specified in a reflected <see cref="TestFrameworkDiscovererAttribute"/>. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="testFrameworkDiscovererAttribute">The test framework discoverer attribute</param> public static ITestFrameworkTypeDiscoverer GetTestFrameworkTypeDiscoverer(IMessageSink diagnosticMessageSink, IAttributeInfo testFrameworkDiscovererAttribute) { var args = testFrameworkDiscovererAttribute.GetConstructorArguments().Cast <string>().ToArray(); var testFrameworkDiscovererType = SerializationHelper.GetType(args[1], args[0]); if (testFrameworkDiscovererType == null) { return(null); } return(GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, testFrameworkDiscovererType)); }
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { var testCase = new WpfTheoryTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod); return(new[] { testCase }); }
protected virtual string GetSkipReason(IAttributeInfo factAttribute) { return(factAttribute.GetNamedArgument <string>("Skip")); }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (ConditionalTestDiscoverer.TryEvaluateSkipConditions(discoveryOptions, DiagnosticMessageSink, testMethod, factAttribute.GetConstructorArguments().ToArray(), out string skipReason, out ExecutionErrorTestCase errorTestCase)) { return(skipReason != null ? (IXunitTestCase) new SkippedTestCase(skipReason, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod) : new SkippedFactTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod)); // Test case skippable at runtime. } return(errorTestCase); }
/// <summary> /// Gets the trait values from the Category attribute. /// </summary> /// <param name="traitAttribute">The trait attribute containing the trait values.</param> /// <returns>The trait values.</returns> public IEnumerable <KeyValuePair <string, string> > GetTraits(IAttributeInfo traitAttribute) { yield return(new KeyValuePair <string, string>(XunitConstants.Category, XunitConstants.OuterLoop)); }
/// <summary> /// Initializes a new instance of the <see cref="XunitTheoryTestCase"/> class. /// </summary> /// <param name="testCollection">The test collection this theory belongs to.</param> /// <param name="assembly">The test assembly.</param> /// <param name="type">The type under test.</param> /// <param name="method">The method under test.</param> /// <param name="theoryAttribute">The theory attribute.</param> public XunitTheoryTestCase(ITestCollection testCollection, IAssemblyInfo assembly, ITypeInfo type, IMethodInfo method, IAttributeInfo theoryAttribute) : base(testCollection, assembly, type, method, theoryAttribute) { }
protected override string GetSkipReason(IAttributeInfo factAttribute) => _skipReason ?? base.GetSkipReason(factAttribute);
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (Environment.GetEnvironmentVariable("Appveyor")?.ToUpperInvariant() == "TRUE") { return(Enumerable.Empty <IXunitTestCase>()); } return(new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) }); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttributeInfo) { var methodDisplay = discoveryOptions.MethodDisplayOrDefault(); yield return(new MefFactTestCase(this.diagnosticMessageSink, methodDisplay, TestMethodDisplayOptions.None, testMethod, factAttributeInfo)); }
/// <summary> /// Gets an xUnit.net v2 test collection factory, as specified in a reflected <see cref="CollectionBehaviorAttribute"/>. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="collectionBehaviorAttribute">The collection behavior attribute.</param> /// <param name="testAssembly">The test assembly.</param> /// <returns>The collection factory.</returns> public static IXunitTestCollectionFactory GetXunitTestCollectionFactory(IMessageSink diagnosticMessageSink, IAttributeInfo collectionBehaviorAttribute, ITestAssembly testAssembly) { try { var testCollectionFactoryType = GetTestCollectionFactoryType(diagnosticMessageSink, collectionBehaviorAttribute); return(GetXunitTestCollectionFactory(diagnosticMessageSink, testCollectionFactoryType, testAssembly)); } catch { return(new CollectionPerClassTestCollectionFactory(testAssembly, diagnosticMessageSink)); } }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var skipReason = testMethod.EvaluateSkipConditions(); return(skipReason != null ? new SkippedTestCase(skipReason, _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod) : base.CreateTestCase(discoveryOptions, testMethod, factAttribute)); }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) => new WpfTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
protected override IEnumerable <IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { return(new[] { new UITestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, dataRow), }); }
public override IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var results = new List <IXunitTestCase>(); if (testMethod.Method.GetParameters().Any()) { results.Add(new ExecutionErrorTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[VsFact] methods are not allowed to have parameters. Did you mean to use [VsTheory]?")); } else if (testMethod.Method.IsGenericMethodDefinition) { results.Add(new ExecutionErrorTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[VsFact] methods are not allowed to be generic.")); } else { try { results.AddRange(VsTestCaseFactory.CreateTestCases(testMethod, null, discoveryOptions.MethodDisplayOrDefault(), DiagnosticMessageSink)); } catch (Exception exception) { results.Add(new ExceptionTestCase(exception, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)); } } return(results); }
/// <summary> /// Ensures the assembly runner is initialized (sets up the collection behavior, /// parallelization options, and test orderers from their assembly attributes). /// </summary> protected void Initialize() { if (initialized) { return; } collectionBehaviorAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(CollectionBehaviorAttribute)).SingleOrDefault(); if (collectionBehaviorAttribute != null) { disableParallelization = collectionBehaviorAttribute.GetNamedArgument <bool>("DisableTestParallelization"); maxParallelThreads = collectionBehaviorAttribute.GetNamedArgument <int>("MaxParallelThreads"); } disableParallelization = ExecutionOptions.DisableParallelization() ?? disableParallelization; var maxParallelThreadsOption = ExecutionOptions.MaxParallelThreads() ?? 0; if (maxParallelThreadsOption > 0) { maxParallelThreads = maxParallelThreadsOption; } var testCaseOrdererAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (testCaseOrdererAttribute != null) { try { var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(DiagnosticMessageSink, testCaseOrdererAttribute); if (testCaseOrderer != null) { TestCaseOrderer = testCaseOrderer; } else { var args = testCaseOrdererAttribute.GetConstructorArguments().Cast <string>().ToList(); DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Could not find type '{0}' in {1} for assembly-level test case orderer", args[0], args[1])); } } catch (Exception ex) { var innerEx = ex.Unwrap(); var args = testCaseOrdererAttribute.GetConstructorArguments().Cast <string>().ToList(); DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test case orderer '{0}' threw '{1}' during construction: {2}", args[0], innerEx.GetType().FullName, innerEx.StackTrace)); } } var testCollectionOrdererAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(TestCollectionOrdererAttribute)).SingleOrDefault(); if (testCollectionOrdererAttribute != null) { try { var testCollectionOrderer = ExtensibilityPointFactory.GetTestCollectionOrderer(DiagnosticMessageSink, testCollectionOrdererAttribute); if (testCollectionOrderer != null) { TestCollectionOrderer = testCollectionOrderer; } else { var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast <string>().ToList(); DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Could not find type '{0}' in {1} for assembly-level test collection orderer", args[0], args[1])); } } catch (Exception ex) { var innerEx = ex.Unwrap(); var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast <string>().ToList(); DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test collection orderer '{0}' threw '{1}' during construction: {2}", args[0], innerEx.GetType().FullName, innerEx.StackTrace)); } } initialized = true; }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (CollectionId.HasValue) { testMethod = WrapTestMethod(testMethod, CollectionId.Value); } if (!Process.GetCurrentProcess().ProcessName.Equals(Path.GetFileNameWithoutExtension(_ExePath), StringComparison.OrdinalIgnoreCase)) { yield return(new XUnitRemoteTestCase(_DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, _Id, _ExePath)); } else { foreach (var testCase in _DefaultTestCaseDiscoverer.Discover(discoveryOptions, testMethod, factAttribute)) { yield return(_TestCaseConverter(testCase)); } } }
/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplay(); // Special case Skip, because we want a single Skip (not one per data item), and a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). if (factAttribute.GetNamedArgument <string>("Skip") != null) { return new[] { new XunitTestCase(defaultMethodDisplay, testMethod) } } ; var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); if (discoveryOptions.PreEnumerateTheories()) { try { using (var memoryStream = new MemoryStream()) { var results = new List <XunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererAttribute); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { // Attempt to serialize the test case, since we need a way to uniquely identify a test // and serialization is the best way to do that. If it's not serializable, this will // throw and we will fall back to a single theory test case that gets its data // at runtime. var testCase = new XunitTestCase(defaultMethodDisplay, testMethod, dataRow); SerializationHelper.Serialize(testCase); results.Add(testCase); } } // REVIEW: Could we re-write LambdaTestCase to just be for exceptions? if (results.Count == 0) { results.Add(new LambdaTestCase(defaultMethodDisplay, testMethod, () => { throw new InvalidOperationException(String.Format("No data found for {0}.{1}", testMethod.TestClass.Class.Name, testMethod.Method.Name)); })); } return(results); } } catch { } } return(new XunitTestCase[] { new XunitTheoryTestCase(defaultMethodDisplay, testMethod) }); } }
public virtual IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { try { var skipReason = EvaluateSkipConditions(testMethod); if (skipReason != null) { _diagnosticMessageSink.OnMessage( new XunitDiagnosticMessage($"Skipping { testMethod.Method.Name }{ Environment.NewLine }Reason: { skipReason }")); return(new List <IXunitTestCase>()); } var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .Select(a => new { Name = a.GetNamedArgument <string>(nameof(BenchmarkVariationAttribute.VariationName)), TestMethodArguments = a.GetNamedArgument <object[]>(nameof(BenchmarkVariationAttribute.Data)), Framework = a.GetNamedArgument <string>(nameof(BenchmarkVariationAttribute.Framework)) }) .ToList(); if (!variations.Any()) { variations.Add(new { Name = "Default", TestMethodArguments = new object[0], Framework = (string)null }); } var tests = new List <IXunitTestCase>(); foreach (var variation in variations) { if (BenchmarkConfig.Instance.RunIterations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument <int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Framework, variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } else { tests.Add(new NonCollectingBenchmarkTestCase( variation.Name, _diagnosticMessageSink, testMethod, variation.TestMethodArguments)); } } return(tests); } catch (Exception e) { Console.WriteLine(e); Console.WriteLine(e.StackTrace); throw; } }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { var scopes = theoryAttribute.GetNamedArgument <string[]>(nameof(DataDrivenTestAttribute.Environments)); if (scopes?.Length > 0) { var env = Environment.GetEnvironmentVariable(Constants.AspNetCoreEnvironment); if (env != null && !scopes.Contains(env, StringComparer.OrdinalIgnoreCase)) { return(new[] { new SkipTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"This test does not apply in '{env}' environment, skipping.") }); } } var skipReason = theoryAttribute.GetNamedArgument <string>(nameof(FactAttribute.Skip)); if (skipReason != null) { return(CreateTestCasesForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason)); } if (!discoveryOptions.PreEnumerateTheoriesOrDefault()) { return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } try { var customAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var testCases = new List <IXunitTestCase>(); foreach (var attributeInfo in customAttributes) { var dataDiscovererAttribute = attributeInfo.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); IDataDiscoverer dataDiscoverer; try { dataDiscoverer = ExtensibilityPointFactory.GetDataDiscoverer(_diagnosticMessageSink, dataDiscovererAttribute); } catch (InvalidCastException) { if (attributeInfo is IReflectionAttributeInfo reflectionAttributeInfo) { testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttributeInfo.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer.")); continue; } testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer.")); continue; } if (dataDiscoverer == null) { if (attributeInfo is IReflectionAttributeInfo reflectionAttributeInfo) { testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Data discoverer specified for {reflectionAttributeInfo.Attribute.GetType()} on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist.")); } else { testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist.")); } } else { skipReason = attributeInfo.GetNamedArgument <string>(nameof(FactAttribute.Skip)); if (!dataDiscoverer.SupportsDiscoveryEnumeration(attributeInfo, testMethod.Method)) { return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var data = dataDiscoverer.GetData(attributeInfo, testMethod.Method); if (data == null) { testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}. Make sure it is statically initialized before this test method is called.")); } else { foreach (var dataRow in data) { if (!XunitSerializationInfo.CanSerializeObject(dataRow)) { _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.")); return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); } var collection = skipReason != null?CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) : CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); testCases.AddRange(collection); } } } } if (testCases.Count == 0) { testCases.Add(new ExecutionErrorTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "No data found for " + testMethod.TestClass.Class.Name + "." + testMethod.Method.Name)); } return(testCases); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute)); }
public Type GetTestFrameworkType(IAttributeInfo attribute) => typeof(FusonicTestFramework);
/// <summary> /// Gets a data discoverer, as specified in a reflected <see cref="DataDiscovererAttribute"/>. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="dataDiscovererAttribute">The data discoverer attribute</param> /// <returns>The data discoverer, if the type is loadable; <c>null</c>, otherwise.</returns> public static IDataDiscoverer GetDataDiscoverer(IMessageSink diagnosticMessageSink, IAttributeInfo dataDiscovererAttribute) { var args = dataDiscovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); if (discovererType == null) { return(null); } return(GetDataDiscoverer(diagnosticMessageSink, discovererType)); }
protected virtual string GetDisplayName(IAttributeInfo factAttribute, string displayName) { return(TestMethod.Method .GetDisplayNameWithArguments(displayName, TestMethodArguments, MethodGenericTypes)); }
/// <inheritdoc/> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { return(WrappedDiscoverer.Discover(discoveryOptions, testMethod, factAttribute).Select(t => WrapTestCase(t, factAttribute))); }
/// <summary> /// Implementation of <see cref="IXunitTestCaseDiscoverer"/> /// </summary> public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { List <IXunitTestCase> result = new List <IXunitTestCase>(); string assemblyName = factAttribute.GetNamedArgument <string>("AssemblyName"); System.Reflection.Assembly testAssembly = null; if (assemblyName == null) { testAssembly = testMethod.TestClass.Class.ToRuntimeType().Assembly; } else { try { testAssembly = System.Reflection.Assembly.Load(assemblyName); } catch (Exception e) { result.Add(SkipErrorTestCase(discoveryOptions, testMethod, "Assembly load issue: " + e.Message)); return(result); } } string testNamespace = factAttribute.GetNamedArgument <string>("TestNamespace") ?? testMethod.TestClass.Class.ToRuntimeType().Namespace; if (testNamespace == null) { result.Add(SkipErrorTestCase(discoveryOptions, testMethod, "Could not find the namespase with tests.")); return(result); } string Suffix = factAttribute.GetNamedArgument <string>("Suffix"); string skipReason = factAttribute.GetNamedArgument <string>("Skip"); string testCasePrefix = factAttribute.GetNamedArgument <string>("TestCasePrefix") ?? ""; IEnumerable <Type> ourTypes = from definedType in testAssembly.DefinedTypes where definedType.Name.EndsWith(Suffix) where definedType.Namespace == testNamespace where typeof(AbstractCallable).IsAssignableFrom(definedType) where typeof(ICallable <QVoid, QVoid>).IsAssignableFrom(definedType) select definedType; foreach (Type operationType in ourTypes) { result.Add( new TestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplay() ?? TestMethodDisplay.Method, testMethod, new TestOperation() { assemblyName = testAssembly.FullName, className = operationType.Name, fullClassName = operationType.FullName, skipReason = skipReason, testCaseNamePrefix = testCasePrefix }) ); } if (result.Count == 0) { result.Add(SkipErrorTestCase(discoveryOptions, testMethod, "No tests were found.")); } return(result); }
protected override IXunitTestCase CreateTestCaseForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { var parameters = new TestParameters(theoryAttribute); return(new TestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, parameters, dataRow)); }