public override IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod(); string conditionMemberName = factAttribute.GetConstructorArguments().FirstOrDefault() as string; MethodInfo conditionMethodInfo; if (conditionMemberName == null || (conditionMethodInfo = LookupConditionalMethod(testMethodInfo.DeclaringType, conditionMemberName)) == null) { return new[] { new ExecutionErrorTestCase( _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, GetFailedLookupString(conditionMemberName)) }; } IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute); if ((bool)conditionMethodInfo.Invoke(null, null)) { return testCases; } else { string skippedReason = "\"" + conditionMemberName + "\" returned false."; return testCases.Select(tc => new SkippedTestCase(tc, skippedReason)); } }
/// <summary> /// Creates a result record. /// </summary> /// <param name="method">Test method metadata object.</param> /// <param name="testClass">Test class metadata object.</param> /// <param name="result">Test result object.</param> /// <param name="exception">Exception instance, if any.</param> public ScenarioResult(ITestMethod method, ITestClass testClass, TestOutcome result, Exception exception) { TestClass = testClass; TestMethod = method; Exception = exception; Result = result; }
/// <summary> /// Finds the tests on a test method. /// </summary> /// <param name="testMethod">The test method.</param> /// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param> /// <param name="messageBus">The message bus to report discovery messages to.</param> /// <param name="discoveryOptions">The options used by the test framework during discovery.</param> /// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns> protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) { var factAttribute = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault(); if (factAttribute == null) return true; var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault(); if (testCaseDiscovererAttribute == null) return true; var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); if (discovererType == null) return true; var discoverer = GetDiscoverer(discovererType); if (discoverer == null) return true; foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute)) if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus)) return false; return true; }
public BenchmarkTestCase( int iterations, int warmupIterations, string variation, IMessageSink diagnosticMessageSink, ITestMethod testMethod, object[] testMethodArguments) : base(diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay.Method, testMethod, null) { // Override display name to avoid getting info about TestMethodArguments in the // name (this is covered by the concept of Variation for benchmarks) var suppliedDisplayName = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute)) .First() .GetNamedArgument<string>("DisplayName"); _diagnosticMessageSink = diagnosticMessageSink; DisplayName = suppliedDisplayName ?? BaseDisplayName; Variation = variation; Iterations = iterations; WarmupIterations = warmupIterations; var methodArguments = new List<object> { MetricCollector }; if (testMethodArguments != null) { methodArguments.AddRange(testMethodArguments); } TestMethodArguments = methodArguments.ToArray(); }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault(); return factAttribute.GetNamedArgument<string>("Skip") != null ? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) } : new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }; }
protected override void Before_all_tests() { base.Before_all_tests(); methodInfo = MockNUnitTestClass.GetPassingTest(); testMethod = new Xunit.TestMethod(methodInfo); }
/// <summary> /// Constructor for a test method manager, which handles executing a single test method /// for a unit test provider. /// </summary> /// <param name="testHarness">The unit test harness object.</param> /// <param name="testClass">The test class metadata object.</param> /// <param name="testMethod">The test method metadata object.</param> /// <param name="instance">The test class instance.</param> /// <param name="provider">The unit test provider.</param> public TestMethodManager(UnitTestHarness testHarness, ITestClass testClass, ITestMethod testMethod, object instance, IUnitTestProvider provider) : base(testHarness, provider) { _testClass = testClass; _testMethod = testMethod; _instance = instance; }
public IEnumerable<IXunitTestCase> Discover (ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault (); if (testMethod.Method.GetParameters ().Any ()) { return new IXunitTestCase[] { new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, "[VsixFact] methods are not allowed to have parameters.") }; } else { var vsVersions = VsVersions.GetFinalVersions(testMethod.GetComputedProperty<string[]>(factAttribute, SpecialNames.VsixAttribute.VisualStudioVersions)); // Process VS-specific traits. var suffix = testMethod.GetComputedArgument<string>(factAttribute, SpecialNames.VsixAttribute.RootSuffix) ?? "Exp"; var newInstance = testMethod.GetComputedArgument<bool?>(factAttribute, SpecialNames.VsixAttribute.NewIdeInstance); var timeout = testMethod.GetComputedArgument<int?>(factAttribute, SpecialNames.VsixAttribute.TimeoutSeconds).GetValueOrDefault(XunitExtensions.DefaultTimeout); var testCases = new List<IXunitTestCase>(); // Add invalid VS versions. testCases.AddRange (vsVersions .Where (v => !VsVersions.InstalledVersions.Contains (v)) .Select (v => new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, string.Format ("Cannot execute test for specified {0}={1} because there is no VSSDK installed for that version.", SpecialNames.VsixAttribute.VisualStudioVersions, v)))); testCases.AddRange (vsVersions .Where (v => VsVersions.InstalledVersions.Contains (v)) .Select (v => new VsixTestCase (messageSink, defaultMethodDisplay, testMethod, v, suffix, newInstance, timeout))); return testCases; } }
/// <summary> /// Initializes a new test run filter using an existing settings file. /// </summary> /// <param name="test">The test class metadata.</param> /// <param name="method">The test method metadata.</param> public RetryTestRunFilter(ITestClass test, ITestMethod method) : base(null, null) { TestRunName = "Retry of " + test.Name + " " + method.Name; _test = test; _method = method; }
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var variations = testMethod.Method .GetCustomAttributes(typeof(BenchmarkVariationAttribute)) .ToDictionary( a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)), a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data))); if (!variations.Any()) { variations.Add("Default", new object[0]); } var tests = new List<IXunitTestCase>(); foreach (var variation in variations) { tests.Add(new BenchmarkTestCase( factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)), factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)), variation.Key, _diagnosticMessageSink, testMethod, variation.Value)); } return tests; }
/// <summary> /// Creates a new method invoke work item for a MethodInfo instance. /// </summary> /// <param name="instance">The type instance.</param> /// <param name="method">The method on the type to invoke when the /// work item is executed.</param> /// <param name="testMethod">The test method metadata.</param> public MethodInvokeWorkItem(object instance, MethodInfo method, ITestMethod testMethod) : base() { _instance = instance; _method = method; _testMethod = testMethod; }
public override IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { string[] conditionMemberNames = factAttribute.GetConstructorArguments().FirstOrDefault() as string[]; IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute); return ConditionalTestDiscoverer.Discover(discoveryOptions, _diagnosticMessageSink, testMethod, testCases, conditionMemberNames); }
/// <summary> /// Constructs a new method container. /// </summary> /// <param name="instance">An instance of the method's type.</param> /// <param name="method">The method reflection object.</param> /// <param name="testMethod">The test method.</param> public MethodContainer(object instance, MethodInfo method, ITestMethod testMethod) : base() { _methodTask = new MethodInvokeWorkItem(instance, method, testMethod); _methodInfo = method; _method = testMethod; }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var skipReason = EvaluateSkipConditions(testMethod); var isTheory = false; IXunitTestCaseDiscoverer innerDiscoverer; if (testMethod.Method.GetCustomAttributes(typeof(TheoryAttribute)).Any()) { isTheory = true; innerDiscoverer = new TheoryDiscoverer(_diagnosticMessageSink); } else { innerDiscoverer = new FactDiscoverer(_diagnosticMessageSink); } var testCases = innerDiscoverer .Discover(discoveryOptions, testMethod, factAttribute) .Select(testCase => new SkipReasonTestCase(isTheory, skipReason, testCase)); return testCases; }
/// <summary> /// Finds the tests on a test method. /// </summary> /// <param name="testMethod">The test method.</param> /// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param> /// <param name="messageBus">The message bus to report discovery messages to.</param> /// <param name="discoveryOptions">The options used by the test framework during discovery.</param> /// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns> protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) { var factAttributes = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).CastOrToList(); if (factAttributes.Count > 1) { var message = string.Format("Test method '{0}.{1}' has multiple [Fact]-derived attributes", testMethod.TestClass.Class.Name, testMethod.Method.Name); var testCase = new ExecutionErrorTestCase(DiagnosticMessageSink, TestMethodDisplay.ClassAndMethod, testMethod, message); return ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus); } var factAttribute = factAttributes.FirstOrDefault(); if (factAttribute == null) return true; var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault(); if (testCaseDiscovererAttribute == null) return true; var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); if (discovererType == null) return true; var discoverer = GetDiscoverer(discovererType); if (discoverer == null) return true; foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute)) if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus)) return false; return true; }
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo testAttribute) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { }
protected override void Before_all_tests() { base.Before_all_tests(); methodInfo = MockXunitTestClass.GetIgnoreTest(); testMethod = new TestMethod(methodInfo); }
public static void AssignTestExecutionMethodInfo(this TestExecutionMethod testExecutionMethod, ITestMethod testMethod) { var methodInfo = testMethod.Method; testExecutionMethod.NamespaceName = methodInfo.ReflectedType.Namespace; testExecutionMethod.ClassName = methodInfo.ReflectedType.ClassNameIncludingParentsIfNested(); testExecutionMethod.MethodName = testMethod.Name; }
public IEnumerable<IXunitTestCase> Discover ( ITestFrameworkDiscoveryOptions discoveryOptions , ITestMethod testMethod , IAttributeInfo factAttribute ) { var inv = InvariantTestCase.InvariantFromMethod(testMethod); return inv == null ? new[] { new InvariantTestCase ( MessageSink , TestMethodDisplay.Method , testMethod , null ) } : inv.AsSeq().Select ( i => new InvariantTestCase ( MessageSink , TestMethodDisplay.Method , testMethod , i ) ); }
protected BenchmarkTestCaseBase( string variation, IMessageSink diagnosticMessageSink, ITestMethod testMethod, object[] testMethodArguments) : base(diagnosticMessageSink, TestMethodDisplay.Method, testMethod, null) { // Override display name to avoid getting info about TestMethodArguments in the // name (this is covered by the concept of Variation for benchmarks) var name = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute)) .First() .GetNamedArgument<string>("DisplayName") ?? BaseDisplayName; TestMethodName = name; DisplayName = $"{name} [Variation: {variation}]"; DiagnosticMessageSink = diagnosticMessageSink; Variation = variation; var methodArguments = new List<object> { MetricCollector }; if (testMethodArguments != null) { methodArguments.AddRange(testMethodArguments); } TestMethodArguments = methodArguments.ToArray(); }
/// <summary> /// Initializes dispatcher-stack attaching method container work item. /// </summary> /// <param name="testHarness">Test harness.</param> /// <param name="instance">Test instance.</param> /// <param name="method">Method reflection object.</param> /// <param name="testMethod">Test method metadata.</param> /// <param name="granularity">Granularity of test.</param> public UnitTestMethodContainer(UnitTestHarness testHarness, object instance, MethodInfo method, ITestMethod testMethod, TestGranularity granularity) : base(instance, method, testMethod) { _granularity = granularity; _harness = testHarness as UnitTestHarness; _testMethod = testMethod; }
public bool Contains(ITestMethod testMethod) { foreach (var criterion in _criteria) { if (criterion.Contains(testMethod)) { return true; } } return false; }
protected override void Before_all_tests() { base.Before_all_tests(); _methodInfo = MockNUnitTestClass.GetPassingTest(); _testMethod = new TestMethod(_methodInfo); _testMethod.ShouldNotBeNull(); }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute ) { yield return new FarmDependentTestCase(_diagnosticSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
public IEnumerable<IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { Guard.AgainstNullArgument("discoveryOptions", discoveryOptions); yield return new ScenarioOutline( this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
public NonCollectingBenchmarkTestCase( string variation, IMessageSink diagnosticMessageSink, ITestMethod testMethod, object[] testMethodArguments) : base(variation, diagnosticMessageSink, testMethod, testMethodArguments) { }
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries"); if (maxRetries < 1) maxRetries = 3; yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries); }
/// <summary> /// Initializes a new instance of the <see cref="XunitTestCase"/> class. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param> /// <param name="testMethod">The test method this test case belongs to.</param> /// <param name="testMethodArguments">The arguments for the test method.</param> public XunitTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object[] testMethodArguments = null) : base(defaultMethodDisplay, testMethod, testMethodArguments) { this.diagnosticMessageSink = diagnosticMessageSink; }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (testMethod.Method.ReturnType.Name == "System.Void" && testMethod.Method.GetCustomAttributes(typeof(AsyncStateMachineAttribute)).Any()) { return new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "Async void methods are not supported."); } return new LegacyUITestCase(UITestCase.SyncContextType.WPF, diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod); }
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true); //in mixed mode we do not want to run any api tests for plugins when running against a snapshot //because the client is "hot" var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection); return TestClient.Configuration.RunIntegrationTests && RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType); }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) => new SkippableTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (Environment.GetEnvironmentVariable("CI")?.ToUpperInvariant() == "TRUE") { return(Enumerable.Empty <IXunitTestCase>()); } return(new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.All, testMethod) }); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { return(_inner.Discover(discoveryOptions, testMethod, factAttribute).Select(t => new ForegroundFactTestCase(t))); }
/// <summary> /// Creates a test case for a skipped theory. By default, returns an instance of <see cref="XunitTestCase"/> /// (which inherently discovers the skip reason via the fact attribute). /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="skipReason">The skip reason that decorates <paramref name="theoryAttribute"/>.</param> /// <returns>The test case</returns> protected virtual NBenchTestCase CreateTestCaseForSkip(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, string skipReason) => new NBenchTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
/// <summary> /// Creates a test case for a single row of data. By default, returns an instance of <see cref="XunitSkippedDataRowTestCase"/> /// with the data row inside of it. /// </summary> /// <remarks>If this method is overridden, the implementation will have to override <see cref="TestMethodTestCase.SkipReason"/> otherwise /// the default behavior will look at the <see cref="TheoryAttribute"/> and the test case will not be skipped.</remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="dataRow">The row of data for this test case.</param> /// <param name="skipReason">The reason this test case is to be skipped</param> /// <returns>The test case</returns> //protected virtual IXunitTestCase CreateTestCaseForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason) // => new XunitSkippedDataRowTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, skipReason, dataRow); /// <summary> /// Discover test cases from a test method. /// </summary> /// <remarks> /// This method performs the following steps: /// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>; /// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>; /// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it; /// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason. /// </remarks> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) { // Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may // not actually have any data (which is quasi-legal, since it's skipped). var skipReason = theoryAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) } } ; if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)); var results = new List <IXunitTestCase>(); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute); //skipReason = dataAttribute.GetNamedArgument<string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method)) { return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) } } ; // GetData may return null, but that's okay; we'll let the NullRef happen and then catch it // down below so that we get the composite test case. foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method)) { //TODO: Revisit when xUnit beta is released to Production // Determine whether we can serialize the test case, since we need a way to uniquely // identify a test and serialization is the best way to do that. If it's not serializable, // this will throw and we will fall back to a single theory test case that gets its data at runtime. //if (!SerializationHelper.IsSerializable(dataRow)) //{ // diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.")); // return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }; //} //var testCase = // skipReason != null // ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) // : CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); var testCase = CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); results.Add(testCase); } } if (results.Count == 0) { results.Add(new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } } return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }); } }
// This helper method evaluates the given condition member names for a given set of test cases. // If any condition member evaluates to 'false', the test cases are marked to be skipped. // The skip reason is the collection of all the condition members that evalated to 'false'. internal static IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, IMessageSink diagnosticMessageSink, ITestMethod testMethod, IEnumerable <IXunitTestCase> testCases, object[] conditionArguments) { // A null or empty list of conditionMemberNames is treated as "no conditions". // and the test cases will not be skipped. // Example: [ConditionalFact()] if (conditionArguments == null || conditionArguments.Length == 0) { return(testCases); } string [] conditionMemberNames; Type calleeType = conditionArguments[0] as Type; if (calleeType != null) { if (conditionArguments.Length < 2) { // [ConditionalFact(typeof(x))] no provided methods. return(testCases); } // [ConditionalFact(typeof(x), "MethodName")] conditionMemberNames = conditionArguments[1] as string[]; } else { // [ConditionalFact("MethodName")] conditionMemberNames = conditionArguments[0] as string[]; } // [ConditionalFact((string[]) null)] int conditionCount = conditionMemberNames == null ? 0 : conditionMemberNames.Count(); if (conditionCount == 0) { return(testCases); } MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod(); Type testMethodDeclaringType = testMethodInfo.DeclaringType; List <string> falseConditions = new List <string>(conditionCount); foreach (string entry in conditionMemberNames) { string conditionMemberName = entry; // Null condition member names are silently tolerated if (string.IsNullOrWhiteSpace(conditionMemberName)) { continue; } Type declaringType; if (calleeType != null) { declaringType = calleeType; } else { declaringType = testMethodDeclaringType; string[] symbols = conditionMemberName.Split('.'); if (symbols.Length == 2) { conditionMemberName = symbols[1]; ITypeInfo type = testMethod.TestClass.Class.Assembly.GetTypes(false).Where(t => t.Name.Contains(symbols[0])).FirstOrDefault(); if (type != null) { declaringType = type.ToRuntimeType(); } } } MethodInfo conditionMethodInfo; if ((conditionMethodInfo = LookupConditionalMethod(declaringType, conditionMemberName)) == null) { return(new[] { new ExecutionErrorTestCase( diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, GetFailedLookupString(conditionMemberName, declaringType)) }); } // In the case of multiple conditions, collect the results of all // of them to produce a summary skip reason. try { if (!(bool)conditionMethodInfo.Invoke(null, null)) { falseConditions.Add(conditionMemberName); } } catch (Exception exc) { falseConditions.Add($"{conditionMemberName} ({exc.GetType().Name})"); } } // Compose a summary of all conditions that returned false. if (falseConditions.Count > 0) { string skippedReason = string.Format("Condition(s) not met: \"{0}\"", string.Join("\", \"", falseConditions)); return(testCases.Select(tc => new SkippedTestCase(tc, skippedReason))); } // No conditions returned false (including the absence of any conditions). return(testCases); }
/// <summary> /// Initializes a new instance of the <see cref="XunitTheoryTestCase"/> class. /// </summary> /// <param name="testMethod">The method under test.</param> public XunitTheoryTestCase(ITestMethod testMethod) : base(testMethod) { }
/// <summary> /// Initializes a new instance of the <see cref="TestMethodTestCase"/> class. /// </summary> /// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param> /// <param name="testMethod">The test method this test case belongs to.</param> /// <param name="testMethodArguments">The arguments for the test method.</param> protected TestMethodTestCase(TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object[] testMethodArguments = null) { DefaultMethodDisplay = defaultMethodDisplay; TestMethod = testMethod; TestMethodArguments = testMethodArguments; }
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) => new[] { new SkippableTheoryTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod) };
protected override IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason) => new[] { new NamedSkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, skipReason, dataRow) };
public SkippableTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, object[] testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) { }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { return(Helper.HasCredentials ? new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) } : Enumerable.Empty <IXunitTestCase>()); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { yield return(new SkippableFactTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttributeInfo) { var methodDisplay = discoveryOptions.MethodDisplayOrDefault(); yield return(new MefFactTestCase(this.diagnosticMessageSink, methodDisplay, testMethod, factAttributeInfo)); }
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { string appName = factAttribute.GetNamedArgument <string>("AppName"); string appAssembly = factAttribute.GetNamedArgument <string>("AppAssembly"); string appFolderPath = SmokeTestRunner.GetApplicationOutputFolderPath(appName); MessageSink.OnMessage(new DiagnosticMessage("Discovering tests case in {0} for application {1}", appFolderPath, appName)); var results = new List <IXunitTestCase>(); if (!System.IO.Directory.Exists(appFolderPath)) { results.Add( new ExecutionErrorTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.None, testMethod, $"Application folder path '{appFolderPath}' does not exist: try compiling application '{appName}' first.")); return(results); } foreach (string folder in System.IO.Directory.GetDirectories(appFolderPath)) { results.Add( new XunitTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, testMethod, new object[] { appName, System.IO.Path.GetFileName(folder), appAssembly })); } if (results.Count == 0) { results.Add( new ExecutionErrorTestCase( MessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.None, testMethod, $"Application '{appName}' does not have any test cases: try compiling the application '{appName}' first.")); } return(results); }
public NamedSkippedDataRowTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, string skipReason, object[] testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, skipReason, testMethodArguments) { }
/// <summary> /// Discover test cases from a test method. By default, if the method is generic, or /// it contains arguments, returns a single <see cref="ExecutionErrorTestCase"/>; /// otherwise, it returns the result of calling <see cref="CreateTestCase"/>. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="factAttribute">The fact attribute attached to the test method.</param> /// <returns>Returns zero or more test cases represented by the test method.</returns> public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { IXunitTestCase testCase; if (testMethod.Method.GetParameters().Any()) { testCase = new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to have parameters. Did you mean to use [Theory]?"); } else if (testMethod.Method.IsGenericMethodDefinition) { testCase = new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to be generic."); } else { testCase = CreateTestCase(discoveryOptions, testMethod, factAttribute); } return(new[] { testCase }); }
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { if (!typeof(ScenarioBase).GetTypeInfo().IsAssignableFrom(testMethod.TestClass.Class.ToRuntimeType())) { throw new NotSupportedException("The Scenario attribute can only be placed on a class inheriting from Kekiri.Xunit.Scenarios"); } return(new ScenarioTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)); }
/// <summary> /// Initializes a new instance of the <see cref="XunitTestCase"/> class. /// </summary> /// <param name="testMethod">The test method this test case belongs to.</param> /// <param name="testMethodArguments">The arguments for the test method.</param> public XunitTestCase(ITestMethod testMethod, object[] testMethodArguments = null) { Initialize(testMethod, testMethodArguments); }
/// <summary> /// Initializes a new instance of the <see cref="XunitTheoryTestCase"/> class. /// </summary> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param> /// <param name="testMethod">The method under test.</param> public XunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { }
public IEnumerable <IXunitTestCase> Discover( ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { var skipReason = factAttribute.GetNamedArgument <string>("Skip"); if (skipReason != null) { return(CreateTestCasesForSkip(discoveryOptions, testMethod, factAttribute, skipReason)); } if (discoveryOptions.PreEnumerateTheoriesOrDefault()) { try { var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)).ToList(); var results = new List <IXunitTestCase>(); for (int index = 0; index < dataAttributes.Count; index++) { var discovererAttribute = dataAttributes[index] .GetCustomAttributes(typeof(DataDiscovererAttribute)) .First(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(MessageSink, discovererAttribute); skipReason = dataAttributes[index].GetNamedArgument <string>("Skip"); if (!discoverer.SupportsDiscoveryEnumeration(dataAttributes[index], testMethod.Method)) { return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, factAttribute) } } ; IEnumerable <object[]> data = discoverer.GetData(dataAttributes[index], testMethod.Method).ToList(); if (data is IDictionary <string, object[]> ) { foreach (KeyValuePair <string, object[]> dataRow in (IDictionary <string, object[]>)data) { var testCase = skipReason != null ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, dataRow.Value, skipReason) : CreateTestCaseForNamedDataRow(discoveryOptions, testMethod, factAttribute, index, dataRow.Key); results.Add(testCase); } } else if (data.GroupBy(x => (x[0]?.ToString()) ?? string.Empty).All(x => x.Count() == 1)) { foreach (object[] dataRow in data) { var testCase = skipReason != null ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, dataRow, skipReason) : CreateTestCaseForNamedDataRow(discoveryOptions, testMethod, factAttribute, index, dataRow[0].ToString() ?? string.Empty); results.Add(testCase); } } else { results.AddRange( data.Select((x, i) => skipReason != null ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, x, skipReason) : CreateTestCaseForDataRow(discoveryOptions, testMethod, factAttribute, index, i))); } } if (results.Count == 0) { results.Add( new ExecutionErrorTestCase( MessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, $"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}")); } return(results); } catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase { MessageSink.OnMessage( new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}")); } } return(CreateTestCasesForTheory(discoveryOptions, testMethod, factAttribute)); }
public SkippableFactTestCase(IMessageSink diagnosticMessageSink, Sdk.TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object[] testMethodArguments = null) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod, testMethodArguments) { }
public SkippableTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod) : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod) { }
/// <summary> /// Creates a test case for the entire theory. This is used when one or more of the theory data items /// are not serializable, or if the user has requested to skip theory pre-enumeration. By default, /// returns an instance of <see cref="NBenchTheoryTestCase"/>, which performs the data discovery at runtime. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>The test case</returns> protected virtual IXunitTestCase CreateTestCaseForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) => new XunitTheoryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
public override TestResult[] Execute(ITestMethod testMethod) { TestResult[] errorResults = ValidateElevated(testMethod); if (errorResults != null) { return(errorResults); } Exception signatureException = GetMethodSignatureException(testMethod); if (signatureException != null) { return(testMethod.CreateExceptionResult(signatureException)); } var runParameters = TestRunParameters.Read(); string logFolder = runParameters.LogFolder; bool shouldLog = !string.IsNullOrEmpty(logFolder); if (shouldLog) { try { logFolder = CreateLogFolder(logFolder, testMethod.TestMethodName); } catch (Exception e) { return(testMethod.CreateExceptionResult(e)); } } int iterations = runParameters.Iterations; var results = new TestResult[iterations]; for (int iteration = 1; iteration <= iterations; iteration++) { string sessionName = $"{testMethod.TestMethodName}-{iteration}"; using (var session = new TraceEventSession(sessionName)) { EnableKernelProviders(session, shouldLog); TraceEventDispatcher source; ZippedETLWriter writer = null; if (shouldLog) { string etlPath = Path.Combine(logFolder, $"Iteration{iteration}.etl"); source = new ETWReloggerTraceEventSource(sessionName, TraceEventSourceType.Session, etlPath); writer = new ZippedETLWriter(etlPath); } else { source = session.Source; } EnableProviders(session); PerformanceTestContext context = CreateContext(source); Task <TestResult> testTask = Task.Run(() => testMethod.Invoke(new object[] { context })); // This is a blocking call that in the case of ETWReloggerTraceEventSource, must be run on the same // thread as ETWReloggerTraceEventSource was created on. It will become unblocked when the // PerformanceTestContext calls StopProcessing on the source. source.Process(); TestResult result = testTask.Result; string displayName = testMethod.TestMethodName; if (iterations > 1) { displayName += $" [{iteration}/{iterations}]"; } result.DisplayName = displayName; session.Flush(); OnIterationEnded(context); context.LogScenarios(); context.LogMemoryDelta(); context.LogMessage($"{displayName} completed. {session.EventsLost} events lost."); context.WriteLogsToResult(result, writer); results[iteration - 1] = result; } } return(results); }
/// <summary> /// Constructs a new method container. /// </summary> /// <param name="instance">An instance of the method's type.</param> /// <param name="method">The method reflection object.</param> /// <param name="testMethod">The test method.</param> public MethodContainer(object instance, MethodInfo method, ITestMethod testMethod) : base() { _methodTask = new MethodInvokeWorkItem(instance, method, testMethod); _methodInfo = method; }
public MefFactTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo factAttributeInfo) : base(diagnosticMessageSink, defaultMethodDisplay, testMethod) { var factAttribute = MefFactAttribute.Instantiate(factAttributeInfo); this.SkipReason = factAttribute.Skip; this.parts = factAttribute.Parts; this.assemblies = factAttribute.Assemblies; this.compositionVersions = factAttribute.CompositionVersions; this.noCompatGoal = factAttribute.NoCompatGoal; this.invalidConfiguration = factAttribute.InvalidConfiguration; if (this.Traits.ContainsKey(Tests.Traits.SkipOnMono) && TestUtilities.IsOnMono) { this.SkipReason = this.SkipReason ?? "Test marked as skipped on Mono runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnMono]); } if (this.Traits.ContainsKey(Tests.Traits.SkipOnCoreCLR) && TestUtilities.IsOnCoreCLR) { this.SkipReason = this.SkipReason ?? "Test marked as skipped on CoreCLR runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnCoreCLR]); } }
/// <summary> /// Creates a test case for a single row of data. By default, returns an instance of <see cref="XunitTestCase"/> /// with the data row inside of it. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="dataRow">The row of data for this test case.</param> /// <returns>The test case</returns> protected virtual NBenchTestCase CreateTestCaseForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) => new NBenchTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow);
// ReSharper disable once TooManyArguments protected override Task <RunSummary> RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method, IEnumerable <IXunitTestCase> testCases, object[] constructorArguments) => new TestMethodRunner(testMethod, Class, method, testCases, DiagnosticMessageSink, MessageBus, new ExceptionAggregator(Aggregator), CancellationTokenSource, constructorArguments) .RunAsync();
void IProtectedTestCase.SetTestMethod(ITestMethod testMethod) { base.TestMethod = testMethod; }