public override IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
string[] conditionMemberNames = factAttribute.GetConstructorArguments().FirstOrDefault() as string[];
IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute);
return ConditionalTestDiscoverer.Discover(discoveryOptions, _diagnosticMessageSink, testMethod, testCases, conditionMemberNames);
}
/// <summary>
/// Finds the tests on a test method.
/// </summary>
/// <param name="testMethod">The test method.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param>
/// <param name="messageBus">The message bus to report discovery messages to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
/// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns>
protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var factAttributes = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).CastOrToList();
if (factAttributes.Count > 1)
{
var message = string.Format("Test method '{0}.{1}' has multiple [Fact]-derived attributes", testMethod.TestClass.Class.Name, testMethod.Method.Name);
var testCase = new ExecutionErrorTestCase(DiagnosticMessageSink, TestMethodDisplay.ClassAndMethod, testMethod, message);
return ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus);
}
var factAttribute = factAttributes.FirstOrDefault();
if (factAttribute == null)
return true;
var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault();
if (testCaseDiscovererAttribute == null)
return true;
var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
if (discovererType == null)
return true;
var discoverer = GetDiscoverer(discovererType);
if (discoverer == null)
return true;
foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute))
if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus))
return false;
return true;
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute)
{
var skipReason = EvaluateSkipConditions(testMethod);
var isTheory = false;
IXunitTestCaseDiscoverer innerDiscoverer;
if (testMethod.Method.GetCustomAttributes(typeof(TheoryAttribute)).Any())
{
isTheory = true;
innerDiscoverer = new TheoryDiscoverer(_diagnosticMessageSink);
}
else
{
innerDiscoverer = new FactDiscoverer(_diagnosticMessageSink);
}
var testCases = innerDiscoverer
.Discover(discoveryOptions, testMethod, factAttribute)
.Select(testCase => new SkipReasonTestCase(isTheory, skipReason, testCase));
return testCases;
}
public IEnumerable<IXunitTestCase> Discover (ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault ();
if (testMethod.Method.GetParameters ().Any ()) {
return new IXunitTestCase[] {
new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, "[VsixFact] methods are not allowed to have parameters.")
};
} else {
var vsVersions = VsVersions.GetFinalVersions(testMethod.GetComputedProperty<string[]>(factAttribute, SpecialNames.VsixAttribute.VisualStudioVersions));
// Process VS-specific traits.
var suffix = testMethod.GetComputedArgument<string>(factAttribute, SpecialNames.VsixAttribute.RootSuffix) ?? "Exp";
var newInstance = testMethod.GetComputedArgument<bool?>(factAttribute, SpecialNames.VsixAttribute.NewIdeInstance);
var timeout = testMethod.GetComputedArgument<int?>(factAttribute, SpecialNames.VsixAttribute.TimeoutSeconds).GetValueOrDefault(XunitExtensions.DefaultTimeout);
var testCases = new List<IXunitTestCase>();
// Add invalid VS versions.
testCases.AddRange (vsVersions
.Where (v => !VsVersions.InstalledVersions.Contains (v))
.Select (v => new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod,
string.Format ("Cannot execute test for specified {0}={1} because there is no VSSDK installed for that version.", SpecialNames.VsixAttribute.VisualStudioVersions, v))));
testCases.AddRange (vsVersions
.Where (v => VsVersions.InstalledVersions.Contains (v))
.Select (v => new VsixTestCase (messageSink, defaultMethodDisplay, testMethod, v, suffix, newInstance, timeout)));
return testCases;
}
}
public IEnumerable<IXunitTestCase> Discover
( ITestFrameworkDiscoveryOptions discoveryOptions
, ITestMethod testMethod
, IAttributeInfo factAttribute
)
{
var inv = InvariantTestCase.InvariantFromMethod(testMethod);
return
inv == null
? new[]
{ new InvariantTestCase
( MessageSink
, TestMethodDisplay.Method
, testMethod
, null
)
}
: inv.AsSeq().Select
( i =>
new InvariantTestCase
( MessageSink
, TestMethodDisplay.Method
, testMethod
, i
)
);
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
return factAttribute.GetNamedArgument<string>("Skip") != null
? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }
: new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) };
}
/// <summary>
/// Finds the tests on a test method.
/// </summary>
/// <param name="testMethod">The test method.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param>
/// <param name="messageBus">The message bus to report discovery messages to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
/// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns>
protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var factAttribute = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault();
if (factAttribute == null)
return true;
var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault();
if (testCaseDiscovererAttribute == null)
return true;
var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
if (discovererType == null)
return true;
var discoverer = GetDiscoverer(discovererType);
if (discoverer == null)
return true;
foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute))
if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus))
return false;
return true;
}
public override IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod();
string conditionMemberName = factAttribute.GetConstructorArguments().FirstOrDefault() as string;
MethodInfo conditionMethodInfo;
if (conditionMemberName == null ||
(conditionMethodInfo = LookupConditionalMethod(testMethodInfo.DeclaringType, conditionMemberName)) == null)
{
return new[] {
new ExecutionErrorTestCase(
_diagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
GetFailedLookupString(conditionMemberName))
};
}
IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute);
if ((bool)conditionMethodInfo.Invoke(null, null))
{
return testCases;
}
else
{
string skippedReason = "\"" + conditionMemberName + "\" returned false.";
return testCases.Select(tc => new SkippedTestCase(tc, skippedReason));
}
}
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var variations = testMethod.Method
.GetCustomAttributes(typeof(BenchmarkVariationAttribute))
.ToDictionary(
a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)),
a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data)));
if (!variations.Any())
{
variations.Add("Default", new object[0]);
}
var tests = new List<IXunitTestCase>();
foreach (var variation in variations)
{
tests.Add(new BenchmarkTestCase(
factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)),
factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)),
variation.Key,
_diagnosticMessageSink,
testMethod,
variation.Value));
}
return tests;
}
/// <summary>
/// Starts the process of running all the tests in the assembly.
/// </summary>
/// <param name="executor">The executor.</param>
/// <param name="executionMessageSink">The message sink to report results back to.</param>
/// <param name="discoveryOptions">The options to be used during test discovery.</param>
/// <param name="executionOptions">The options to be used during test execution.</param>
public static void RunAll(this ITestFrameworkExecutor executor,
IMessageSinkWithTypes executionMessageSink,
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestFrameworkExecutionOptions executionOptions)
{
executor.RunAll(MessageSinkAdapter.Wrap(executionMessageSink), discoveryOptions, executionOptions);
}
/// <summary>
/// Starts the process of finding all tests in an assembly.
/// </summary>
/// <param name="discoverer">The discoverer.</param>
/// <param name="includeSourceInformation">Whether to include source file information, if possible.</param>
/// <param name="discoveryMessageSink">The message sink to report results back to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
public static void Find(this ITestFrameworkDiscoverer discoverer,
bool includeSourceInformation,
IMessageSinkWithTypes discoveryMessageSink,
ITestFrameworkDiscoveryOptions discoveryOptions)
{
discoverer.Find(includeSourceInformation, MessageSinkAdapter.Wrap(discoveryMessageSink), discoveryOptions);
}
/// <summary>
/// Initializes a new instance of the <see cref="TestAssemblyDiscoveryStarting"/> class.
/// </summary>
/// <param name="assembly">Information about the assembly that is being discovered</param>
/// <param name="appDomain">Indicates whether the tests will be discovered and run in a separate app domain</param>
/// <param name="discoveryOptions">The discovery options</param>
public TestAssemblyDiscoveryStarting(XunitProjectAssembly assembly,
bool appDomain,
ITestFrameworkDiscoveryOptions discoveryOptions)
{
Assembly = assembly;
AppDomain = appDomain;
DiscoveryOptions = discoveryOptions;
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
Guard.AgainstNullArgument("discoveryOptions", discoveryOptions);
yield return new ScenarioOutline(
this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute
)
{
yield return new FarmDependentTestCase(_diagnosticSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
public FactDiscovererTests()
{
aggregator = new ExceptionAggregator();
cancellationTokenSource = new CancellationTokenSource();
factAttribute = Mocks.FactAttribute();
messageBus = new SpyMessageBus();
options = TestFrameworkOptions.ForDiscovery();
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries");
if (maxRetries < 1)
maxRetries = 3;
yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries);
}
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true);
//in mixed mode we do not want to run any api tests for plugins when running against a snapshot
//because the client is "hot"
var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection);
return TestClient.Configuration.RunIntegrationTests && RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType);
}
/// <summary>
/// Initializes a new instance of the <see cref="TestAssemblyDiscoveryStarting"/> class.
/// </summary>
/// <param name="assembly">Information about the assembly that is being discovered</param>
/// <param name="discoveryOptions">The discovery options</param>
/// <param name="executionOptions">The execution options</param>
public TestAssemblyDiscoveryStarting(XunitProjectAssembly assembly,
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestFrameworkExecutionOptions executionOptions)
{
Assembly = assembly;
DiscoveryOptions = discoveryOptions;
ExecutionOptions = executionOptions;
}
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) {
if (testMethod.Method.ReturnType.Name == "System.Void" &&
testMethod.Method.GetCustomAttributes(typeof(AsyncStateMachineAttribute)).Any()) {
return new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "Async void methods are not supported.");
}
return new LegacyUITestCase(UITestCase.SyncContextType.WPF, diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
/// <summary>
/// Discover test cases from a test method. By default, inspects the test method's argument list
/// to ensure it's empty, and if not, returns a single <see cref="ExecutionErrorTestCase"/>;
/// otherwise, it returns the result of calling <see cref="CreateTestCase"/>.
/// </summary>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="factAttribute">The fact attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var testCase =
testMethod.Method.GetParameters().Any()
? new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to have parameters. Did you mean to use [Theory]?")
: CreateTestCase(discoveryOptions, testMethod, factAttribute);
return new[] { testCase };
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
if (Helper.Organization == null)
{
return Enumerable.Empty<IXunitTestCase>();
}
return new[] { new XunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
if (Helper.Credentials == null)
return Enumerable.Empty<IXunitTestCase>();
if (!Helper.IsPaidAccount)
return Enumerable.Empty<IXunitTestCase>();
return new[] { new XunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
}
/// <summary>
/// Initializes a new instance of the <see cref="TestAssemblyDiscoveryFinished"/> class.
/// </summary>
/// <param name="assembly">Information about the assembly that is being discovered</param>
/// <param name="discoveryOptions">The discovery options</param>
/// <param name="testCasesDiscovered">The number of test cases discovered</param>
/// <param name="testCasesToRun">The number of test cases to be run</param>
public TestAssemblyDiscoveryFinished(XunitProjectAssembly assembly,
ITestFrameworkDiscoveryOptions discoveryOptions,
int testCasesDiscovered,
int testCasesToRun)
{
Assembly = assembly;
DiscoveryOptions = discoveryOptions;
TestCasesDiscovered = testCasesDiscovered;
TestCasesToRun = testCasesToRun;
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var ctorArgs = factAttribute.GetConstructorArguments().ToArray();
var cultures = Reflector.ConvertArguments(ctorArgs, new[] { typeof(string[]) }).Cast<string[]>().Single();
if (cultures == null || cultures.Length == 0)
cultures = new[] { "en-US", "fr-FR" };
return cultures.Select(culture => new CulturedXunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, culture)).ToList();
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
if (String.IsNullOrWhiteSpace(Helper.ClientId)
&& String.IsNullOrWhiteSpace(Helper.ClientSecret))
{
return Enumerable.Empty<IXunitTestCase>();
}
return new[] { new XunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
}
public override IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
if (_databaseExists.Value)
{
return base.Discover(discoveryOptions, testMethod, factAttribute);
}
else
{
return new IXunitTestCase[] { new SkippedTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
}
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
// Unlike fact discovery, the underlying algorithm for theories is complex, so we let the theory discoverer
// do its work, and do a little on-the-fly conversion into our own test cases.
return theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute)
.Select(testCase => testCase is XunitTheoryTestCase
? (IXunitTestCase)new SkippableTheoryTestCase(diagnosticMessageSink, defaultMethodDisplay, testCase.TestMethod)
: new SkippableFactTestCase(diagnosticMessageSink, defaultMethodDisplay, testCase.TestMethod, testCase.TestMethodArguments));
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
TestParameters parameters = new TestParameters(factAttribute);
List<IXunitTestCase> cases = _theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute).AsList();
if (parameters.ThreadType == ThreadType.UI)
{
return cases.Select(c => new XunitMainThreadTestCaseDecorator(c)).ToList();
}
return cases.Select(c => new XunitTestCaseDecorator(c)).ToList();
}
/// <inheritdoc/>
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var methodDisplay = discoveryOptions.MethodDisplayOrDefault();
IXunitTestCase testCase;
if (testMethod.Method.GetParameters().Any())
testCase = new ExecutionErrorTestCase(diagnosticMessageSink, methodDisplay, testMethod, "[Fact] methods are not allowed to have parameters. Did you mean to use [Theory]?");
else
testCase = new XunitTestCase(diagnosticMessageSink, methodDisplay, testMethod);
return new[] { testCase };
}
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true);
var method = classOfMethod.GetMethod(testMethod.Method.Name, BindingFlags.FlattenHierarchy | BindingFlags.NonPublic | BindingFlags.Public)
?? testMethod.Method.ToRuntimeMethod();
var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection);
return TypeSkipVersionAttributeSatisfies(classOfMethod) ||
MethodSkipVersionAttributeSatisfies(method) ||
RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType);
}
/// <summary>
/// Gets a flag that determines whether theories are pre-enumerated. If enabled, then the
/// discovery system will return a test case for each row of test data; if disabled, then the
/// discovery system will return a single test case for the theory. If the flag is not present,
/// returns the default value (<c>true</c>).
/// </summary>
public static bool PreEnumerateTheoriesOrDefault(this ITestFrameworkDiscoveryOptions discoveryOptions)
{
return(discoveryOptions.PreEnumerateTheories() ?? true);
}
/// <summary>
/// Gets a flag that determines whether xUnit.net should report test results synchronously.
/// If the flag is not set, returns the default value (<c>false</c>).
/// </summary>
public static bool SynchronousMessageReportingOrDefault(this ITestFrameworkDiscoveryOptions discoveryOptions)
{
return(discoveryOptions.SynchronousMessageReporting() ?? false);
}
/// <inheritdoc/>
protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
foreach (var method in testClass.Class.GetMethods(true))
{
var testMethod = new TestMethod(testClass, method);
if (!FindTestsForMethod(testMethod, includeSourceInformation, messageBus, discoveryOptions))
{
return(false);
}
}
return(true);
}
protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
EnsureArg.IsNotNull(testClass, nameof(testClass));
EnsureArg.IsNotNull(messageBus, nameof(messageBus));
EnsureArg.IsNotNull(discoveryOptions, nameof(discoveryOptions));
var attributeInfo = testClass.Class.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault();
if (attributeInfo == null)
{
return(base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions));
}
// get the class-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB)
SingleFlag[][] classLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(attributeInfo);
// convert these to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB)
SingleFlag[][] classLevelClosedParameterSets = CartesianProduct(classLevelOpenParameterSets).Select(e => e.ToArray()).ToArray();
foreach (var method in testClass.Class.GetMethods(true))
{
IAttributeInfo fixtureParameterAttributeInfo = method.GetCustomAttributes(typeof(FixtureArgumentSetsAttribute)).SingleOrDefault();
SingleFlag[][] closedSets = classLevelClosedParameterSets;
if (fixtureParameterAttributeInfo != null)
{
// get the method-level parameter sets in the form (Arg1.OptionA, Arg1.OptionB), (Arg2.OptionA, Arg2.OptionB)
SingleFlag[][] methodLevelOpenParameterSets = ExpandEnumFlagsFromAttributeData(fixtureParameterAttributeInfo);
bool hasOverride = false;
for (int i = 0; i < methodLevelOpenParameterSets.Length; i++)
{
if (methodLevelOpenParameterSets[i]?.Length > 0)
{
hasOverride = true;
}
else
{
// means take the class-level set
methodLevelOpenParameterSets[i] = classLevelOpenParameterSets[i];
}
}
if (hasOverride)
{
// convert to the form (Arg1.OptionA, Arg2.OptionA), (Arg1.OptionA, Arg2.OptionB), (Arg1.OptionB, Arg2.OptionA), (Arg1.OptionB, Arg2.OptionB)
closedSets = CartesianProduct(methodLevelOpenParameterSets).Select(e => e.ToArray()).ToArray();
}
}
foreach (SingleFlag[] closedVariant in closedSets)
{
var closedVariantTestClass = new TestClassWithFixtureArguments(testClass.TestCollection, testClass.Class, closedVariant);
var closedVariantTestMethod = new TestMethod(closedVariantTestClass, method);
if (!FindTestsForMethod(closedVariantTestMethod, includeSourceInformation, messageBus, discoveryOptions))
{
return(false);
}
}
}
return(true);
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForSkip(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
IAttributeInfo theoryAttribute, string skipReason)
{
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) => new WpfTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, _wpfTestSerializationGate);
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
return(_inner.Discover(discoveryOptions, testMethod, factAttribute).Select(t => new ForegroundFactTestCase(t)));
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
yield return(new UITheoryTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) =>
!_condition
? Enumerable.Empty <IXunitTestCase>()
: new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) };
protected override IEnumerable <IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
{
yield return(new UITestCase(UITestCase.SyncContextType.Portable, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
yield return(new SkippableFactTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod));
}
bool FindTestsForTypeAndWrapExceptions(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
try
{
return(FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions));
}
catch (Exception ex)
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception during discovery:{Environment.NewLine}{ex}"));
return(true); // Keep going on to the next type
}
}
/// <inheritdoc/>
public void Find(bool includeSourceInformation, IMessageSink discoveryMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions)
{
Guard.ArgumentNotNull("discoveryMessageSink", discoveryMessageSink);
Guard.ArgumentNotNull("discoveryOptions", discoveryOptions);
XunitWorkerThread.QueueUserWorkItem(() =>
{
using (var messageBus = CreateMessageBus(discoveryMessageSink, discoveryOptions))
using (new PreserveWorkingFolder(AssemblyInfo))
{
foreach (var type in AssemblyInfo.GetTypes(false).Where(IsValidTestClass))
{
var testClass = CreateTestClass(type);
if (!FindTestsForTypeAndWrapExceptions(testClass, includeSourceInformation, messageBus, discoveryOptions))
{
break;
}
}
messageBus.QueueMessage(new DiscoveryCompleteMessage());
}
});
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
return(Helper.IsUsingToken
? new[] { new XunitTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) }
: Enumerable.Empty <IXunitTestCase>());
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
var testCase = new WpfTheoryTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
return(SpecializedCollections.SingletonEnumerable(testCase));
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForDataRow(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
=> CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, "Skipped");
protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var type = testClass.Class;
var summaryMethods = type.GetMethods(includePrivateMethods: false).Where(m => m.GetCustomAttributes(typeof(SummaryAttribute)).Any()).ToList();
if (summaryMethods.Count == 0)
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Test case class {type.Name} does not have a Summary method. This will not be discovered."));
return(true);
}
if (summaryMethods.Count > 1)
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Test case class {type.Name} has more than one Summary methods. This will not be discovered."));
return(true);
}
return(base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
return(whiteSpacePreservations
.Select(x => new ReadPerformanceTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, x)));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var maxRetries = Math.Max(1, factAttribute.GetNamedArgument <int>("MaxRetries"));
yield return(new RetryTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, maxRetries));
}
public OptionsWithPreEnumerationEnabled(ITestFrameworkDiscoveryOptions original) => _original = original;
protected override IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
IAttributeInfo theoryAttribute, object[] dataRow, string skipReason)
{
return(new[] { new SkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, skipReason, dataRow) });
}
/// <summary>
/// Creates test cases for a single row of skipped data. By default, returns a single instance of <see cref="XunitSkippedDataRowTestCase"/>
/// with the data row inside of it.
/// </summary>
/// <remarks>If this method is overridden, the implementation will have to override <see cref="TestMethodTestCase.SkipReason"/> otherwise
/// the default behavior will look at the <see cref="TheoryAttribute"/> and the test case will not be skipped.</remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <param name="dataRow">The row of data for this test case.</param>
/// <param name="skipReason">The reason this test case is to be skipped</param>
/// <returns>The test cases</returns>
#pragma warning disable CS0618
protected virtual IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason)
=> new[] { CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) };
/// <summary>
/// Finds the tests on a test method.
/// </summary>
/// <param name="testMethod">The test method.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param>
/// <param name="messageBus">The message bus to report discovery messages to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
/// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns>
protected internal virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var factAttributes = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).CastOrToList();
if (factAttributes.Count > 1)
{
var message = $"Test method '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}' has multiple [Fact]-derived attributes";
var testCase = new ExecutionErrorTestCase(DiagnosticMessageSink, TestMethodDisplay.ClassAndMethod, testMethod, message);
return(ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus));
}
var factAttribute = factAttributes.FirstOrDefault();
if (factAttribute == null)
{
return(true);
}
var factAttributeType = (factAttribute as IReflectionAttributeInfo)?.Attribute.GetType();
Type discovererType = null;
if (factAttributeType == null || !DiscovererTypeCache.TryGetValue(factAttributeType, out discovererType))
{
var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault();
if (testCaseDiscovererAttribute != null)
{
var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast <string>().ToList();
discovererType = SerializationHelper.GetType(args[1], args[0]);
}
if (factAttributeType != null)
{
DiscovererTypeCache[factAttributeType] = discovererType;
}
}
if (discovererType == null)
{
return(true);
}
var discoverer = GetDiscoverer(discovererType);
if (discoverer == null)
{
return(true);
}
foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute))
{
if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus))
{
return(false);
}
}
return(true);
}
#pragma warning restore CS0618
/// <summary>
/// Discover test cases from a test method.
/// </summary>
/// <remarks>
/// This method performs the following steps:
/// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>;
/// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>;
/// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it;
/// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason.
/// </remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
// Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
var skipReason = theoryAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return(CreateTestCasesForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason));
}
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var results = new List <IXunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
IDataDiscoverer discoverer;
try
{
discoverer = ExtensibilityPointFactory.GetDataDiscoverer(DiagnosticMessageSink, discovererAttribute);
}
catch (InvalidCastException)
{
if (dataAttribute is IReflectionAttributeInfo reflectionAttribute)
{
var msg = $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on "
+ $"{testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not implement IDataDiscoverer.";
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
msg));
}
else
{
var msg = $"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} "
+ "does not implement IDataDiscoverer.";
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
msg));
}
continue;
}
if (discoverer == null)
{
if (dataAttribute is IReflectionAttributeInfo reflectionAttribute)
{
var msg = $"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on "
+ $"{testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist.";
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
msg));
}
else
{
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"A data discoverer specified on {testMethod.TestClass.Class.Name}.{testMethod.Method.Name} does not exist."));
}
continue;
}
skipReason = dataAttribute.GetNamedArgument <string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
var data = discoverer.GetData(dataAttribute, testMethod.Method);
if (data == null)
{
var msg = $"Test data returned null for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}."
+ $" Make sure it is statically initialized before this test method is called.";
results.Add(
new ExecutionErrorTestCase(
DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
msg));
continue;
}
foreach (var dataRow in data)
{
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
if (!SerializationHelper.IsSerializable(dataRow))
{
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case."));
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
var testCases =
skipReason != null
? CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason)
: CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
results.AddRange(testCases);
}
}
if (results.Count == 0)
{
results.Add(new ExecutionErrorTestCase(DiagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
discoveryOptions.MethodDisplayOptionsOrDefault(),
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"));
}
return(results);
}
catch (Exception ex)
{
// If something goes wrong, fall through to return just the XunitTestCase
DiagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}"));
}
}
return(CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute));
}
/// <summary>
/// Starts the process of running all the xUnit.net v2 tests in the assembly.
/// </summary>
/// <param name="messageSink">The message sink to report results back to.</param>
/// <param name="discoveryOptions">The options to be used during test discovery.</param>
/// <param name="executionOptions">The options to be used during test execution.</param>
public void RunAll(IMessageSink messageSink, ITestFrameworkDiscoveryOptions discoveryOptions, ITestFrameworkExecutionOptions executionOptions)
{
executor.RunAll(messageSink, discoveryOptions, executionOptions);
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) => VsTestCaseFactory.CreateTheoryTestCases(testMethod, discoveryOptions.MethodDisplayOrDefault(), DiagnosticMessageSink);
/// <summary>
/// Gets a flag that determines whether xUnit.net should report test results synchronously.
/// </summary>
public static bool?SynchronousMessageReporting(this ITestFrameworkDiscoveryOptions discoveryOptions)
{
return(discoveryOptions.GetValue <bool?>(TestOptionsNames.Execution.SynchronousMessageReporting));
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason) => VsTestCaseFactory.CreateSkippedDataRowTestCases(testMethod, discoveryOptions.MethodDisplayOrDefault(), DiagnosticMessageSink, dataRow, skipReason);
/// <summary> /// Core implementation to discover unit tests in a given test class. /// </summary> /// <param name="testClass">The test class.</param> /// <param name="includeSourceInformation">Set to <c>true</c> to attempt to include source information.</param> /// <param name="messageBus">The message sink to send discovery messages to.</param> /// <param name="discoveryOptions">The options used by the test framework during discovery.</param> /// <returns>Returns <c>true</c> if discovery should continue; <c>false</c> otherwise.</returns> protected abstract bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions);
/// <inheritdoc/>
public void Find(string typeName, bool includeSourceInformation, IMessageSink discoveryMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions)
{
Guard.ArgumentNotNullOrEmpty("typeName", typeName);
Guard.ArgumentNotNull("discoveryMessageSink", discoveryMessageSink);
Guard.ArgumentNotNull("discoveryOptions", discoveryOptions);
XunitWorkerThread.QueueUserWorkItem(() =>
{
using (var messageBus = CreateMessageBus(discoveryMessageSink, discoveryOptions))
using (new PreserveWorkingFolder(AssemblyInfo))
{
var typeInfo = AssemblyInfo.GetType(typeName);
if (typeInfo != null && IsValidTestClass(typeInfo))
{
var testClass = CreateTestClass(typeInfo);
FindTestsForTypeAndWrapExceptions(testClass, includeSourceInformation, messageBus, discoveryOptions);
}
messageBus.QueueMessage(new DiscoveryCompleteMessage());
}
});
}
