public override IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod();
string conditionMemberName = factAttribute.GetConstructorArguments().FirstOrDefault() as string;
MethodInfo conditionMethodInfo;
if (conditionMemberName == null ||
(conditionMethodInfo = LookupConditionalMethod(testMethodInfo.DeclaringType, conditionMemberName)) == null)
{
return new[] {
new ExecutionErrorTestCase(
_diagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
GetFailedLookupString(conditionMemberName))
};
}
IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute);
if ((bool)conditionMethodInfo.Invoke(null, null))
{
return testCases;
}
else
{
string skippedReason = "\"" + conditionMemberName + "\" returned false.";
return testCases.Select(tc => new SkippedTestCase(tc, skippedReason));
}
}
/// <summary>
/// Creates a result record.
/// </summary>
/// <param name="method">Test method metadata object.</param>
/// <param name="testClass">Test class metadata object.</param>
/// <param name="result">Test result object.</param>
/// <param name="exception">Exception instance, if any.</param>
public ScenarioResult(ITestMethod method, ITestClass testClass, TestOutcome result, Exception exception)
{
TestClass = testClass;
TestMethod = method;
Exception = exception;
Result = result;
}
/// <summary>
/// Finds the tests on a test method.
/// </summary>
/// <param name="testMethod">The test method.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param>
/// <param name="messageBus">The message bus to report discovery messages to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
/// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns>
protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var factAttribute = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault();
if (factAttribute == null)
return true;
var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault();
if (testCaseDiscovererAttribute == null)
return true;
var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
if (discovererType == null)
return true;
var discoverer = GetDiscoverer(discovererType);
if (discoverer == null)
return true;
foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute))
if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus))
return false;
return true;
}
public BenchmarkTestCase(
int iterations,
int warmupIterations,
string variation,
IMessageSink diagnosticMessageSink,
ITestMethod testMethod,
object[] testMethodArguments)
: base(diagnosticMessageSink, Xunit.Sdk.TestMethodDisplay.Method, testMethod, null)
{
// Override display name to avoid getting info about TestMethodArguments in the
// name (this is covered by the concept of Variation for benchmarks)
var suppliedDisplayName = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute))
.First()
.GetNamedArgument<string>("DisplayName");
_diagnosticMessageSink = diagnosticMessageSink;
DisplayName = suppliedDisplayName ?? BaseDisplayName;
Variation = variation;
Iterations = iterations;
WarmupIterations = warmupIterations;
var methodArguments = new List<object> { MetricCollector };
if (testMethodArguments != null)
{
methodArguments.AddRange(testMethodArguments);
}
TestMethodArguments = methodArguments.ToArray();
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault();
return factAttribute.GetNamedArgument<string>("Skip") != null
? new[] { new XunitTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) }
: new XunitTestCase[] { new ScenarioTestCase(_diagnosticMessageSink, defaultMethodDisplay, testMethod) };
}
protected override void Before_all_tests()
{
base.Before_all_tests();
methodInfo = MockNUnitTestClass.GetPassingTest();
testMethod = new Xunit.TestMethod(methodInfo);
}
/// <summary>
/// Constructor for a test method manager, which handles executing a single test method
/// for a unit test provider.
/// </summary>
/// <param name="testHarness">The unit test harness object.</param>
/// <param name="testClass">The test class metadata object.</param>
/// <param name="testMethod">The test method metadata object.</param>
/// <param name="instance">The test class instance.</param>
/// <param name="provider">The unit test provider.</param>
public TestMethodManager(UnitTestHarness testHarness, ITestClass testClass, ITestMethod testMethod, object instance, IUnitTestProvider provider)
: base(testHarness, provider)
{
_testClass = testClass;
_testMethod = testMethod;
_instance = instance;
}
public IEnumerable<IXunitTestCase> Discover (ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var defaultMethodDisplay = discoveryOptions.MethodDisplayOrDefault ();
if (testMethod.Method.GetParameters ().Any ()) {
return new IXunitTestCase[] {
new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod, "[VsixFact] methods are not allowed to have parameters.")
};
} else {
var vsVersions = VsVersions.GetFinalVersions(testMethod.GetComputedProperty<string[]>(factAttribute, SpecialNames.VsixAttribute.VisualStudioVersions));
// Process VS-specific traits.
var suffix = testMethod.GetComputedArgument<string>(factAttribute, SpecialNames.VsixAttribute.RootSuffix) ?? "Exp";
var newInstance = testMethod.GetComputedArgument<bool?>(factAttribute, SpecialNames.VsixAttribute.NewIdeInstance);
var timeout = testMethod.GetComputedArgument<int?>(factAttribute, SpecialNames.VsixAttribute.TimeoutSeconds).GetValueOrDefault(XunitExtensions.DefaultTimeout);
var testCases = new List<IXunitTestCase>();
// Add invalid VS versions.
testCases.AddRange (vsVersions
.Where (v => !VsVersions.InstalledVersions.Contains (v))
.Select (v => new ExecutionErrorTestCase (messageSink, defaultMethodDisplay, testMethod,
string.Format ("Cannot execute test for specified {0}={1} because there is no VSSDK installed for that version.", SpecialNames.VsixAttribute.VisualStudioVersions, v))));
testCases.AddRange (vsVersions
.Where (v => VsVersions.InstalledVersions.Contains (v))
.Select (v => new VsixTestCase (messageSink, defaultMethodDisplay, testMethod, v, suffix, newInstance, timeout)));
return testCases;
}
}
/// <summary>
/// Initializes a new test run filter using an existing settings file.
/// </summary>
/// <param name="test">The test class metadata.</param>
/// <param name="method">The test method metadata.</param>
public RetryTestRunFilter(ITestClass test, ITestMethod method) : base(null, null)
{
TestRunName = "Retry of " + test.Name + " " + method.Name;
_test = test;
_method = method;
}
public virtual IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var variations = testMethod.Method
.GetCustomAttributes(typeof(BenchmarkVariationAttribute))
.ToDictionary(
a => a.GetNamedArgument<string>(nameof(BenchmarkVariationAttribute.VariationName)),
a => a.GetNamedArgument<object[]>(nameof(BenchmarkVariationAttribute.Data)));
if (!variations.Any())
{
variations.Add("Default", new object[0]);
}
var tests = new List<IXunitTestCase>();
foreach (var variation in variations)
{
tests.Add(new BenchmarkTestCase(
factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.Iterations)),
factAttribute.GetNamedArgument<int>(nameof(BenchmarkAttribute.WarmupIterations)),
variation.Key,
_diagnosticMessageSink,
testMethod,
variation.Value));
}
return tests;
}
/// <summary>
/// Creates a new method invoke work item for a MethodInfo instance.
/// </summary>
/// <param name="instance">The type instance.</param>
/// <param name="method">The method on the type to invoke when the
/// work item is executed.</param>
/// <param name="testMethod">The test method metadata.</param>
public MethodInvokeWorkItem(object instance, MethodInfo method, ITestMethod testMethod)
: base()
{
_instance = instance;
_method = method;
_testMethod = testMethod;
}
public override IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
string[] conditionMemberNames = factAttribute.GetConstructorArguments().FirstOrDefault() as string[];
IEnumerable<IXunitTestCase> testCases = base.Discover(discoveryOptions, testMethod, factAttribute);
return ConditionalTestDiscoverer.Discover(discoveryOptions, _diagnosticMessageSink, testMethod, testCases, conditionMemberNames);
}
/// <summary>
/// Constructs a new method container.
/// </summary>
/// <param name="instance">An instance of the method's type.</param>
/// <param name="method">The method reflection object.</param>
/// <param name="testMethod">The test method.</param>
public MethodContainer(object instance, MethodInfo method, ITestMethod testMethod)
: base()
{
_methodTask = new MethodInvokeWorkItem(instance, method, testMethod);
_methodInfo = method;
_method = testMethod;
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute)
{
var skipReason = EvaluateSkipConditions(testMethod);
var isTheory = false;
IXunitTestCaseDiscoverer innerDiscoverer;
if (testMethod.Method.GetCustomAttributes(typeof(TheoryAttribute)).Any())
{
isTheory = true;
innerDiscoverer = new TheoryDiscoverer(_diagnosticMessageSink);
}
else
{
innerDiscoverer = new FactDiscoverer(_diagnosticMessageSink);
}
var testCases = innerDiscoverer
.Discover(discoveryOptions, testMethod, factAttribute)
.Select(testCase => new SkipReasonTestCase(isTheory, skipReason, testCase));
return testCases;
}
/// <summary>
/// Finds the tests on a test method.
/// </summary>
/// <param name="testMethod">The test method.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to indicate that source information should be included.</param>
/// <param name="messageBus">The message bus to report discovery messages to.</param>
/// <param name="discoveryOptions">The options used by the test framework during discovery.</param>
/// <returns>Return <c>true</c> to continue test discovery, <c>false</c>, otherwise.</returns>
protected virtual bool FindTestsForMethod(ITestMethod testMethod, bool includeSourceInformation, IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions)
{
var factAttributes = testMethod.Method.GetCustomAttributes(typeof(FactAttribute)).CastOrToList();
if (factAttributes.Count > 1)
{
var message = string.Format("Test method '{0}.{1}' has multiple [Fact]-derived attributes", testMethod.TestClass.Class.Name, testMethod.Method.Name);
var testCase = new ExecutionErrorTestCase(DiagnosticMessageSink, TestMethodDisplay.ClassAndMethod, testMethod, message);
return ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus);
}
var factAttribute = factAttributes.FirstOrDefault();
if (factAttribute == null)
return true;
var testCaseDiscovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitTestCaseDiscovererAttribute)).FirstOrDefault();
if (testCaseDiscovererAttribute == null)
return true;
var args = testCaseDiscovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
if (discovererType == null)
return true;
var discoverer = GetDiscoverer(discovererType);
if (discoverer == null)
return true;
foreach (var testCase in discoverer.Discover(discoveryOptions, testMethod, factAttribute))
if (!ReportDiscoveredTestCase(testCase, includeSourceInformation, messageBus))
return false;
return true;
}
public KuduXunitTheoryTestCase(IMessageSink diagnosticMessageSink,
TestMethodDisplay defaultMethodDisplay,
ITestMethod testMethod,
IAttributeInfo testAttribute)
: base(diagnosticMessageSink, defaultMethodDisplay, testMethod)
{
}
protected override void Before_all_tests()
{
base.Before_all_tests();
methodInfo = MockXunitTestClass.GetIgnoreTest();
testMethod = new TestMethod(methodInfo);
}
public static void AssignTestExecutionMethodInfo(this TestExecutionMethod testExecutionMethod, ITestMethod testMethod)
{
var methodInfo = testMethod.Method;
testExecutionMethod.NamespaceName = methodInfo.ReflectedType.Namespace;
testExecutionMethod.ClassName = methodInfo.ReflectedType.ClassNameIncludingParentsIfNested();
testExecutionMethod.MethodName = testMethod.Name;
}
public IEnumerable<IXunitTestCase> Discover
( ITestFrameworkDiscoveryOptions discoveryOptions
, ITestMethod testMethod
, IAttributeInfo factAttribute
)
{
var inv = InvariantTestCase.InvariantFromMethod(testMethod);
return
inv == null
? new[]
{ new InvariantTestCase
( MessageSink
, TestMethodDisplay.Method
, testMethod
, null
)
}
: inv.AsSeq().Select
( i =>
new InvariantTestCase
( MessageSink
, TestMethodDisplay.Method
, testMethod
, i
)
);
}
protected BenchmarkTestCaseBase(
string variation,
IMessageSink diagnosticMessageSink,
ITestMethod testMethod,
object[] testMethodArguments)
: base(diagnosticMessageSink, TestMethodDisplay.Method, testMethod, null)
{
// Override display name to avoid getting info about TestMethodArguments in the
// name (this is covered by the concept of Variation for benchmarks)
var name = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute))
.First()
.GetNamedArgument<string>("DisplayName") ?? BaseDisplayName;
TestMethodName = name;
DisplayName = $"{name} [Variation: {variation}]";
DiagnosticMessageSink = diagnosticMessageSink;
Variation = variation;
var methodArguments = new List<object> { MetricCollector };
if (testMethodArguments != null)
{
methodArguments.AddRange(testMethodArguments);
}
TestMethodArguments = methodArguments.ToArray();
}
/// <summary>
/// Initializes dispatcher-stack attaching method container work item.
/// </summary>
/// <param name="testHarness">Test harness.</param>
/// <param name="instance">Test instance.</param>
/// <param name="method">Method reflection object.</param>
/// <param name="testMethod">Test method metadata.</param>
/// <param name="granularity">Granularity of test.</param>
public UnitTestMethodContainer(UnitTestHarness testHarness, object instance, MethodInfo method, ITestMethod testMethod, TestGranularity granularity)
: base(instance, method, testMethod)
{
_granularity = granularity;
_harness = testHarness as UnitTestHarness;
_testMethod = testMethod;
}
public bool Contains(ITestMethod testMethod) {
foreach (var criterion in _criteria) {
if (criterion.Contains(testMethod)) {
return true;
}
}
return false;
}
protected override void Before_all_tests()
{
base.Before_all_tests();
_methodInfo = MockNUnitTestClass.GetPassingTest();
_testMethod = new TestMethod(_methodInfo);
_testMethod.ShouldNotBeNull();
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute
)
{
yield return new FarmDependentTestCase(_diagnosticSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
public IEnumerable<IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
Guard.AgainstNullArgument("discoveryOptions", discoveryOptions);
yield return new ScenarioOutline(
this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
public NonCollectingBenchmarkTestCase(
string variation,
IMessageSink diagnosticMessageSink,
ITestMethod testMethod,
object[] testMethodArguments)
: base(variation, diagnosticMessageSink, testMethod, testMethodArguments)
{
}
public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries");
if (maxRetries < 1)
maxRetries = 3;
yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCase"/> class.
/// </summary>
/// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param>
/// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param>
/// <param name="testMethod">The test method this test case belongs to.</param>
/// <param name="testMethodArguments">The arguments for the test method.</param>
public XunitTestCase(IMessageSink diagnosticMessageSink,
TestMethodDisplay defaultMethodDisplay,
ITestMethod testMethod,
object[] testMethodArguments = null)
: base(defaultMethodDisplay, testMethod, testMethodArguments)
{
this.diagnosticMessageSink = diagnosticMessageSink;
}
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) {
if (testMethod.Method.ReturnType.Name == "System.Void" &&
testMethod.Method.GetCustomAttributes(typeof(AsyncStateMachineAttribute)).Any()) {
return new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "Async void methods are not supported.");
}
return new LegacyUITestCase(UITestCase.SyncContextType.WPF, diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
}
protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
var classOfMethod = Type.GetType(testMethod.TestClass.Class.Name, true, true);
//in mixed mode we do not want to run any api tests for plugins when running against a snapshot
//because the client is "hot"
var collectionType = TestAssemblyRunner.GetClusterForCollection(testMethod.TestClass?.TestCollection);
return TestClient.Configuration.RunIntegrationTests && RequiresPluginButRunningAgainstSnapshot(classOfMethod, collectionType);
}
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) => new SkippableTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod);
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
if (Environment.GetEnvironmentVariable("CI")?.ToUpperInvariant() == "TRUE")
{
return(Enumerable.Empty <IXunitTestCase>());
}
return(new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.All, testMethod) });
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
return(_inner.Discover(discoveryOptions, testMethod, factAttribute).Select(t => new ForegroundFactTestCase(t)));
}
/// <summary> /// Creates a test case for a skipped theory. By default, returns an instance of <see cref="XunitTestCase"/> /// (which inherently discovers the skip reason via the fact attribute). /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="skipReason">The skip reason that decorates <paramref name="theoryAttribute"/>.</param> /// <returns>The test case</returns> protected virtual NBenchTestCase CreateTestCaseForSkip(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, string skipReason) => new NBenchTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
/// <summary>
/// Creates a test case for a single row of data. By default, returns an instance of <see cref="XunitSkippedDataRowTestCase"/>
/// with the data row inside of it.
/// </summary>
/// <remarks>If this method is overridden, the implementation will have to override <see cref="TestMethodTestCase.SkipReason"/> otherwise
/// the default behavior will look at the <see cref="TheoryAttribute"/> and the test case will not be skipped.</remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <param name="dataRow">The row of data for this test case.</param>
/// <param name="skipReason">The reason this test case is to be skipped</param>
/// <returns>The test case</returns>
//protected virtual IXunitTestCase CreateTestCaseForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason)
// => new XunitSkippedDataRowTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, skipReason, dataRow);
/// <summary>
/// Discover test cases from a test method.
/// </summary>
/// <remarks>
/// This method performs the following steps:
/// - If the theory attribute is marked with Skip, returns the single test case from <see cref="CreateTestCaseForSkip"/>;
/// - If pre-enumeration is off, or any of the test data is non serializable, returns the single test case from <see cref="CreateTestCaseForTheory"/>;
/// - If there is no theory data, returns a single test case of <see cref="ExecutionErrorTestCase"/> with the error in it;
/// - Otherwise, it returns one test case per data row, created by calling <see cref="CreateTestCaseForDataRow"/> or <see cref="CreateTestCaseForSkippedDataRow"/> if the data attribute has a skip reason.
/// </remarks>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="theoryAttribute">The theory attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
{
// Special case Skip, because we want a single Skip (not one per data item); plus, a skipped test may
// not actually have any data (which is quasi-legal, since it's skipped).
var skipReason = theoryAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return new[] { CreateTestCaseForSkip(discoveryOptions, testMethod, theoryAttribute, skipReason) }
}
;
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var results = new List <IXunitTestCase>();
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererAttribute);
//skipReason = dataAttribute.GetNamedArgument<string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttribute, testMethod.Method))
{
return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) }
}
;
// GetData may return null, but that's okay; we'll let the NullRef happen and then catch it
// down below so that we get the composite test case.
foreach (var dataRow in discoverer.GetData(dataAttribute, testMethod.Method))
{
//TODO: Revisit when xUnit beta is released to Production
// Determine whether we can serialize the test case, since we need a way to uniquely
// identify a test and serialization is the best way to do that. If it's not serializable,
// this will throw and we will fall back to a single theory test case that gets its data at runtime.
//if (!SerializationHelper.IsSerializable(dataRow))
//{
// diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Non-serializable data ('{dataRow.GetType().FullName}') found for '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case."));
// return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) };
//}
//var testCase =
// skipReason != null
// ? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason)
// : CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
var testCase = CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
results.Add(testCase);
}
}
if (results.Count == 0)
{
results.Add(new ExecutionErrorTestCase(diagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"));
}
return(results);
}
catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}"));
}
}
return(new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, theoryAttribute) });
}
}
// This helper method evaluates the given condition member names for a given set of test cases.
// If any condition member evaluates to 'false', the test cases are marked to be skipped.
// The skip reason is the collection of all the condition members that evalated to 'false'.
internal static IEnumerable <IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
IMessageSink diagnosticMessageSink,
ITestMethod testMethod,
IEnumerable <IXunitTestCase> testCases,
object[] conditionArguments)
{
// A null or empty list of conditionMemberNames is treated as "no conditions".
// and the test cases will not be skipped.
// Example: [ConditionalFact()]
if (conditionArguments == null || conditionArguments.Length == 0)
{
return(testCases);
}
string [] conditionMemberNames;
Type calleeType = conditionArguments[0] as Type;
if (calleeType != null)
{
if (conditionArguments.Length < 2)
{
// [ConditionalFact(typeof(x))] no provided methods.
return(testCases);
}
// [ConditionalFact(typeof(x), "MethodName")]
conditionMemberNames = conditionArguments[1] as string[];
}
else
{
// [ConditionalFact("MethodName")]
conditionMemberNames = conditionArguments[0] as string[];
}
// [ConditionalFact((string[]) null)]
int conditionCount = conditionMemberNames == null ? 0 : conditionMemberNames.Count();
if (conditionCount == 0)
{
return(testCases);
}
MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod();
Type testMethodDeclaringType = testMethodInfo.DeclaringType;
List <string> falseConditions = new List <string>(conditionCount);
foreach (string entry in conditionMemberNames)
{
string conditionMemberName = entry;
// Null condition member names are silently tolerated
if (string.IsNullOrWhiteSpace(conditionMemberName))
{
continue;
}
Type declaringType;
if (calleeType != null)
{
declaringType = calleeType;
}
else
{
declaringType = testMethodDeclaringType;
string[] symbols = conditionMemberName.Split('.');
if (symbols.Length == 2)
{
conditionMemberName = symbols[1];
ITypeInfo type = testMethod.TestClass.Class.Assembly.GetTypes(false).Where(t => t.Name.Contains(symbols[0])).FirstOrDefault();
if (type != null)
{
declaringType = type.ToRuntimeType();
}
}
}
MethodInfo conditionMethodInfo;
if ((conditionMethodInfo = LookupConditionalMethod(declaringType, conditionMemberName)) == null)
{
return(new[]
{
new ExecutionErrorTestCase(
diagnosticMessageSink,
discoveryOptions.MethodDisplayOrDefault(),
TestMethodDisplayOptions.None,
testMethod,
GetFailedLookupString(conditionMemberName, declaringType))
});
}
// In the case of multiple conditions, collect the results of all
// of them to produce a summary skip reason.
try
{
if (!(bool)conditionMethodInfo.Invoke(null, null))
{
falseConditions.Add(conditionMemberName);
}
}
catch (Exception exc)
{
falseConditions.Add($"{conditionMemberName} ({exc.GetType().Name})");
}
}
// Compose a summary of all conditions that returned false.
if (falseConditions.Count > 0)
{
string skippedReason = string.Format("Condition(s) not met: \"{0}\"", string.Join("\", \"", falseConditions));
return(testCases.Select(tc => new SkippedTestCase(tc, skippedReason)));
}
// No conditions returned false (including the absence of any conditions).
return(testCases);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTheoryTestCase"/> class.
/// </summary>
/// <param name="testMethod">The method under test.</param>
public XunitTheoryTestCase(ITestMethod testMethod)
: base(testMethod)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="TestMethodTestCase"/> class.
/// </summary>
/// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param>
/// <param name="testMethod">The test method this test case belongs to.</param>
/// <param name="testMethodArguments">The arguments for the test method.</param>
protected TestMethodTestCase(TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object[] testMethodArguments = null)
{
DefaultMethodDisplay = defaultMethodDisplay;
TestMethod = testMethod;
TestMethodArguments = testMethodArguments;
}
protected override IEnumerable <IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
=> new[] { new SkippableTheoryTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod) };
protected override IEnumerable <IXunitTestCase> CreateTestCasesForSkippedDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow, string skipReason)
=> new[] { new NamedSkippedDataRowTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, skipReason, dataRow) };
public SkippableTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, object[] testMethodArguments = null)
: base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments)
{
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
return(Helper.HasCredentials
? new[] { new XunitTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) }
: Enumerable.Empty <IXunitTestCase>());
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
yield return(new SkippableFactTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttributeInfo)
{
var methodDisplay = discoveryOptions.MethodDisplayOrDefault();
yield return(new MefFactTestCase(this.diagnosticMessageSink, methodDisplay, testMethod, factAttributeInfo));
}
public IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
string appName = factAttribute.GetNamedArgument <string>("AppName");
string appAssembly = factAttribute.GetNamedArgument <string>("AppAssembly");
string appFolderPath = SmokeTestRunner.GetApplicationOutputFolderPath(appName);
MessageSink.OnMessage(new DiagnosticMessage("Discovering tests case in {0} for application {1}", appFolderPath, appName));
var results = new List <IXunitTestCase>();
if (!System.IO.Directory.Exists(appFolderPath))
{
results.Add(
new ExecutionErrorTestCase(
MessageSink,
TestMethodDisplay.Method,
TestMethodDisplayOptions.None,
testMethod,
$"Application folder path '{appFolderPath}' does not exist: try compiling application '{appName}' first."));
return(results);
}
foreach (string folder in System.IO.Directory.GetDirectories(appFolderPath))
{
results.Add(
new XunitTestCase(
MessageSink,
TestMethodDisplay.Method,
TestMethodDisplayOptions.All,
testMethod,
new object[] { appName, System.IO.Path.GetFileName(folder), appAssembly }));
}
if (results.Count == 0)
{
results.Add(
new ExecutionErrorTestCase(
MessageSink,
TestMethodDisplay.Method,
TestMethodDisplayOptions.None,
testMethod,
$"Application '{appName}' does not have any test cases: try compiling the application '{appName}' first."));
}
return(results);
}
public NamedSkippedDataRowTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod, string skipReason, object[] testMethodArguments = null)
: base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, skipReason, testMethodArguments)
{
}
/// <summary>
/// Discover test cases from a test method. By default, if the method is generic, or
/// it contains arguments, returns a single <see cref="ExecutionErrorTestCase"/>;
/// otherwise, it returns the result of calling <see cref="CreateTestCase"/>.
/// </summary>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="factAttribute">The fact attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
{
IXunitTestCase testCase;
if (testMethod.Method.GetParameters().Any())
{
testCase = new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to have parameters. Did you mean to use [Theory]?");
}
else if (testMethod.Method.IsGenericMethodDefinition)
{
testCase = new ExecutionErrorTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, "[Fact] methods are not allowed to be generic.");
}
else
{
testCase = CreateTestCase(discoveryOptions, testMethod, factAttribute);
}
return(new[] { testCase });
}
protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod,
IAttributeInfo factAttribute)
{
if (!typeof(ScenarioBase).GetTypeInfo().IsAssignableFrom(testMethod.TestClass.Class.ToRuntimeType()))
{
throw new NotSupportedException("The Scenario attribute can only be placed on a class inheriting from Kekiri.Xunit.Scenarios");
}
return(new ScenarioTestCase(_diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod));
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTestCase"/> class.
/// </summary>
/// <param name="testMethod">The test method this test case belongs to.</param>
/// <param name="testMethodArguments">The arguments for the test method.</param>
public XunitTestCase(ITestMethod testMethod, object[] testMethodArguments = null)
{
Initialize(testMethod, testMethodArguments);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTheoryTestCase"/> class.
/// </summary>
/// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param>
/// <param name="defaultMethodDisplay">Default method display to use (when not customized).</param>
/// <param name="testMethod">The method under test.</param>
public XunitTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod)
: base(diagnosticMessageSink, defaultMethodDisplay, testMethod)
{
}
public IEnumerable <IXunitTestCase> Discover(
ITestFrameworkDiscoveryOptions discoveryOptions,
ITestMethod testMethod,
IAttributeInfo factAttribute)
{
var skipReason = factAttribute.GetNamedArgument <string>("Skip");
if (skipReason != null)
{
return(CreateTestCasesForSkip(discoveryOptions, testMethod, factAttribute, skipReason));
}
if (discoveryOptions.PreEnumerateTheoriesOrDefault())
{
try
{
var dataAttributes = testMethod.Method.GetCustomAttributes(typeof(DataAttribute)).ToList();
var results = new List <IXunitTestCase>();
for (int index = 0; index < dataAttributes.Count; index++)
{
var discovererAttribute =
dataAttributes[index]
.GetCustomAttributes(typeof(DataDiscovererAttribute))
.First();
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(MessageSink, discovererAttribute);
skipReason = dataAttributes[index].GetNamedArgument <string>("Skip");
if (!discoverer.SupportsDiscoveryEnumeration(dataAttributes[index], testMethod.Method))
{
return new[] { CreateTestCaseForTheory(discoveryOptions, testMethod, factAttribute) }
}
;
IEnumerable <object[]> data =
discoverer.GetData(dataAttributes[index], testMethod.Method).ToList();
if (data is IDictionary <string, object[]> )
{
foreach (KeyValuePair <string, object[]> dataRow in (IDictionary <string, object[]>)data)
{
var testCase =
skipReason != null
? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, dataRow.Value, skipReason)
: CreateTestCaseForNamedDataRow(discoveryOptions, testMethod, factAttribute, index, dataRow.Key);
results.Add(testCase);
}
}
else if (data.GroupBy(x => (x[0]?.ToString()) ?? string.Empty).All(x => x.Count() == 1))
{
foreach (object[] dataRow in data)
{
var testCase =
skipReason != null
? CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, dataRow, skipReason)
: CreateTestCaseForNamedDataRow(discoveryOptions, testMethod, factAttribute, index, dataRow[0].ToString() ?? string.Empty);
results.Add(testCase);
}
}
else
{
results.AddRange(
data.Select((x, i) =>
skipReason != null ?
CreateTestCaseForSkippedDataRow(discoveryOptions, testMethod, factAttribute, x, skipReason) :
CreateTestCaseForDataRow(discoveryOptions, testMethod, factAttribute, index, i)));
}
}
if (results.Count == 0)
{
results.Add(
new ExecutionErrorTestCase(
MessageSink,
discoveryOptions.MethodDisplayOrDefault(),
TestMethodDisplayOptions.None,
testMethod,
$"No data found for {testMethod.TestClass.Class.Name}.{testMethod.Method.Name}"));
}
return(results);
}
catch (Exception ex) // If something goes wrong, fall through to return just the XunitTestCase
{
MessageSink.OnMessage(
new DiagnosticMessage($"Exception thrown during theory discovery on '{testMethod.TestClass.Class.Name}.{testMethod.Method.Name}'; falling back to single test case.{Environment.NewLine}{ex}"));
}
}
return(CreateTestCasesForTheory(discoveryOptions, testMethod, factAttribute));
}
public SkippableFactTestCase(IMessageSink diagnosticMessageSink, Sdk.TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, object[] testMethodArguments = null)
: base(diagnosticMessageSink, defaultMethodDisplay, testMethod, testMethodArguments)
{
}
public SkippableTheoryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, TestMethodDisplayOptions defaultMethodDisplayOptions, ITestMethod testMethod)
: base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
{
}
/// <summary> /// Creates a test case for the entire theory. This is used when one or more of the theory data items /// are not serializable, or if the user has requested to skip theory pre-enumeration. By default, /// returns an instance of <see cref="NBenchTheoryTestCase"/>, which performs the data discovery at runtime. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <returns>The test case</returns> protected virtual IXunitTestCase CreateTestCaseForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute) => new XunitTheoryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod);
public override TestResult[] Execute(ITestMethod testMethod)
{
TestResult[] errorResults = ValidateElevated(testMethod);
if (errorResults != null)
{
return(errorResults);
}
Exception signatureException = GetMethodSignatureException(testMethod);
if (signatureException != null)
{
return(testMethod.CreateExceptionResult(signatureException));
}
var runParameters = TestRunParameters.Read();
string logFolder = runParameters.LogFolder;
bool shouldLog = !string.IsNullOrEmpty(logFolder);
if (shouldLog)
{
try
{
logFolder = CreateLogFolder(logFolder, testMethod.TestMethodName);
}
catch (Exception e)
{
return(testMethod.CreateExceptionResult(e));
}
}
int iterations = runParameters.Iterations;
var results = new TestResult[iterations];
for (int iteration = 1; iteration <= iterations; iteration++)
{
string sessionName = $"{testMethod.TestMethodName}-{iteration}";
using (var session = new TraceEventSession(sessionName))
{
EnableKernelProviders(session, shouldLog);
TraceEventDispatcher source;
ZippedETLWriter writer = null;
if (shouldLog)
{
string etlPath = Path.Combine(logFolder, $"Iteration{iteration}.etl");
source = new ETWReloggerTraceEventSource(sessionName, TraceEventSourceType.Session, etlPath);
writer = new ZippedETLWriter(etlPath);
}
else
{
source = session.Source;
}
EnableProviders(session);
PerformanceTestContext context = CreateContext(source);
Task <TestResult> testTask = Task.Run(() => testMethod.Invoke(new object[] { context }));
// This is a blocking call that in the case of ETWReloggerTraceEventSource, must be run on the same
// thread as ETWReloggerTraceEventSource was created on. It will become unblocked when the
// PerformanceTestContext calls StopProcessing on the source.
source.Process();
TestResult result = testTask.Result;
string displayName = testMethod.TestMethodName;
if (iterations > 1)
{
displayName += $" [{iteration}/{iterations}]";
}
result.DisplayName = displayName;
session.Flush();
OnIterationEnded(context);
context.LogScenarios();
context.LogMemoryDelta();
context.LogMessage($"{displayName} completed. {session.EventsLost} events lost.");
context.WriteLogsToResult(result, writer);
results[iteration - 1] = result;
}
}
return(results);
}
/// <summary>
/// Constructs a new method container.
/// </summary>
/// <param name="instance">An instance of the method's type.</param>
/// <param name="method">The method reflection object.</param>
/// <param name="testMethod">The test method.</param>
public MethodContainer(object instance, MethodInfo method, ITestMethod testMethod)
: base()
{
_methodTask = new MethodInvokeWorkItem(instance, method, testMethod);
_methodInfo = method;
}
public MefFactTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay defaultMethodDisplay, ITestMethod testMethod, IAttributeInfo factAttributeInfo)
: base(diagnosticMessageSink, defaultMethodDisplay, testMethod)
{
var factAttribute = MefFactAttribute.Instantiate(factAttributeInfo);
this.SkipReason = factAttribute.Skip;
this.parts = factAttribute.Parts;
this.assemblies = factAttribute.Assemblies;
this.compositionVersions = factAttribute.CompositionVersions;
this.noCompatGoal = factAttribute.NoCompatGoal;
this.invalidConfiguration = factAttribute.InvalidConfiguration;
if (this.Traits.ContainsKey(Tests.Traits.SkipOnMono) && TestUtilities.IsOnMono)
{
this.SkipReason = this.SkipReason ?? "Test marked as skipped on Mono runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnMono]);
}
if (this.Traits.ContainsKey(Tests.Traits.SkipOnCoreCLR) && TestUtilities.IsOnCoreCLR)
{
this.SkipReason = this.SkipReason ?? "Test marked as skipped on CoreCLR runtime due to unsupported feature: " + string.Join(", ", this.Traits[Tests.Traits.SkipOnCoreCLR]);
}
}
/// <summary> /// Creates a test case for a single row of data. By default, returns an instance of <see cref="XunitTestCase"/> /// with the data row inside of it. /// </summary> /// <param name="discoveryOptions">The discovery options to be used.</param> /// <param name="testMethod">The test method the test cases belong to.</param> /// <param name="theoryAttribute">The theory attribute attached to the test method.</param> /// <param name="dataRow">The row of data for this test case.</param> /// <returns>The test case</returns> protected virtual NBenchTestCase CreateTestCaseForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) => new NBenchTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow);
// ReSharper disable once TooManyArguments
protected override Task <RunSummary> RunTestMethodAsync(ITestMethod testMethod, IReflectionMethodInfo method,
IEnumerable <IXunitTestCase> testCases,
object[] constructorArguments)
=> new TestMethodRunner(testMethod, Class, method, testCases, DiagnosticMessageSink, MessageBus,
new ExceptionAggregator(Aggregator), CancellationTokenSource, constructorArguments)
.RunAsync();
void IProtectedTestCase.SetTestMethod(ITestMethod testMethod)
{
base.TestMethod = testMethod;
}
