internal WcfTestCase(IXunitTestCase testCase, string skippedReason = null, bool isTheory = false, IMessageSink diagnosticMessageSink = null)
{
_testCase = testCase;
_skippedReason = skippedReason;
_isTheory = isTheory;
_diagnosticMessageSink = diagnosticMessageSink;
}
public ScenarioOutlineRunner(
IMessageSink diagnosticMessageSink,
IXunitTestCase scenarioOutline,
string displayName,
string skipReason,
object[] constructorArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(
scenarioOutline,
displayName,
skipReason,
constructorArguments,
noArguments,
messageBus,
aggregator,
cancellationTokenSource)
{
this.diagnosticMessageSink = diagnosticMessageSink;
this.scenarioRunnerFactory = new ScenarioRunnerFactory(
this.TestCase,
this.DisplayName,
this.MessageBus,
this.TestClass,
this.ConstructorArguments,
this.SkipReason,
this.BeforeAfterAttributes,
this.Aggregator,
this.CancellationTokenSource);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTheoryTestCaseRunner"/> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
public XunitTheoryTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, NoArguments, messageBus, aggregator, cancellationTokenSource) { }
public Task<Task<RunSummary>> InitializeAsync(IXunitTestCase testCase, MethodInfo methodInfo, IMessageBus messageBus) {
_runSummaryTcs = new TaskCompletionSource<RunSummary>();
_stopwatch = Stopwatch.StartNew();
_testCase = testCase;
_messageBus = messageBus;
_observedTasks = new ConcurrentDictionary<Task, Lazy<IDisposable>>();
return Task.FromResult(_runSummaryTcs.Task);
}
/// <summary>
/// Initializes a new instance of the <see cref="UITestCaseRunner" /> class.
/// </summary>
/// <param name="testCase">The test case to be run.</param>
/// <param name="displayName">The display name of the test case.</param>
/// <param name="skipReason">The skip reason, if the test is to be skipped.</param>
/// <param name="constructorArguments">The arguments to be passed to the test class constructor.</param>
/// <param name="testMethodArguments">The arguments to be passed to the test method.</param>
/// <param name="messageBus">The message bus to report run status to.</param>
/// <param name="aggregator">The exception aggregator used to run code and collect exceptions.</param>
/// <param name="cancellationTokenSource">The task cancellation token source, used to cancel the test run.</param>
/// <param name="syncContextAdapter">The <see cref="SynchronizationContext" /> adapter to use.</param>
internal UITestCaseRunner(
IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource,
SyncContextAdapter syncContextAdapter)
: base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource) {
this.syncContextAdapter = syncContextAdapter;
}
public BenchmarkTestCaseRunner(IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource)
{
_diagnosticMessageSink = diagnosticMessageSink;
_discoverArguments = testMethodArguments == null;
}
public TestCaseRunnerWithMethodFixtures(IXunitTestCase testCase, string displayName, string skipReason, object[] constructorArguments, object[] testMethodArguments, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus, aggregator, cancellationTokenSource) {
for (var i = 0; i < constructorArguments.Length; i++) {
var methodFixture = constructorArguments[i] as IMethodFixture;
if (methodFixture == null) {
continue;
}
var methodFixtureType = methodFixture.GetType();
methodFixture = (IMethodFixture)Activator.CreateInstance(methodFixtureType);
constructorArguments[i] = methodFixture;
_methodFixtures.Add(methodFixture);
}
}
public void UsingMetadata(
string text, IStepContext stepContext, IStep step, IScenario scenario, IXunitTestCase scenarioOutline)
{
"When I execute a step"
.f(c => stepContext = c);
"Then the step context contains metadata about the step"
.f(() => (step = stepContext.Step.Should().NotBeNull().And.Subject.As<IStep>())
.DisplayName.Should().Be("Xbehave.Test.Acceptance.MetadataFeature.UsingMetadata(text: \"abc\") [01] When I execute a step"));
"And the step contains metadata about the scenario"
.f(() => (scenario = step.Scenario.Should().NotBeNull().And.Subject.As<IScenario>())
.DisplayName.Should().Be("Xbehave.Test.Acceptance.MetadataFeature.UsingMetadata(text: \"abc\")"));
"And the step contains metadata about the scenario outline"
.f(() => scenario.ScenarioOutline.Should().NotBeNull().And.Subject.As<IXunitTestCase>()
.DisplayName.Should().Be("Xbehave.Test.Acceptance.MetadataFeature.UsingMetadata"));
}
private void ApplyScenarioInfo(IXunitTestCase testCase, object[] dataRow)
{
if (dataRow.Length < 1)
{
return;
}
var scenario = dataRow[0] as Scenario;
if (scenario == null)
{
return;
}
testCase.Traits.Add("grading", new List <string>()
{
scenario.Version
});
}
TestableXunitTestInvoker(
ITest test,
IMessageBus messageBus,
Type testClass,
object[] constructorArguments,
MethodInfo testMethod,
object[] testMethodArguments,
IReadOnlyList <BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource,
Action lambda)
: base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cancellationTokenSource)
{
this.lambda = lambda;
TestCase = (IXunitTestCase)test.TestCase;
Aggregator = aggregator;
TokenSource = cancellationTokenSource;
}
public ScenarioOutlineTestCaseRunner(
IMessageSink diagnosticMessageSink,
IXunitTestCase scenarioOutline,
string displayName,
string skipReason,
object[] constructorArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
: base(
scenarioOutline,
displayName,
skipReason,
constructorArguments,
noArguments,
messageBus,
aggregator,
cancellationTokenSource)
{
this.diagnosticMessageSink = diagnosticMessageSink;
}
public InterpreterTestCaseRunner(
IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
object[] testMethodArguments,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource
) : base(
testCase,
displayName,
skipReason,
constructorArguments,
testMethodArguments,
messageBus,
aggregator,
cancellationTokenSource
)
{
}
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
{
var parameters = string.Empty;
if (testCase.TestMethodArguments != null)
{
parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
}
var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Starting execution of {test}"));
try
{
return(await base.RunTestCaseAsync(testCase));
}
finally
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Finished execution of {test}"));
}
}
/// <summary>
/// Computes values from the test case and resolves the test method arguments. To be called by the public RunAsync method that
/// will end up being exposed by the derived class as the primary public API.
/// </summary>
/// <param name="testCase">The test case that is being run</param>
/// <param name="testMethodArguments">The test method arguments to be converted</param>
protected (Type TestClass, MethodInfo TestMethod, IReadOnlyCollection <BeforeAfterTestAttribute> BeforeAfterTestAttributes) Initialize(
IXunitTestCase testCase,
ref object?[]?testMethodArguments)
{
// TODO: This means XunitTestFramework can never run test cases without a class & method
var testClass = testCase.TestClass?.Class.ToRuntimeType() ?? throw new ArgumentException("testCase.TestClass.Class does not map to a Type object", nameof(testCase));
var testMethod = testCase.Method.ToRuntimeMethod() ?? throw new ArgumentException("testCase.TestMethod does not map to a MethodInfo object", nameof(testCase));
var parameters = testMethod.GetParameters();
var parameterTypes = new Type[parameters.Length];
for (var i = 0; i < parameters.Length; i++)
{
parameterTypes[i] = parameters[i].ParameterType;
}
testMethodArguments = Reflector.ConvertArguments(testMethodArguments, parameterTypes);
IEnumerable <Attribute> beforeAfterTestCollectionAttributes;
if (testCase.TestCollection.CollectionDefinition is _IReflectionTypeInfo collectionDefinition)
{
beforeAfterTestCollectionAttributes = collectionDefinition.Type.GetCustomAttributes(typeof(BeforeAfterTestAttribute));
}
else
{
beforeAfterTestCollectionAttributes = Enumerable.Empty <Attribute>();
}
var beforeAfterTestAttributes =
beforeAfterTestCollectionAttributes
.Concat(testClass.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Concat(testMethod.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Concat(testClass.Assembly.GetCustomAttributes(typeof(BeforeAfterTestAttribute)))
.Cast <BeforeAfterTestAttribute>()
.CastOrToReadOnlyCollection();
return(testClass, testMethod, beforeAfterTestAttributes);
}
protected override Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
{
if (testCase.GetType() == typeof(XunitTestCase))
{
// If we get here this is a 'regular' test case, not something that represents a skipped test.
//
// We can take control of it's invocation thusly.
var runner = new AspNetTestCaseRunner(
testCase,
testCase.DisplayName,
testCase.SkipReason,
_constructorArguments,
testCase.TestMethodArguments,
MessageBus,
new ExceptionAggregator(Aggregator),
CancellationTokenSource);
return(runner.RunAsync());
}
if (testCase.GetType() == typeof(XunitTheoryTestCase))
{
// If we get here this is a 'regular' theory test case, not something that represents a skipped test.
//
// We can take control of it's invocation thusly.
var runner = new AspNetTheoryTestCaseRunner(
testCase,
testCase.DisplayName,
testCase.SkipReason,
_constructorArguments,
_diagnosticMessageSink,
MessageBus,
new ExceptionAggregator(Aggregator),
CancellationTokenSource);
return(runner.RunAsync());
}
return(base.RunTestCaseAsync(testCase));
}
private async Task RunActionAsync(IXunitTestCase testCase, Func <Task> action, RunSummary runSummary, string timeoutMessage)
{
Exception exception = null;
_stopwatch.Restart();
try {
var task = action();
await ParallelTools.When(task, 60_000, timeoutMessage);
} catch (Exception ex) {
exception = ex;
}
_stopwatch.Stop();
var time = (decimal)_stopwatch.Elapsed.TotalSeconds;
var taskRunSummary = exception != null
? RegisterFailedRunSummary(testCase, time, exception)
: new RunSummary
{
Time = time
};
runSummary.Aggregate(taskRunSummary);
}
/// <inheritdoc />
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
{
// Tracing test methods invocation gives an ability to see what tests were running when
// the console runner crashed with stack overflow or similar unrecoverable error.
m_logger.LogVerbose($"Starting '{testCase.DisplayName}'... ");
try
{
// Unset any synchronization restrictions set by the caller (looking at you, xunit)
SynchronizationContext.SetSynchronizationContext(null);
return(await base.RunTestCaseAsync(testCase).WithTimeoutAsync(TimeSpan.FromMinutes(5)));
}
catch (TimeoutException)
{
// Tracing the test case that fails with timeout
m_logger.LogVerbose($"Test '{testCase.DisplayName}' failed with timeout.");
throw;
}
finally
{
m_logger.LogVerbose($"Finished '{testCase.DisplayName}'");
}
}
private async Task <RunSummary> RunTestCaseWithMethodFixturesAsync(IXunitTestCase testCase, TaskObserver taskObserver, ITestMainThread testMainThread)
{
var runSummary = new RunSummary();
var methodFixtures = CreateMethodFixtures(testCase, runSummary);
if (Aggregator.HasExceptions)
{
return(runSummary);
}
var testCaseConstructorArguments = await InitializeMethodFixturesAsync(testCase, runSummary, methodFixtures);
if (!Aggregator.HasExceptions)
{
var testCaseRunSummary = await GetTestRunSummary(RunTestCaseAsync(testCase, testCaseConstructorArguments), taskObserver.Task);
runSummary.Aggregate(testCaseRunSummary);
}
await DisposeMethodFixturesAsync(testCase, runSummary, methodFixtures);
await WaitForObservedTasksAsync(testCase, runSummary, taskObserver, testMainThread);
return(runSummary);
}
public WpfTheoryTestCaseRunner(
WpfTestSharedData sharedData,
IXunitTestCase testCase,
string displayName,
string skipReason,
object[] constructorArguments,
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource
)
: base(
testCase,
displayName,
skipReason,
constructorArguments,
diagnosticMessageSink,
messageBus,
aggregator,
cancellationTokenSource
)
{
SharedData = sharedData;
}
public async Task<Task<RunSummary>> InitializeAsync(IXunitTestCase testCase, MethodInfo methodInfo, IMessageBus messageBus) {
await ContainerHost.Increment();
return MethodFixtureBase.DefaultInitializeResult;
}
public static TestableXunitTestAssemblyRunner Create(ITestAssembly assembly = null,
IXunitTestCase[] testCases = null,
ITestFrameworkOptions options = null)
{
if (testCases == null)
testCases = new[] { Mocks.XunitTestCase<ClassUnderTest>("Passing") };
return new TestableXunitTestAssemblyRunner(
assembly ?? testCases.First().TestMethod.TestClass.TestCollection.TestAssembly,
testCases ?? new IXunitTestCase[0],
SpyMessageSink.Create(),
options ?? new TestFrameworkOptions()
);
}
public Scenario(IXunitTestCase scenarioOutline, string displayName)
{
this.ScenarioOutline = scenarioOutline;
this.DisplayName = displayName;
}
public ScenarioTestCaseRunner(IXunitTestCase testCase, string displayName, string skipReason, object[] constructorArguments, IMessageSink diagnosticMessageSink, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, NoArguments, messageBus, aggregator, cancellationTokenSource)
{
_diagnosticMessageSink = diagnosticMessageSink;
}
public UITestCase(IXunitTestCase testCase)
{
this.testCase = testCase;
}
public override Task<Task<RunSummary>> InitializeAsync(IXunitTestCase testCase, MethodInfo methodInfo, IMessageBus messageBus) {
DisplayName = testCase.DisplayName;
MethodInfo = methodInfo;
return base.InitializeAsync(testCase, methodInfo, messageBus);
}
/// <summary>
/// Initializes a new instance of the <see cref="XunitTest"/> class.
/// </summary>
/// <param name="testCase">The test case this test belongs to.</param>
/// <param name="displayName">The display name for this test.</param>
public XunitTest(IXunitTestCase testCase, string displayName)
{
TestCase = testCase;
DisplayName = displayName;
}
/// <summary>
/// Initializes a new instance of the <see cref="StaTestCase"/> class.
/// </summary>
/// <param name="testCase">The test case.</param>
public StaTestCase(IXunitTestCase testCase)
{
this.testCase = testCase;
}
internal SkippedTestCase(IXunitTestCase testCase, string skippedReason)
{
_testCase = testCase;
_skippedReason = skippedReason;
}
public SkipReasonTestCase(bool isTheory, string skipReason, IXunitTestCase wrappedTestCase)
{
_isTheory = isTheory;
_skipReason = wrappedTestCase.SkipReason ?? skipReason;
_wrappedTestCase = wrappedTestCase;
}
TestableXunitTestInvoker(ITest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, IReadOnlyList<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, Action lambda)
: base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, beforeAfterAttributes, aggregator, cancellationTokenSource)
{
this.lambda = lambda;
TestCase = (IXunitTestCase)test.TestCase;
Aggregator = aggregator;
TokenSource = cancellationTokenSource;
}
internal SkippedTestCase(IXunitTestCase testCase, string skippedReason)
{
_testCase = testCase;
_skippedReason = skippedReason;
}
public WpfTestCase(IXunitTestCase testCase)
{
this.testCase = testCase;
}
/// <summary>
/// Discover test cases from a test method. By default, if the method is generic, or
/// it contains arguments, returns a single <see cref="ExecutionErrorTestCase"/>;
/// otherwise, it returns the result of calling <see cref="CreateTestCase"/>.
/// </summary>
/// <param name="discoveryOptions">The discovery options to be used.</param>
/// <param name="testMethod">The test method the test cases belong to.</param>
/// <param name="testCaseAttribute">The fact attribute attached to the test method.</param>
/// <returns>Returns zero or more test cases represented by the test method.</returns>
public virtual IEnumerable <IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo testCaseAttribute)
{
IXunitTestCase testCase = CreateTestCase(discoveryOptions, testMethod, testCaseAttribute);
return(new[] { testCase });
}
public Scenario(IXunitTestCase scenarioOutline, string displayName)
{
this.scenarioOutline = scenarioOutline;
this.displayName = displayName;
}
public void Deserialize(IXunitSerializationInfo info)
{
testCase = info.GetValue<IXunitTestCase>("InnerTestCase");
}
public TestCase(IXunitTestCase @case, Action action) : this(@case, new TestMethod(@case.TestMethod),
new First(), action)
{
}
/// <nodoc /> public MtaTestCase(IXunitTestCase testCase) => _testCase = testCase;
public DependencyInjectionTheoryTestCaseRunner(IServiceProvider provider, IXunitTestCase testCase,
string displayName, string skipReason, object?[] constructorArguments, IMessageSink diagnosticMessageSink,
IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, diagnosticMessageSink, messageBus, aggregator, cancellationTokenSource) =>
public XunitMainThreadTestCaseDecorator(IXunitTestCase testCase) : base(testCase)
{
}
protected XunitTestCaseDecoratorBase(IXunitTestCase testCase) {
_testCase = testCase;
}
public AutofacTest(ILifetimeScope lifetimeScope, IXunitTestCase testCase, string displayName)
: base(testCase, displayName)
{
LifetimeScope = lifetimeScope;
}
public SkipReasonTestCase(string skipReason, IXunitTestCase wrappedTestCase)
{
SkipReason = wrappedTestCase.SkipReason ?? skipReason;
_wrappedTestCase = wrappedTestCase;
}
public XunitTestCaseDecorator(IXunitTestCase testCase) : base(testCase)
{
}
protected override Task<RunSummary> RunTestCaseAsync (IXunitTestCase testCase)
{
return vsClient.RunAsync((VsixTestCase)testCase, MessageBus, Aggregator, constructorArguments);
}
void IXunitSerializable.Deserialize(IXunitSerializationInfo info)
{
this.testCase = info.GetValue <IXunitTestCase>(nameof(testCase));
}
public virtual Task<Task<RunSummary>> InitializeAsync(IXunitTestCase testCase, MethodInfo methodInfo, IMessageBus messageBus) => DefaultInitializeTask;
public TestCaseWithoutTraits(IXunitTestCase source)
{
_source = source;
Traits = new Dictionary <string, List <string> >();
}
public SkipReasonTestCase(string skipReason, IXunitTestCase wrappedTestCase)
{
_skipReason = wrappedTestCase.SkipReason ?? skipReason;
_wrappedTestCase = wrappedTestCase;
}
public DependencyInjectionTestCaseRunner(IServiceProvider provider, IXunitTestCase testCase,
string displayName, string skipReason, object?[] constructorArguments, object[] testMethodArguments,
IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
: base(testCase, displayName, skipReason, constructorArguments, testMethodArguments, messageBus,
aggregator, cancellationTokenSource) => _provider = provider;
public ScheduledTestCase(IXunitTestCase testCase)
{
_TestCase = testCase;
}
public DeviceTestCase(IXunitTestCase inner)
{
_inner = inner;
}
protected override ITest CreateTest(IXunitTestCase testCase, string displayName)
{
throw new NotSupportedException("Test can only be created with scenario");
}
public void Deserialize(IXunitSerializationInfo info) {
_testCase = info.GetValue<IXunitTestCase>("testCase");
_suppressDebugFail = info.GetValue<bool>("suppressDebugFail");
}
protected virtual ScenarioTest CreateTest(IXunitTestCase testCase, IScenarioIdentifier scenarioIdentifier)
{
var displayName = $"{testCase.DisplayName} (Scenario: {scenarioIdentifier})";
return(new ScenarioTest((MultipleScenariosTestCase)testCase, scenarioIdentifier, displayName));
}
/// <inheritdoc/>
public void Deserialize(IXunitSerializationInfo info)
{
this.testCase = info.GetValue <IXunitTestCase>("InnerTestCase");
}
protected override ITest CreateTest(IXunitTestCase testCase, string displayName)
{
return(new HandlerTest(testCase, displayName, _handlerName));
}
/// <inheritdoc/>
protected override Task<RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
=> testCase.RunAsync(DiagnosticMessageSink, MessageBus, ConstructorArguments, new ExceptionAggregator(Aggregator), CancellationTokenSource);
public XunitTestCaseDecorator(IXunitTestCase testCase) : base(testCase)
{
}
