public void CreateFixtures()
{
foreach (var fixtureType in _fixtureTypes)
{
_aggregator.Run(() =>
{
var fixture = ObjectFactory.CreateInstance(fixtureType, null);
_fixtures.Add(fixtureType, fixture);
});
}
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = await TaskExecutor.RunAsync(
CancellationTokenSource.Token,
_testRunners.Select(r => (Func <Task <RunSummary> >)r.RunScenarioAsync).ToArray(),
TestCase.TestMethod.TestClass);
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose));
}
runSummary.Time += timer.Total;
return(runSummary);
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in _testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
protected override Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
if (lambda != null)
{
aggregator.Run(lambda);
}
InvokeTestAsync_Called = true;
return(Task.FromResult(Tuple.Create(runTime, output)));
}
protected override Task <decimal> InvokeTestAsync(ExceptionAggregator aggregator)
{
if (lambda != null)
{
aggregator.Run(lambda);
}
InvokeTestAsync_Called = true;
return(Task.FromResult(runTime));
}
protected override ValueTask <decimal> InvokeTestMethodAsync(
XunitTestInvokerContext ctxt,
object?testClassInstance)
{
if (lambda == null)
{
return(base.InvokeTestMethodAsync(ctxt, testClassInstance));
}
Aggregator.Run(lambda);
return(default);
protected override ValueTask <(decimal ExecutionTime, string Output)?> InvokeTestAsync(TestRunnerContext ctxt)
{
if (lambda != null)
{
aggregator.Run(lambda);
}
InvokeTestAsync_Called = true;
return(new((runTime, output)));
}
public void Dispose()
{
var aggregator = new ExceptionAggregator();
var tasks = _collectionFixtureMappings.Values.OfType <IAsyncLifetime>()
.Select(asyncFixture => aggregator.RunAsync(asyncFixture.DisposeAsync))
.Concat(_assemblyFixtureMappings.Values.OfType <IAsyncLifetime>()
.Select(asyncFixture => aggregator.RunAsync(asyncFixture.DisposeAsync)))
.ToArray();
foreach (var disposable in _assemblyFixtureMappings.Values.OfType <IDisposable>()
.Concat(_collectionFixtureMappings.Values.OfType <IDisposable>()))
{
aggregator.Run(disposable.Dispose);
}
Trace.Listeners.Clear();
Constants.Tracer.Listeners.Clear();
Task.WaitAll(tasks);
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in _testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
if (dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var trackedObject in disposalTracker.TrackedObjects)
{
if (trackedObject is IAsyncDisposable asyncDisposable)
{
await timer.AggregateAsync(() => cleanupAggregator.RunAsync(asyncDisposable.DisposeAsync));
}
if (trackedObject is IDisposable disposable)
{
timer.Aggregate(() => cleanupAggregator.Run(disposable.Dispose));
}
}
disposalTracker.Clear();
runSummary.Time += timer.Total;
return(runSummary);
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List <Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
/// <summary>
/// Runs this fixture.
/// </summary>
/// <param name="output">The test output helper.</param>
/// <returns></returns>
public async Task RunAsync(ITestOutputHelper output = null)
{
_run = true;
try
{
if (RequestMessage == null)
{
throw new InvalidOperationException($"Must call {nameof(When)} to configure an HTTP request");
}
if (!_assertions.Any())
{
throw new InvalidOperationException("No assertions to run");
}
if (output != null)
{
_loggerProvider.SetTestOutputHelper(output);
}
using var client = Factory.CreateClient();
var provider = Factory.Server.Services;
// Bootstrap.
if (_bootstrapFunctions.Any())
{
using var scope = provider.CreateScope();
foreach (var bootstrap in _bootstrapFunctions)
{
await bootstrap(scope.ServiceProvider);
}
}
var logger = provider.GetRequiredService <ILogger <MvcFunctionalTestFixture <TStartup> > >();
var requestBody = await(RequestMessage.Content?.ReadAsStringAsync() ?? Task.FromResult <string>(null));
logger.LogInformation($"Sending request {RequestMessage}\n{requestBody ?? "<no body>"}");
var response = await client.SendAsync(RequestMessage);
var responseBody = await(response.Content?.ReadAsStringAsync() ?? Task.FromResult <string>(null));
logger.LogInformation($"Received {response}\n{responseBody ?? "<no body>"}");
var aggregator = new ExceptionAggregator();
foreach (var assertion in _assertions)
{
switch (assertion)
{
case ResponseAssertion ra:
aggregator.Run(() => ra.Assertion(response));
break;
case ResponseBodyAssertion ra:
if (aggregator.TryRun(() => ra.Deserializer(responseBody), out var result))
{
foreach (var a in ra.Assertions)
{
await aggregator.RunAsync(() => a(provider, result));
}
}
break;
case ServiceAssertion sa:
using (var scope = provider.CreateScope())
{
await aggregator.RunAsync(() => sa.Assertion(scope.ServiceProvider));
}
break;
case RequestAssertion ra:
var context = await ra.ContextFactory(response);
var fixture = new MvcFunctionalTestFixture <TStartup>(_loggerProvider, this);
ra.Configurator(context, fixture);
await aggregator.RunAsync(() => fixture.RunAsync());
break;
}
}
aggregator.ThrowIfHasExceptions();
}
finally
{
Reset();
}
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
string output = "";
var gherkinDocument = await this.TestCase.FeatureTypeInfo.GetDocumentAsync();
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out var example, out var exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() =>
{
var stopwatch = Stopwatch.StartNew();
testOutputHelper.Initialize(MessageBus, test);
try
{
RunScenario(gherkinDocument, scenario);
}
finally
{
stopwatch.Stop();
summary.Time = (decimal)stopwatch.Elapsed.TotalSeconds;
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
}
);
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output);
}
else
{
testResult = new TestFailed(test, summary.Time, output, exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(summary);
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
var output = new StringBuilder();
var gherkinDocument = await SpecFlowParserHelper.ParseSpecFlowDocumentAsync(TestCase.FeatureFile.FeatureFilePath);
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
Examples example = null;
Gherkin.Ast.TableRow exampleRow = null;
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out example, out exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() => RunScenario(gherkinDocument, scenario, output));
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output.ToString());
}
else
{
testResult = new TestFailed(test, summary.Time, output.ToString(), exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output.ToString())))
{
CancellationTokenSource.Cancel();
}
return(summary);
}