// This method is called by the xUnit test framework class es to run the test case. We will do the
// loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
// continue to re-run the test until the aggregator has an error (meaning that some internal error
// condition happened), or the test runs without failure, or we've hit the maximum number of tries.
/// <inheritdoc />
public override async Task <RunSummary> RunAsync(
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
int runCount = 0;
while (true)
{
// This is really the only tricky bit: we need to capture and delay messages (since those will
// contain run status) until we know we've decided to accept the final result;
var delayedMessageBus = new DelayedMessageBus(messageBus);
try
{
var summary = await base.RunAsync(
diagnosticMessageSink,
delayedMessageBus,
constructorArguments,
aggregator,
cancellationTokenSource).WithTimeout(30_000);
if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= _maxRetries)
{
delayedMessageBus.Dispose(); // Sends all the delayed messages
return(summary);
}
}
catch (TimeoutException ex)
{
if (++runCount >= _maxRetries)
{
var runSummary = new RunSummary {
Total = 1, Failed = 1
};
aggregator.Add(ex);
if (!delayedMessageBus.QueueMessage(new TestCleanupFailure(
new XunitTest(this, DisplayName),
aggregator.ToException() !)))
{
cancellationTokenSource.Cancel();
}
delayedMessageBus.Dispose(); // Sends all the delayed messages
return(runSummary);
}
}
diagnosticMessageSink.OnMessage(
new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName,
runCount));
GC.Collect();
await Task.Delay(100);
}
}
public async Task <RunSummary> RunAsync(VsixTestCase vsixTest, IMessageBus messageBus, ExceptionAggregator aggregator)
{
// We don't apply retry behavior when a debugger is attached, since that
// typically means the developer is actually debugging a failing test.
#if !DEBUG
if (Debugger.IsAttached)
{
return(await RunAsyncCore(vsixTest, messageBus, aggregator));
}
#endif
var bufferBus = new InterceptingMessageBus();
var summary = await RunAsyncCore(vsixTest, bufferBus, aggregator);
var shouldRecycle = vsixTest.RecycleOnFailure.GetValueOrDefault();
// Special case for MEF cache corruption, clear cache and restart the test.
if (summary.Failed != 0 && (
(aggregator.HasExceptions && aggregator.ToException().GetType().FullName == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException") ||
(bufferBus.Messages.OfType <IFailureInformation>().Where(fail => fail.ExceptionTypes.Any(type => type == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException")).Any())
))
{
shouldRecycle = true;
try
{
var path = VsSetup.GetComponentModelCachePath(_devEnvPath, new Version(_visualStudioVersion), _rootSuffix);
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
}
catch (IOException)
{
s_tracer.TraceEvent(TraceEventType.Warning, 0, "Failed to clear MEF cache after a failed test caused by an InvalidMEFCacheException.");
}
}
if (summary.Failed != 0 && shouldRecycle)
{
Recycle();
aggregator.Clear();
summary = await RunAsyncCore(vsixTest, messageBus, aggregator);
}
else
{
// Dispatch messages from the first run to actual bus.
foreach (var msg in bufferBus.Messages)
{
messageBus.QueueMessage(msg);
}
}
return(summary);
}
private AppDomainFixtureContainer CreateRemoteFixtureContainer(ExceptionAggregator aggregator)
{
var remoteContainer = RemoteFunc.Invoke(_appDomainContext.Domain,
_appDomainFixtureTypes,
aggregator.ToException(),
(fixtureTypes, exception) =>
{
var exceptionAggregator = ObjectFactory.CreateExceptionAggregator(exception);
return(new AppDomainFixtureContainer(fixtureTypes, exceptionAggregator));
});
remoteContainer.CreateFixtures();
return(remoteContainer);
}
private async Task <decimal> InvokeTestMethodAsync(ExceptionAggregator aggregator, ITestOutputHelper output)
{
var retryAttribute = GetRetryAttribute(TestMethod);
var collectDump = TestMethod.GetCustomAttribute <CollectDumpAttribute>() != null;
if (!typeof(LoggedTestBase).IsAssignableFrom(TestClass) || retryAttribute == null)
{
return(await new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource, output, null, collectDump).RunAsync());
}
var retryPredicateMethodName = retryAttribute.RetryPredicateName;
var retryPredicateMethod = TestClass.GetMethod(retryPredicateMethodName,
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static,
null,
new Type[] { typeof(Exception) },
null)
?? throw new InvalidOperationException($"No valid static retry predicate method {retryPredicateMethodName} was found on the type {TestClass.FullName}.");
if (retryPredicateMethod.ReturnType != typeof(bool))
{
throw new InvalidOperationException($"Retry predicate method {retryPredicateMethodName} on {TestClass.FullName} does not return bool.");
}
var retryContext = new RetryContext()
{
Limit = retryAttribute.RetryLimit,
Reason = retryAttribute.RetryReason,
};
var retryAggregator = new ExceptionAggregator();
var loggedTestInvoker = new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, retryAggregator, CancellationTokenSource, output, retryContext, collectDump);
var totalTime = 0.0M;
do
{
retryAggregator.Clear();
totalTime += await loggedTestInvoker.RunAsync();
retryContext.CurrentIteration++;
}while (retryAggregator.HasExceptions &&
retryContext.CurrentIteration < retryContext.Limit &&
(retryPredicateMethod.IsStatic
? (bool)retryPredicateMethod.Invoke(null, new object[] { retryAggregator.ToException() })
: (bool)retryPredicateMethod.Invoke(retryContext.TestClassInstance, new object[] { retryAggregator.ToException() }))
);
aggregator.Aggregate(retryAggregator);
return(totalTime);
}
public async Task <RunSummary> RunAsync()
{
if (!string.IsNullOrEmpty(this.skipReason))
{
this.messageBus.Queue(
this.scenario, test => new TestSkipped(test, this.skipReason), this.cancellationTokenSource);
return(new RunSummary {
Total = 1, Skipped = 1
});
}
else
{
var summary = new RunSummary();
var output = string.Empty;
var childAggregator = new ExceptionAggregator(this.parentAggregator);
if (!childAggregator.HasExceptions)
{
var tuple = await childAggregator.RunAsync(() => this.InvokeScenarioAsync(childAggregator));
summary.Aggregate(tuple.Item1);
output = tuple.Item2;
}
var exception = childAggregator.ToException();
if (exception != null)
{
summary.Total++;
summary.Failed++;
this.messageBus.Queue(
this.scenario,
test => new TestFailed(test, summary.Time, output, exception),
this.cancellationTokenSource);
}
else if (summary.Total == 0)
{
summary.Total++;
this.messageBus.Queue(
this.scenario, test => new TestPassed(test, summary.Time, output), this.cancellationTokenSource);
}
return(summary);
}
}
public static async Task ExecuteTestMethodAsync <T>(ScenarioReport report,
string methodName, ScenarioRunner runner = null, object[] parameters = null)
{
parameters = parameters ?? new object[] { };
var methodInfo = typeof(T).GetMethod(methodName);
if (methodInfo == null)
{
throw new InvalidOperationException($"Unknown method {methodName} on type {typeof(T)}.FullName");
}
var method = new FakeMethodInfo(Reflector.Wrap(methodInfo),
Reflector.Wrap(typeof(T)),
new [] { new FakeAttributeInfo(
parameters.Length == 0? new FactAttribute() :new TheoryAttribute(),
new object[] {},
new Dictionary <string, object>()
{
["DisplayName"] = null, ["Skip"] = null
}) });
var aggregator = new ExceptionAggregator();
var testMethod = Mock.TestMethod <T>(method);
var invoker = new ScenarioReportingTestInvoker(
scenarioRunner: runner,
report: report,
test: new XunitTest(new ScenarioReportingXunitTestCase(new NullMessageSink(), TestMethodDisplay.ClassAndMethod, testMethod, parameters), "Test"),
messageBus: new MessageBus(new NullMessageSink()),
constructorArguments: new object[] { },
testClass: typeof(T),
testMethod: method.ToRuntimeMethod(),
testMethodArguments: parameters,
beforeAfterAttributes: new BeforeAfterTestAttribute[] { },
aggregator: aggregator,
cancellationTokenSource: new CancellationTokenSource(1000)
);
await invoker.RunAsync();
if (aggregator.HasExceptions)
{
throw aggregator.ToException();
}
await report.WriteFinalAsync();
}
public VsixRunSummary Run(VsixTestCase testCase, IMessageBus messageBus)
{
messageBus.QueueMessage(new DiagnosticMessage("Running {0}", testCase.DisplayName));
var aggregator = new ExceptionAggregator();
var runner = _collectionRunnerMap.GetOrAdd(testCase.TestMethod.TestClass.TestCollection, tc => new VsRemoteTestCollectionRunner(tc, _assemblyFixtureMappings, _collectionFixtureMappings));
if (SynchronizationContext.Current == null)
{
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
}
try
{
using (var bus = new TestMessageBus(messageBus))
{
var result = runner.RunAsync(testCase, bus, aggregator)
.Result
.ToVsixRunSummary();
if (aggregator.HasExceptions && result != null)
{
result.Exception = aggregator.ToException();
}
return(result);
}
}
catch (AggregateException aex)
{
return(new VsixRunSummary
{
Failed = 1,
Exception = aex.Flatten().InnerException
});
}
}
internal static async Task <RunSummary> PerformRetry(
Func <IMessageSink,
IMessageBus,
object[],
ExceptionAggregator,
CancellationTokenSource, Task <RunSummary> > executer,
int maxRetries,
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
XunitTestCase testCase,
CancellationTokenSource cancellationTokenSource)
{
var testRetryCount = 0;
var rnd = new Random();
using var delayedMessageBus = new DelayedMessageBus(messageBus);
while (true)
{
var summary = await executer(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
var lastAssert = delayedMessageBus.LastFailure;
if (aggregator.HasExceptions || summary.Failed == 0 || testRetryCount + 1 > maxRetries)
{
delayedMessageBus.Complete();
if (testRetryCount > 0)
{
LogMessage($"test '{testCase.DisplayName}' retry finished. Number of failed tests of last run: {summary.Failed}, retry state: {testRetryCount}/{maxRetries}");
if (summary.Failed == 0)
{
LogMessage($"test '{testCase.DisplayName}' succeeded after {testRetryCount}/{maxRetries} executions.");
}
else if (testRetryCount == maxRetries)
{
LogMessage($"test '{testCase.DisplayName}' failed after {testRetryCount}/{maxRetries} executions.", lastAssert);
}
}
if (aggregator.HasExceptions)
{
LogMessage($"test '{testCase.DisplayName}' failed with exception. {aggregator.ToException()}");
}
return(summary);
}
testRetryCount++;
var retryDelay = (int)Math.Min(180_000, Math.Pow(2, testRetryCount) * rnd.Next(5000, 30_000));
var msg = $"performing retry number {testRetryCount}/{maxRetries} in {retryDelay}ms for Test '{testCase.DisplayName}'";
LogMessage(msg, lastAssert);
await Task.Delay(retryDelay, cancellationTokenSource.Token);
}
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var scenarioTypeInfo = this.scenarioClass.GetTypeInfo();
var filters = scenarioTypeInfo.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(scenarioTypeInfo.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var summary = new RunSummary();
string skipReason = null;
var scenarioTeardowns = new List <Tuple <StepContext, Func <IStepContext, Task> > >();
var stepNumber = 0;
foreach (var stepDefinition in filters.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current)))
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepDefinition.DisplayTextFunc?.Invoke(stepDefinition.Text, stepNumber <= backGroundStepDefinitions.Count));
var step = new StepTest(this.scenario, stepDisplayName);
using (var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = $"Failed to execute preceding step: {step.DisplayName}";
}
}))
{
var stepContext = new StepContext(step);
var stepRunner = new StepTestRunner(
stepContext,
stepDefinition.Body,
step,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new BeforeAfterTestAttribute[0],
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
var stepTeardowns = stepContext.Disposables
.Where(disposable => disposable != null)
.Select((Func <IDisposable, Func <IStepContext, Task> >)(disposable =>
context =>
{
disposable.Dispose();
return(Task.FromResult(0));
}))
.Concat(stepDefinition.Teardowns)
.Where(teardown => teardown != null)
.Select(teardown => Tuple.Create(stepContext, teardown));
scenarioTeardowns.AddRange(stepTeardowns);
}
}
if (scenarioTeardowns.Any())
{
scenarioTeardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in scenarioTeardowns)
{
await Invoker.Invoke(() => teardown.Item2(teardown.Item1), teardownAggregator, teardownTimer);
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(this.scenario.DisplayName, ++stepNumber, "(Teardown)");
this.messageBus.Queue(
new StepTest(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
/// <summary>
/// Invokes the Steps given Background, Scenario, and TearDown steps.
/// </summary>
/// <param name="backgroundSteps"></param>
/// <param name="scenarioSteps"></param>
/// <param name="tearDownSteps"></param>
/// <returns></returns>
private async Task <RunSummary> InvokeStepsAsync(ICollection <IStepDefinition> backgroundSteps
, ICollection <IStepDefinition> scenarioSteps, ICollection <IStepDefinition> tearDownSteps)
{
var scenarioTypeInfo = this._scenarioClass.GetTypeInfo();
var filters = scenarioTypeInfo.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(scenarioTypeInfo.GetCustomAttributes(typeof(Attribute)))
.Concat(this._scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var summary = new RunSummary();
string skipReason = null;
var scenarioRollbacks = new List <(StepContext context, Func <IStepContext, Task> callback)>();
var stepNumber = 0;
foreach (var stepDefinition in filters.Aggregate(backgroundSteps.Concat(scenarioSteps).Concat(tearDownSteps)
, (current, filter) => filter.Filter(current)))
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(this._scenario.DisplayName
, ++stepNumber
, stepDefinition.OnDisplayText?.Invoke(
stepDefinition.Text, stepDefinition.StepDefinitionType)
);
var step = new StepTest(this._scenario, stepDisplayName);
// #1 MWP 2020-07-01 11:56:45 AM: After testing, this should be fine, and better behaved we think.
using (var interceptingBus = new DelegatingMessageBus(
this._messageBus
, message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = $"Failed to execute preceding step: {step.DisplayName}";
}
})
)
{
var stepContext = new StepContext(step);
// TODO: TBD: #1 MWP 2020-07-01 11:57:26 AM: it is an xUnit thing, could possibly be IDisposable itself...
// TODO: TBD: including assumed ownership of the IDisposable messageBus, for instance, TODO: TBD: but that is outside the scope of xWellBehaved...
var stepRunner = new StepTestRunner(
stepContext
, stepDefinition.Body
, step
, interceptingBus
, this._scenarioClass
, this._constructorArguments
, this._scenarioMethod
, this._scenarioMethodArguments
, stepDefinition.SkipReason
, Array.Empty <BeforeAfterTestAttribute>()
, new ExceptionAggregator(this._aggregator)
, this._cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
// TODO: TBD: could we use disposable?.Dispose() here?
var stepRollbacks = stepContext.Disposables
.Where(disposable => disposable != null)
.Select((Func <IDisposable, Func <IStepContext, Task> >)(disposable => context =>
{
disposable.Dispose();
return(Task.FromResult(0));
}))
.Concat(stepDefinition.Rollbacks)
.Where(onRollback => onRollback != null)
.Select(onRollback => (stepContext, onRollback));
scenarioRollbacks.AddRange(stepRollbacks);
}
}
if (scenarioRollbacks.Any())
{
scenarioRollbacks.Reverse();
var rollbackTimer = new ExecutionTimer();
var rollbackAggregator = new ExceptionAggregator();
// "Teardowns" not to be confused with TearDown versus Background.
foreach (var(context, onRollback) in scenarioRollbacks)
{
await Invoker.Invoke(() => onRollback.Invoke(context), rollbackAggregator, rollbackTimer);
}
summary.Time += rollbackTimer.Total;
if (rollbackAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(this._scenario.DisplayName, ++stepNumber, $"({StepType.Rollback})");
this._messageBus.Queue(new StepTest(this._scenario, stepDisplayName)
, test => new TestFailed(test, rollbackTimer.Total, null, rollbackAggregator.ToException())
, this._cancellationTokenSource);
}
}
return(summary);
}
public async Task <RunSummary> RunScenarioAsync()
{
var runSummary = new RunSummary {
Total = 1
};
var output = string.Empty;
if (!MessageBus.QueueMessage(new TestStarting(Test)))
{
CancellationTokenSource.Cancel();
}
else
{
AfterTestStarting();
if (!string.IsNullOrEmpty(SkipReason))
{
runSummary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(Test, SkipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator));
runSummary.Time = tuple.Item1;
output = tuple.Item2;
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(Test, runSummary.Time, output);
}
else if (exception is IgnoreException)
{
testResult = new TestSkipped(Test, exception.Message);
runSummary.Skipped++;
}
else
{
testResult = new TestFailed(Test, runSummary.Time, output, exception);
runSummary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
Aggregator.Clear();
BeforeTestFinished();
if (Aggregator.HasExceptions)
{
if (!MessageBus.QueueMessage(new TestCleanupFailure(Test, Aggregator.ToException())))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(runSummary);
}
private async Task <Tuple <decimal, string> > RunTestCaseWithRetryAsync(RetryAttribute retryAttribute, ExceptionAggregator aggregator)
{
var totalTimeTaken = 0m;
List <string> messages = new();
var numAttempts = Math.Max(1, retryAttribute.MaxRetries);
for (var attempt = 1; attempt <= numAttempts; attempt++)
{
var result = await base.InvokeTestAsync(aggregator).ConfigureAwait(false);
totalTimeTaken += result.Item1;
messages.Add(result.Item2);
if (!aggregator.HasExceptions)
{
break;
}
else if (attempt < numAttempts)
{
// We can't use the ITestOutputHelper here because there's no active test
messages.Add($"[{TestCase.DisplayName}] Attempt {attempt} of {retryAttribute.MaxRetries} failed due to {aggregator.ToException()}");
await Task.Delay(5000).ConfigureAwait(false);
aggregator.Clear();
}
}
return(new(totalTimeTaken, string.Join(Environment.NewLine, messages)));
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List <Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
string output = "";
var gherkinDocument = await this.TestCase.FeatureTypeInfo.GetDocumentAsync();
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out var example, out var exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() =>
{
var stopwatch = Stopwatch.StartNew();
testOutputHelper.Initialize(MessageBus, test);
try
{
RunScenario(gherkinDocument, scenario);
}
finally
{
stopwatch.Stop();
summary.Time = (decimal)stopwatch.Elapsed.TotalSeconds;
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
}
);
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output);
}
else
{
testResult = new TestFailed(test, summary.Time, output, exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(summary);
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
var output = new StringBuilder();
var gherkinDocument = await SpecFlowParserHelper.ParseSpecFlowDocumentAsync(TestCase.FeatureFile.FeatureFilePath);
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
Examples example = null;
Gherkin.Ast.TableRow exampleRow = null;
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out example, out exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() => RunScenario(gherkinDocument, scenario, output));
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output.ToString());
}
else
{
testResult = new TestFailed(test, summary.Time, output.ToString(), exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output.ToString())))
{
CancellationTokenSource.Cancel();
}
return(summary);
}