protected override async Task <RunSummary> RunTestAsync()
{
if (this.dataDiscoveryException != null)
{
this.MessageBus.Queue(
new XunitTest(this.TestCase, this.DisplayName),
test => new TestFailed(test, 0, null, this.dataDiscoveryException.Unwrap()),
this.CancellationTokenSource);
return(new RunSummary {
Total = 1, Failed = 1
});
}
var summary = new RunSummary();
foreach (var scenarioRunner in this.scenarioRunners)
{
summary.Aggregate(await scenarioRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in this.disposables)
{
timer.Aggregate(() => this.cleanupAggregator.Run(() => disposable.Dispose()));
}
summary.Time += timer.Total;
return(summary);
}
/// <summary>
/// Disposes the test class instance. Sends the <see cref="ITestClassDisposeStarting"/> and <see cref="ITestClassDisposeFinished"/>
/// messages as appropriate.
/// </summary>
/// <param name="test">The test</param>
/// <param name="testClass">The test class instance to be disposed</param>
/// <param name="messageBus">The message bus used to send the test messages</param>
/// <param name="timer">The timer used to measure the time taken for construction</param>
/// <param name="cancellationTokenSource">The cancellation token source</param>
public static void DisposeTestClass(this ITest test,
object testClass,
IMessageBus messageBus,
ExecutionTimer timer,
CancellationTokenSource cancellationTokenSource)
{
if (!(testClass is IDisposable disposable))
{
return;
}
if (!messageBus.QueueMessage(new TestClassDisposeStarting(test)))
{
cancellationTokenSource.Cancel();
}
else
{
try
{
timer.Aggregate(disposable.Dispose);
}
finally
{
if (!messageBus.QueueMessage(new TestClassDisposeFinished(test)))
{
cancellationTokenSource.Cancel();
}
}
}
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in _testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(() => disposable.Dispose()));
}
runSummary.Time += timer.Total;
return(runSummary);
}
protected override async Task <RunSummary> RunTestClassAsync(ITestClass testClass,
IReflectionTypeInfo @class,
IEnumerable <ObservationTestCase> testCases)
{
var timer = new ExecutionTimer();
var specification = Activator.CreateInstance(testClass.Class.ToRuntimeType()) as Specification;
if (specification == null)
{
Aggregator.Add(new InvalidOperationException(String.Format("Test class {0} cannot be static, and must derive from Specification.", testClass.Class.Name)));
return(FailedSummary);
}
Aggregator.Run(specification.OnStart);
if (Aggregator.HasExceptions)
{
return(FailedSummary);
}
var result = await new ObservationTestClassRunner(specification, testClass, @class, testCases, diagnosticMessageSink, MessageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource).RunAsync();
Aggregator.Run(specification.OnFinish);
var disposable = specification as IDisposable;
if (disposable != null)
{
timer.Aggregate(disposable.Dispose);
}
return(result);
}
/// <summary>
/// Creates an instance of the test class for the given test case. Sends the <see cref="ITestClassConstructionStarting"/>
/// and <see cref="ITestClassConstructionFinished"/> messages as appropriate.
/// </summary>
/// <param name="test">The test</param>
/// <param name="testClassType">The type of the test class</param>
/// <param name="constructorArguments">The constructor arguments for the test class</param>
/// <param name="messageBus">The message bus used to send the test messages</param>
/// <param name="timer">The timer used to measure the time taken for construction</param>
/// <param name="cancellationTokenSource">The cancellation token source</param>
/// <returns></returns>
public static object CreateTestClass(this ITest test,
Type testClassType,
object[] constructorArguments,
IMessageBus messageBus,
ExecutionTimer timer,
CancellationTokenSource cancellationTokenSource)
{
object testClass = null;
if (!messageBus.QueueMessage(new TestClassConstructionStarting(test)))
{
cancellationTokenSource.Cancel();
}
else
{
try
{
if (!cancellationTokenSource.IsCancellationRequested)
{
timer.Aggregate(() => testClass = Activator.CreateInstance(testClassType, constructorArguments));
}
}
finally
{
if (!messageBus.QueueMessage(new TestClassConstructionFinished(test)))
{
cancellationTokenSource.Cancel();
}
}
}
return(testClass);
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = await TaskExecutor.RunAsync(
CancellationTokenSource.Token,
_testRunners.Select(r => (Func <Task <RunSummary> >)r.RunScenarioAsync).ToArray(),
TestCase.TestMethod.TestClass);
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <summary>
/// Disposes the test class instance. Sends the <see cref="ITestClassDisposeStarting"/> and <see cref="ITestClassDisposeFinished"/>
/// messages as appropriate.
/// </summary>
/// <param name="testCase">The test case</param>
/// <param name="testClass">The test class instance to be disposed</param>
/// <param name="displayName">The display name of the test case</param>
/// <param name="messageBus">The message bus used to send the test messages</param>
/// <param name="timer">The timer used to measure the time taken for construction</param>
/// <param name="cancellationTokenSource">The cancellation token source</param>
public static void DisposeTestClass(this ITestCase testCase,
object testClass,
string displayName,
IMessageBus messageBus,
ExecutionTimer timer,
CancellationTokenSource cancellationTokenSource)
{
var disposable = testClass as IDisposable;
if (disposable == null)
{
return;
}
if (!messageBus.QueueMessage(new TestClassDisposeStarting(testCase, displayName)))
{
cancellationTokenSource.Cancel();
}
try
{
timer.Aggregate(disposable.Dispose);
}
finally
{
if (!messageBus.QueueMessage(new TestClassDisposeFinished(testCase, displayName)))
{
cancellationTokenSource.Cancel();
}
}
}
public void Delayer_Should_Delay_With_The_Given_Time(int seconds)
{
var affector = new Delayer(TimeSpan.FromSeconds(seconds));
var t = new ExecutionTimer();
t.Aggregate(() => affector.Affect());
Assert.InRange(t.Total, seconds - 1, seconds + 1);
}
public bool AfterStarting(IMessageBus messageBus, TestStarting message)
{
_timer = new ExecutionTimer();
var testMethod = message.TestMethod.Method.ToRuntimeMethod();
foreach (var beforeAfterTestCaseAttribute in _beforeAfterAttributes)
{
try
{
_timer.Aggregate(() => beforeAfterTestCaseAttribute.Before(testMethod));
_attributesStack.Push(beforeAfterTestCaseAttribute);
}
catch (Exception e)
{
_exceptions.Add(e);
}
}
return(true);
}
public bool BeforeFinished(IMessageBus messageBus, TestFinished message)
{
var testMethod = message.TestMethod.Method.ToRuntimeMethod();
foreach (BeforeCtorAfterDisposeAttribute beforeAfterTestCaseAttribute in _attributesStack)
{
try
{
_timer.Aggregate(() => beforeAfterTestCaseAttribute.After(testMethod));
}
catch (Exception e)
{
_exceptions.Add(e);
}
}
if (_exceptions.Any())
{
return(messageBus.QueueMessage(new TestCleanupFailure(message.Test, new AggregateException(_exceptions))));
}
return(true);
}
public void SlowItDownBy_Should_Slow_Down_A_Method_Call()
{
// Arrange
var affectedService = ChaosEngine.Affect <DummyService>();
affectedService
.WhenCalling(x => x.RetrieveData())
.SlowItDownBy(TimeSpan.FromSeconds(5));
var service = affectedService.Instance;
var result = 0;
// Act
var t = new ExecutionTimer();
t.Aggregate(() => {
result = service.RetrieveData();
});
// Assert
Assert.Equal(1, result);
Assert.InRange(t.Total, 4.8m, 5.2m);
}
protected override async Task <RunSummary> RunTestAsync()
{
if (_dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in _testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
var timer = new ExecutionTimer();
foreach (var disposable in _toDispose)
{
timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose));
}
runSummary.Time += timer.Total;
return(runSummary);
}
/// <inheritdoc/>
protected override async Task <RunSummary> RunTestAsync()
{
if (dataDiscoveryException != null)
{
return(RunTest_DataDiscoveryException());
}
var runSummary = new RunSummary();
foreach (var testRunner in testRunners)
{
runSummary.Aggregate(await testRunner.RunAsync());
}
// Run the cleanup here so we can include cleanup time in the run summary,
// but save any exceptions so we can surface them during the cleanup phase,
// so they get properly reported as test case cleanup failures.
var timer = new ExecutionTimer();
foreach (var trackedObject in disposalTracker.TrackedObjects)
{
if (trackedObject is IAsyncDisposable asyncDisposable)
{
await timer.AggregateAsync(() => cleanupAggregator.RunAsync(asyncDisposable.DisposeAsync));
}
if (trackedObject is IDisposable disposable)
{
timer.Aggregate(() => cleanupAggregator.Run(disposable.Dispose));
}
}
disposalTracker.Clear();
runSummary.Time += timer.Total;
return(runSummary);
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List <Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <WebDriverAttribute> driverAttributes, object[] dataRow)
{
foreach (var driverAttribute in driverAttributes)
{
foreach (var driver in driverAttribute.GetDrivers(runtimeMethod))
{
Fixture newFixture = null;
object initializerReturn = null;
ITypeInfo[] resolvedTypes = null;
var methodToRun = runtimeMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
List <object> parameterList = new List <object>();
var parameters = methodToRun.GetParameters().ToArray();
try
{
newFixture = FixtureCreationAttribute.GetNewFixture(driver, runtimeMethod);
var initializeDataAttributes = ReflectionHelper.GetAttributes <FixtureInitializationAttribute>(runtimeMethod);
foreach (var initializeDataAttribute in initializeDataAttributes)
{
if (initializeDataAttribute is IMethodInfoAware)
{
#if DNX
var property = initializeDataAttribute.GetType().GetRuntimeProperty("Method");
property.SetValue(initializeDataAttribute, runtimeMethod);
#else
((IMethodInfoAware)initializeDataAttribute).Method = runtimeMethod;
#endif
}
initializeDataAttribute.Initialize(newFixture.Data);
}
var initializeAttribute = ReflectionHelper.GetAttribute <IFixtureInitializationAttribute>(runtimeMethod);
if (initializeAttribute != null)
{
initializerReturn = initializeAttribute.Initialize(runtimeMethod, newFixture);
}
int dataRowIndex = 0;
for (int i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
var attributes = parameter.GetCustomAttributes(true);
if (parameter.ParameterType == typeof(IWebDriver))
{
parameterList.Add(driver);
}
else if (parameter.ParameterType == typeof(Fixture))
{
parameterList.Add(newFixture);
}
else if (attributes.Any(a => a is GenerateAttribute))
{
var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute);
InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter);
var constraintName = generateAttribute.ConstraintName ?? parameter.Name;
var min = generateAttribute.Min;
var max = generateAttribute.Max;
var value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
parameterList.Add(value);
}
else if (attributes.Any(a => a is LocateAttribute))
{
var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute);
InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter);
var value = locateAttribute.Value;
if (value == null)
{
value = newFixture.Data.Generate(new SimpleFixture.DataRequest(null,
newFixture.Data,
parameter.ParameterType,
parameter.Name,
false,
null,
null));
}
parameterList.Add(value);
}
else if (attributes.Any(a => a is FreezeAttribute))
{
var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute);
InitializeCustomAttribute(freeze, runtimeMethod, parameter);
var value = freeze.Value;
if (value == null)
{
var constraintName = freeze.ConstraintName ?? parameter.Name;
var min = freeze.Min;
var max = freeze.Max;
value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
}
parameterList.Add(value);
object lastObject = parameterList.Last();
var closedFreezeMethod =
FreezeMethod.MakeGenericMethod(lastObject.GetType());
closedFreezeMethod.Invoke(null, new object[] { newFixture.Data, value, freeze.For });
}
else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType())
{
parameterList.Add(initializerReturn);
initializerReturn = null;
}
else if (dataRowIndex < dataRow.Length)
{
var dataValue = dataRow[dataRowIndex];
dataRowIndex++;
parameterList.Add(dataValue);
}
else
{
var value = newFixture.Data.Generate(parameter.ParameterType, parameter.Name);
parameterList.Add(value);
}
}
}
catch (Exception exp)
{
Aggregator.Add(exp);
}
var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray());
var theoryDisplayName =
TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + GetDriverName(driver),
dataRow,
resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason;
XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource);
runSummary.Aggregate(await testRunner.RunAsync());
var timer = new ExecutionTimer();
timer.Aggregate(() => DisposeOfData(driverAttribute, driver, newFixture, dataRow));
runSummary.Time += timer.Total;
}
}
}
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <IApplicationAttribute> applicationAttributes, object[] dataRow)
{
foreach (var applicationAttribute in applicationAttributes)
{
Application application = null;
Window mainWindow = null;
WindowFixture fixture = null;
try
{
application = applicationAttribute.ProvideApplication(runtimeMethod);
application.WaitWhileBusy();
mainWindow = null;
if (!string.IsNullOrEmpty(applicationAttribute.Window))
{
mainWindow = application.GetWindow(applicationAttribute.Window);
}
else
{
mainWindow = application.GetWindows().FirstOrDefault();
}
if (mainWindow == null)
{
throw new Exception("Could not locate main window " + applicationAttribute.Window);
}
fixture = new WindowFixture(application, mainWindow);
ITypeInfo[] resolvedTypes = null;
var methodToRun = runtimeMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
List <object> parameterList = new List <object>();
var parameters = methodToRun.GetParameters().ToArray();
object initializerReturn = null;
int dataRowIndex = 0;
for (int i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
var attributes = parameter.GetCustomAttributes(true);
if (parameter.ParameterType == typeof(Window))
{
parameterList.Add(mainWindow);
}
else if (parameter.ParameterType == typeof(IWindowFixture))
{
parameterList.Add(fixture);
}
else if (attributes.Any(a => a is GenerateAttribute))
{
var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute);
InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter);
var constraintName = generateAttribute.ConstraintName ?? parameter.Name;
var min = generateAttribute.Min;
var max = generateAttribute.Max;
var value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
parameterList.Add(value);
}
else if (attributes.Any(a => a is LocateAttribute))
{
var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute);
InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter);
var value = locateAttribute.Value;
if (value == null)
{
value = fixture.Data.Generate(new SimpleFixture.DataRequest(null,
fixture.Data,
parameter.ParameterType,
parameter.Name,
false,
null,
null));
}
parameterList.Add(value);
}
else if (attributes.Any(a => a is FreezeAttribute))
{
var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute);
InitializeCustomAttribute(freeze, runtimeMethod, parameter);
var value = freeze.Value;
if (value == null)
{
var constraintName = freeze.ConstraintName ?? parameter.Name;
var min = freeze.Min;
var max = freeze.Max;
value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
}
parameterList.Add(value);
object lastObject = parameterList.Last();
var closedFreezeMethod =
FreezeMethod.MakeGenericMethod(lastObject.GetType());
closedFreezeMethod.Invoke(null, new object[] { fixture.Data, value, freeze.For });
}
else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType())
{
parameterList.Add(initializerReturn);
initializerReturn = null;
}
else if (dataRowIndex < dataRow.Length)
{
var dataValue = dataRow[dataRowIndex];
dataRowIndex++;
parameterList.Add(dataValue);
}
else
{
var value = fixture.Data.Generate(parameter.ParameterType, parameter.Name);
parameterList.Add(value);
}
}
var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray());
var theoryDisplayName =
TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + applicationAttribute.Application,
dataRow,
resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason;
XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource);
runSummary.Aggregate(await testRunner.RunAsync());
}
catch (Exception exp)
{
Aggregator.Add(exp);
}
finally
{
var timer = new ExecutionTimer();
timer.Aggregate(() => DisposeOfData(application, mainWindow, fixture, dataRow));
runSummary.Time += timer.Total;
}
}
}
