protected override async Task <RunSummary> RunTestAsync() { if (this.dataDiscoveryException != null) { this.MessageBus.Queue( new XunitTest(this.TestCase, this.DisplayName), test => new TestFailed(test, 0, null, this.dataDiscoveryException.Unwrap()), this.CancellationTokenSource); return(new RunSummary { Total = 1, Failed = 1 }); } var summary = new RunSummary(); foreach (var scenarioRunner in this.scenarioRunners) { summary.Aggregate(await scenarioRunner.RunAsync()); } // Run the cleanup here so we can include cleanup time in the run summary, // but save any exceptions so we can surface them during the cleanup phase, // so they get properly reported as test case cleanup failures. var timer = new ExecutionTimer(); foreach (var disposable in this.disposables) { timer.Aggregate(() => this.cleanupAggregator.Run(() => disposable.Dispose())); } summary.Time += timer.Total; return(summary); }
/// <summary> /// Disposes the test class instance. Sends the <see cref="ITestClassDisposeStarting"/> and <see cref="ITestClassDisposeFinished"/> /// messages as appropriate. /// </summary> /// <param name="test">The test</param> /// <param name="testClass">The test class instance to be disposed</param> /// <param name="messageBus">The message bus used to send the test messages</param> /// <param name="timer">The timer used to measure the time taken for construction</param> /// <param name="cancellationTokenSource">The cancellation token source</param> public static void DisposeTestClass(this ITest test, object testClass, IMessageBus messageBus, ExecutionTimer timer, CancellationTokenSource cancellationTokenSource) { if (!(testClass is IDisposable disposable)) { return; } if (!messageBus.QueueMessage(new TestClassDisposeStarting(test))) { cancellationTokenSource.Cancel(); } else { try { timer.Aggregate(disposable.Dispose); } finally { if (!messageBus.QueueMessage(new TestClassDisposeFinished(test))) { cancellationTokenSource.Cancel(); } } } }
protected override async Task <RunSummary> RunTestAsync() { if (_dataDiscoveryException != null) { return(RunTest_DataDiscoveryException()); } var runSummary = new RunSummary(); foreach (var testRunner in _testRunners) { runSummary.Aggregate(await testRunner.RunAsync()); } // Run the cleanup here so we can include cleanup time in the run summary, // but save any exceptions so we can surface them during the cleanup phase, // so they get properly reported as test case cleanup failures. var timer = new ExecutionTimer(); foreach (var disposable in _toDispose) { timer.Aggregate(() => _cleanupAggregator.Run(() => disposable.Dispose())); } runSummary.Time += timer.Total; return(runSummary); }
protected override async Task <RunSummary> RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, IEnumerable <ObservationTestCase> testCases) { var timer = new ExecutionTimer(); var specification = Activator.CreateInstance(testClass.Class.ToRuntimeType()) as Specification; if (specification == null) { Aggregator.Add(new InvalidOperationException(String.Format("Test class {0} cannot be static, and must derive from Specification.", testClass.Class.Name))); return(FailedSummary); } Aggregator.Run(specification.OnStart); if (Aggregator.HasExceptions) { return(FailedSummary); } var result = await new ObservationTestClassRunner(specification, testClass, @class, testCases, diagnosticMessageSink, MessageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource).RunAsync(); Aggregator.Run(specification.OnFinish); var disposable = specification as IDisposable; if (disposable != null) { timer.Aggregate(disposable.Dispose); } return(result); }
/// <summary> /// Creates an instance of the test class for the given test case. Sends the <see cref="ITestClassConstructionStarting"/> /// and <see cref="ITestClassConstructionFinished"/> messages as appropriate. /// </summary> /// <param name="test">The test</param> /// <param name="testClassType">The type of the test class</param> /// <param name="constructorArguments">The constructor arguments for the test class</param> /// <param name="messageBus">The message bus used to send the test messages</param> /// <param name="timer">The timer used to measure the time taken for construction</param> /// <param name="cancellationTokenSource">The cancellation token source</param> /// <returns></returns> public static object CreateTestClass(this ITest test, Type testClassType, object[] constructorArguments, IMessageBus messageBus, ExecutionTimer timer, CancellationTokenSource cancellationTokenSource) { object testClass = null; if (!messageBus.QueueMessage(new TestClassConstructionStarting(test))) { cancellationTokenSource.Cancel(); } else { try { if (!cancellationTokenSource.IsCancellationRequested) { timer.Aggregate(() => testClass = Activator.CreateInstance(testClassType, constructorArguments)); } } finally { if (!messageBus.QueueMessage(new TestClassConstructionFinished(test))) { cancellationTokenSource.Cancel(); } } } return(testClass); }
protected override async Task <RunSummary> RunTestAsync() { if (_dataDiscoveryException != null) { return(RunTest_DataDiscoveryException()); } var runSummary = await TaskExecutor.RunAsync( CancellationTokenSource.Token, _testRunners.Select(r => (Func <Task <RunSummary> >)r.RunScenarioAsync).ToArray(), TestCase.TestMethod.TestClass); // Run the cleanup here so we can include cleanup time in the run summary, // but save any exceptions so we can surface them during the cleanup phase, // so they get properly reported as test case cleanup failures. var timer = new ExecutionTimer(); foreach (var disposable in _toDispose) { timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose)); } runSummary.Time += timer.Total; return(runSummary); }
/// <summary> /// Disposes the test class instance. Sends the <see cref="ITestClassDisposeStarting"/> and <see cref="ITestClassDisposeFinished"/> /// messages as appropriate. /// </summary> /// <param name="testCase">The test case</param> /// <param name="testClass">The test class instance to be disposed</param> /// <param name="displayName">The display name of the test case</param> /// <param name="messageBus">The message bus used to send the test messages</param> /// <param name="timer">The timer used to measure the time taken for construction</param> /// <param name="cancellationTokenSource">The cancellation token source</param> public static void DisposeTestClass(this ITestCase testCase, object testClass, string displayName, IMessageBus messageBus, ExecutionTimer timer, CancellationTokenSource cancellationTokenSource) { var disposable = testClass as IDisposable; if (disposable == null) { return; } if (!messageBus.QueueMessage(new TestClassDisposeStarting(testCase, displayName))) { cancellationTokenSource.Cancel(); } try { timer.Aggregate(disposable.Dispose); } finally { if (!messageBus.QueueMessage(new TestClassDisposeFinished(testCase, displayName))) { cancellationTokenSource.Cancel(); } } }
public void Delayer_Should_Delay_With_The_Given_Time(int seconds) { var affector = new Delayer(TimeSpan.FromSeconds(seconds)); var t = new ExecutionTimer(); t.Aggregate(() => affector.Affect()); Assert.InRange(t.Total, seconds - 1, seconds + 1); }
public bool AfterStarting(IMessageBus messageBus, TestStarting message) { _timer = new ExecutionTimer(); var testMethod = message.TestMethod.Method.ToRuntimeMethod(); foreach (var beforeAfterTestCaseAttribute in _beforeAfterAttributes) { try { _timer.Aggregate(() => beforeAfterTestCaseAttribute.Before(testMethod)); _attributesStack.Push(beforeAfterTestCaseAttribute); } catch (Exception e) { _exceptions.Add(e); } } return(true); }
public bool BeforeFinished(IMessageBus messageBus, TestFinished message) { var testMethod = message.TestMethod.Method.ToRuntimeMethod(); foreach (BeforeCtorAfterDisposeAttribute beforeAfterTestCaseAttribute in _attributesStack) { try { _timer.Aggregate(() => beforeAfterTestCaseAttribute.After(testMethod)); } catch (Exception e) { _exceptions.Add(e); } } if (_exceptions.Any()) { return(messageBus.QueueMessage(new TestCleanupFailure(message.Test, new AggregateException(_exceptions)))); } return(true); }
public void SlowItDownBy_Should_Slow_Down_A_Method_Call() { // Arrange var affectedService = ChaosEngine.Affect <DummyService>(); affectedService .WhenCalling(x => x.RetrieveData()) .SlowItDownBy(TimeSpan.FromSeconds(5)); var service = affectedService.Instance; var result = 0; // Act var t = new ExecutionTimer(); t.Aggregate(() => { result = service.RetrieveData(); }); // Assert Assert.Equal(1, result); Assert.InRange(t.Total, 4.8m, 5.2m); }
protected override async Task <RunSummary> RunTestAsync() { if (_dataDiscoveryException != null) { return(RunTest_DataDiscoveryException()); } var runSummary = new RunSummary(); foreach (var testRunner in _testRunners) { runSummary.Aggregate(await testRunner.RunAsync()); } var timer = new ExecutionTimer(); foreach (var disposable in _toDispose) { timer.Aggregate(() => _cleanupAggregator.Run(disposable.Dispose)); } runSummary.Time += timer.Total; return(runSummary); }
/// <inheritdoc/> protected override async Task <RunSummary> RunTestAsync() { if (dataDiscoveryException != null) { return(RunTest_DataDiscoveryException()); } var runSummary = new RunSummary(); foreach (var testRunner in testRunners) { runSummary.Aggregate(await testRunner.RunAsync()); } // Run the cleanup here so we can include cleanup time in the run summary, // but save any exceptions so we can surface them during the cleanup phase, // so they get properly reported as test case cleanup failures. var timer = new ExecutionTimer(); foreach (var trackedObject in disposalTracker.TrackedObjects) { if (trackedObject is IAsyncDisposable asyncDisposable) { await timer.AggregateAsync(() => cleanupAggregator.RunAsync(asyncDisposable.DisposeAsync)); } if (trackedObject is IDisposable disposable) { timer.Aggregate(() => cleanupAggregator.Run(disposable.Dispose)); } } disposalTracker.Clear(); runSummary.Time += timer.Total; return(runSummary); }
private async Task <RunSummary> InvokeStepsAsync( ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions) { var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute)) .Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute))) .Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute))) .OfType <IFilter <IStepDefinition> >(); var stepDefinitions = filters .Aggregate( backGroundStepDefinitions.Concat(scenarioStepDefinitions), (current, filter) => filter.Filter(current)) .ToArray(); var summary = new RunSummary(); string skipReason = null; var teardowns = new List <Action>(); var stepNumber = 0; foreach (var stepDefinition in stepDefinitions) { stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason; var stepDisplayName = GetStepDisplayName( this.scenario.DisplayName, ++stepNumber, stepNumber <= backGroundStepDefinitions.Count, stepDefinition.Text, this.scenarioMethodArguments); var step = new Step(this.scenario, stepDisplayName); var interceptingBus = new DelegatingMessageBus( this.messageBus, message => { if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip) { skipReason = string.Format( CultureInfo.InvariantCulture, "Failed to execute preceding step: {0}", step.DisplayName); } }); var stepRunner = new StepRunner( step, stepDefinition.Body, interceptingBus, this.scenarioClass, this.constructorArguments, this.scenarioMethod, this.scenarioMethodArguments, stepDefinition.SkipReason, new ExceptionAggregator(this.aggregator), this.cancellationTokenSource); summary.Aggregate(await stepRunner.RunAsync()); teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose) .Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray()); } if (teardowns.Any()) { teardowns.Reverse(); var teardownTimer = new ExecutionTimer(); var teardownAggregator = new ExceptionAggregator(); foreach (var teardown in teardowns) { teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown())); } summary.Time += teardownTimer.Total; if (teardownAggregator.HasExceptions) { summary.Failed++; summary.Total++; var stepDisplayName = GetStepDisplayName( this.scenario.DisplayName, ++stepNumber, false, "(Teardown)", this.scenarioMethodArguments); this.messageBus.Queue( new Step(this.scenario, stepDisplayName), test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()), this.cancellationTokenSource); } } return(summary); }
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <WebDriverAttribute> driverAttributes, object[] dataRow) { foreach (var driverAttribute in driverAttributes) { foreach (var driver in driverAttribute.GetDrivers(runtimeMethod)) { Fixture newFixture = null; object initializerReturn = null; ITypeInfo[] resolvedTypes = null; var methodToRun = runtimeMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } List <object> parameterList = new List <object>(); var parameters = methodToRun.GetParameters().ToArray(); try { newFixture = FixtureCreationAttribute.GetNewFixture(driver, runtimeMethod); var initializeDataAttributes = ReflectionHelper.GetAttributes <FixtureInitializationAttribute>(runtimeMethod); foreach (var initializeDataAttribute in initializeDataAttributes) { if (initializeDataAttribute is IMethodInfoAware) { #if DNX var property = initializeDataAttribute.GetType().GetRuntimeProperty("Method"); property.SetValue(initializeDataAttribute, runtimeMethod); #else ((IMethodInfoAware)initializeDataAttribute).Method = runtimeMethod; #endif } initializeDataAttribute.Initialize(newFixture.Data); } var initializeAttribute = ReflectionHelper.GetAttribute <IFixtureInitializationAttribute>(runtimeMethod); if (initializeAttribute != null) { initializerReturn = initializeAttribute.Initialize(runtimeMethod, newFixture); } int dataRowIndex = 0; for (int i = 0; i < parameters.Length; i++) { var parameter = parameters[i]; var attributes = parameter.GetCustomAttributes(true); if (parameter.ParameterType == typeof(IWebDriver)) { parameterList.Add(driver); } else if (parameter.ParameterType == typeof(Fixture)) { parameterList.Add(newFixture); } else if (attributes.Any(a => a is GenerateAttribute)) { var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute); InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter); var constraintName = generateAttribute.ConstraintName ?? parameter.Name; var min = generateAttribute.Min; var max = generateAttribute.Max; var value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max }); parameterList.Add(value); } else if (attributes.Any(a => a is LocateAttribute)) { var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute); InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter); var value = locateAttribute.Value; if (value == null) { value = newFixture.Data.Generate(new SimpleFixture.DataRequest(null, newFixture.Data, parameter.ParameterType, parameter.Name, false, null, null)); } parameterList.Add(value); } else if (attributes.Any(a => a is FreezeAttribute)) { var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute); InitializeCustomAttribute(freeze, runtimeMethod, parameter); var value = freeze.Value; if (value == null) { var constraintName = freeze.ConstraintName ?? parameter.Name; var min = freeze.Min; var max = freeze.Max; value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max }); } parameterList.Add(value); object lastObject = parameterList.Last(); var closedFreezeMethod = FreezeMethod.MakeGenericMethod(lastObject.GetType()); closedFreezeMethod.Invoke(null, new object[] { newFixture.Data, value, freeze.For }); } else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType()) { parameterList.Add(initializerReturn); initializerReturn = null; } else if (dataRowIndex < dataRow.Length) { var dataValue = dataRow[dataRowIndex]; dataRowIndex++; parameterList.Add(dataValue); } else { var value = newFixture.Data.Generate(parameter.ParameterType, parameter.Name); parameterList.Add(value); } } } catch (Exception exp) { Aggregator.Add(exp); } var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray()); var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + GetDriverName(driver), dataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); var skipReason = SkipReason; XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource); runSummary.Aggregate(await testRunner.RunAsync()); var timer = new ExecutionTimer(); timer.Aggregate(() => DisposeOfData(driverAttribute, driver, newFixture, dataRow)); runSummary.Time += timer.Total; } } }
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <IApplicationAttribute> applicationAttributes, object[] dataRow) { foreach (var applicationAttribute in applicationAttributes) { Application application = null; Window mainWindow = null; WindowFixture fixture = null; try { application = applicationAttribute.ProvideApplication(runtimeMethod); application.WaitWhileBusy(); mainWindow = null; if (!string.IsNullOrEmpty(applicationAttribute.Window)) { mainWindow = application.GetWindow(applicationAttribute.Window); } else { mainWindow = application.GetWindows().FirstOrDefault(); } if (mainWindow == null) { throw new Exception("Could not locate main window " + applicationAttribute.Window); } fixture = new WindowFixture(application, mainWindow); ITypeInfo[] resolvedTypes = null; var methodToRun = runtimeMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } List <object> parameterList = new List <object>(); var parameters = methodToRun.GetParameters().ToArray(); object initializerReturn = null; int dataRowIndex = 0; for (int i = 0; i < parameters.Length; i++) { var parameter = parameters[i]; var attributes = parameter.GetCustomAttributes(true); if (parameter.ParameterType == typeof(Window)) { parameterList.Add(mainWindow); } else if (parameter.ParameterType == typeof(IWindowFixture)) { parameterList.Add(fixture); } else if (attributes.Any(a => a is GenerateAttribute)) { var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute); InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter); var constraintName = generateAttribute.ConstraintName ?? parameter.Name; var min = generateAttribute.Min; var max = generateAttribute.Max; var value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max }); parameterList.Add(value); } else if (attributes.Any(a => a is LocateAttribute)) { var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute); InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter); var value = locateAttribute.Value; if (value == null) { value = fixture.Data.Generate(new SimpleFixture.DataRequest(null, fixture.Data, parameter.ParameterType, parameter.Name, false, null, null)); } parameterList.Add(value); } else if (attributes.Any(a => a is FreezeAttribute)) { var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute); InitializeCustomAttribute(freeze, runtimeMethod, parameter); var value = freeze.Value; if (value == null) { var constraintName = freeze.ConstraintName ?? parameter.Name; var min = freeze.Min; var max = freeze.Max; value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max }); } parameterList.Add(value); object lastObject = parameterList.Last(); var closedFreezeMethod = FreezeMethod.MakeGenericMethod(lastObject.GetType()); closedFreezeMethod.Invoke(null, new object[] { fixture.Data, value, freeze.For }); } else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType()) { parameterList.Add(initializerReturn); initializerReturn = null; } else if (dataRowIndex < dataRow.Length) { var dataValue = dataRow[dataRowIndex]; dataRowIndex++; parameterList.Add(dataValue); } else { var value = fixture.Data.Generate(parameter.ParameterType, parameter.Name); parameterList.Add(value); } } var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray()); var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + applicationAttribute.Application, dataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); var skipReason = SkipReason; XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource); runSummary.Aggregate(await testRunner.RunAsync()); } catch (Exception exp) { Aggregator.Add(exp); } finally { var timer = new ExecutionTimer(); timer.Aggregate(() => DisposeOfData(application, mainWindow, fixture, dataRow)); runSummary.Time += timer.Total; } } }