/// <inheritdoc/> protected override Task <RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, TestCase.DisplayName); var summary = new RunSummary { Total = 1 }; if (!MessageBus.QueueMessage(new TestStarting(test))) { CancellationTokenSource.Cancel(); } else { summary.Failed = 1; var testFailed = new TestFailed(test, 0, null, new[] { typeof(InvalidOperationException).FullName }, new[] { TestCase.ErrorMessage }, new[] { "" }, new[] { -1 }); if (!MessageBus.QueueMessage(testFailed)) { CancellationTokenSource.Cancel(); } } if (!MessageBus.QueueMessage(new TestFinished(test, 0, null))) { CancellationTokenSource.Cancel(); } return(Task.FromResult(summary)); }
protected override async Task<RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, DisplayName); var aggregator = new ExceptionAggregator(Aggregator); var runner = new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, aggregator, CancellationTokenSource); return await KuduXunitTestRunnerUtils.RunTestAsync(runner, MessageBus, aggregator); }
protected override async Task<RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, DisplayName); var aggregator = new ExceptionAggregator(Aggregator); var disableRetry = ((KuduXunitTestCase)TestCase).DisableRetry; if (!disableRetry) { var value = ConfigurationManager.AppSettings["DisableRetry"]; disableRetry = string.IsNullOrEmpty(value) || bool.Parse(value); } var runner = new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, aggregator, CancellationTokenSource); return await KuduXunitTestRunnerUtils.RunTestAsync(runner, MessageBus, aggregator, disableRetry); }
protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); if (_discoverArguments) { try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(_diagnosticMessageSink, discovererAttribute); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { _toDispose.AddRange(dataRow.OfType<IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); _testRunners.Add(new BenchmarkTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync _dataDiscoveryException = ex; } } if (_testRunners.Count == 0) { var test = new XunitTest(TestCase, DisplayName); _testRunners.Add(new BenchmarkTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } }
/// <inheritdoc/> protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererType); IEnumerable <object[]> data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method); if (data == null) { Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called.")); continue; } foreach (var dataRow in data) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow); if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes); var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip"); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync dataDiscoveryException = ex; } }
/// <inheritdoc/> protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync dataDiscoveryException = ex; } }
/// <inheritdoc/> protected override Task <RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, TestCase.DisplayName); var summary = new RunSummary { Total = 1 }; var timer = new ExecutionTimer(); if (!MessageBus.QueueMessage(new TestStarting(test))) { CancellationTokenSource.Cancel(); } else { try { timer.Aggregate(TestCase.Lambda); if (!MessageBus.QueueMessage(new TestPassed(test, timer.Total, null))) { CancellationTokenSource.Cancel(); } } catch (Exception ex) { summary.Failed++; if (!MessageBus.QueueMessage(new TestFailed(test, timer.Total, null, ex))) { CancellationTokenSource.Cancel(); } } } if (!MessageBus.QueueMessage(new TestFinished(test, timer.Total, null))) { CancellationTokenSource.Cancel(); } summary.Time = timer.Total; return(Task.FromResult(summary)); }
RunSummary RunTest_DataDiscoveryException() { var test = new XunitTest(TestCase, DisplayName); if (!MessageBus.QueueMessage(new TestStarting(test))) { CancellationTokenSource.Cancel(); } else if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, dataDiscoveryException.Unwrap()))) { CancellationTokenSource.Cancel(); } if (!MessageBus.QueueMessage(new TestFinished(test, 0, null))) { CancellationTokenSource.Cancel(); } return(new RunSummary { Total = 1, Failed = 1 }); }
/// <inheritdoc/> protected override async Task AfterTestCaseStartingAsync() { await base.AfterTestCaseStartingAsync(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = SerializationHelper.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(diagnosticMessageSink, discovererType); IEnumerable<object[]> data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method); if (data == null) { Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called.")); continue; } foreach (var dataRow in data) { toDispose.AddRange(dataRow.OfType<IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow); if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes); var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); var skipReason = SkipReason ?? dataAttribute.GetNamedArgument<string>("Skip"); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { // Stash the exception so we can surface it during RunTestAsync dataDiscoveryException = ex; } }
public async Task<RunSummary> RunAsync (VsixTestCase testCase, IMessageBus messageBus, ExceptionAggregator aggregator, object[] constructorArguments) { if (Process == null) { if (!Start ()) { Stop (); if (!Start ()) { Stop (); messageBus.QueueMessage (new TestFailed (new XunitTest (testCase, testCase.DisplayName), 0, string.Format ("Failed to start Visual Studio {0}{1}.", visualStudioVersion, rootSuffix), new TimeoutException ())); return new RunSummary { Failed = 1, }; } } } if (runner == null) { var hostUrl = RemotingUtil.GetHostUri (pipeName); var clientPipeName = Guid.NewGuid ().ToString ("N"); clientChannel = RemotingUtil.CreateChannel (Constants.ClientChannelName + clientPipeName, clientPipeName); try { runner = (IVsRemoteRunner)RemotingServices.Connect (typeof (IVsRemoteRunner), hostUrl); // We don't require restart anymore since we write to the registry directly the binding paths, // rather than installing a VSIX //if (runner.ShouldRestart ()) { // Stop (); // return await RunAsync (testCase, messageBus, aggregator, constructorArguments); //} if (Debugger.IsAttached) { // Add default trace listeners to the remote process. foreach (var listener in Trace.Listeners.OfType<TraceListener> ()) { runner.AddListener (listener); } } } catch (Exception ex) { messageBus.QueueMessage (new TestFailed (new XunitTest (testCase, testCase.DisplayName), 0, ex.Message, ex)); return new RunSummary { Failed = 1 }; } } var xunitTest = new XunitTest (testCase, testCase.DisplayName); try { var outputHelper = constructorArguments.OfType<TestOutputHelper> ().FirstOrDefault (); if (outputHelper != null) outputHelper.Initialize (messageBus, xunitTest); // Special case for test output, since it's not MBR. var args = constructorArguments.Select (arg => { var helper = arg as ITestOutputHelper; if (helper != null) { var remoteHeper = new RemoteTestOutputHelper (helper); remoteObjects.Add (remoteHeper); return remoteHeper; } return arg; }).ToArray (); var remoteBus = new RemoteMessageBus (messageBus); remoteObjects.Add (remoteBus); var summary = await System.Threading.Tasks.Task.Run ( () => runner.Run (testCase, remoteBus, args)) .TimeoutAfter (testCase.TimeoutSeconds * 1000); // Dump output only if a debugger is attached, meaning that most likely // there is a single test being run/debugged. if (Debugger.IsAttached && outputHelper != null && !string.IsNullOrEmpty (outputHelper.Output)) { Trace.WriteLine (outputHelper.Output); Debugger.Log (0, "", outputHelper.Output); Console.WriteLine (outputHelper.Output); } if (summary.Exception != null) aggregator.Add (summary.Exception); return summary.ToRunSummary (); } catch (Exception ex) { aggregator.Add (ex); messageBus.QueueMessage (new TestFailed (xunitTest, 0, ex.Message, ex)); return new RunSummary { Failed = 1 }; } finally { var outputHelper = constructorArguments.OfType<TestOutputHelper> ().FirstOrDefault (); if (outputHelper != null) outputHelper.Uninitialize (); } }
/// <inheritdoc/> protected override async Task<RunSummary> RunTestAsync() { var testRunners = new List<XunitTestRunner>(); var toDispose = new List<IDisposable>(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { toDispose.AddRange(dataRow.OfType<IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { var test = new XunitTest(TestCase, DisplayName); if (!MessageBus.QueueMessage(new TestStarting(test))) CancellationTokenSource.Cancel(); else { if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, ex.Unwrap()))) CancellationTokenSource.Cancel(); } if (!MessageBus.QueueMessage(new TestFinished(test, 0, null))) CancellationTokenSource.Cancel(); return new RunSummary { Total = 1, Failed = 1 }; } var runSummary = new RunSummary(); foreach (var testRunner in testRunners) runSummary.Aggregate(await testRunner.RunAsync()); var timer = new ExecutionTimer(); var aggregator = new ExceptionAggregator(); // REVIEW: What should be done with these leftover errors? foreach (var disposable in toDispose) timer.Aggregate(() => aggregator.Run(() => disposable.Dispose())); runSummary.Time += timer.Total; return runSummary; }
public PartialTrustTestRunner(XunitTest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator exceptionAggregator, CancellationTokenSource cancellationTokenSource) : base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, exceptionAggregator, cancellationTokenSource) { }
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public override async Task<RunSummary> RunAsync #pragma warning restore CS1998 ( IMessageSink diagnosticMessageSink , IMessageBus bus , object[] constructorArguments , ExceptionAggregator aggregator , CancellationTokenSource cancellationTokenSource ) { var summary = new RunSummary(); var test = new XunitTest(this, DisplayName); if (Inv == null) { var msg = "Return Type must be Invariant."; bus.QueueMessage ( new TestFailed(test, 0, null, new Exception(msg)) ); summary.Aggregate(new RunSummary { Total = 1, Failed = 1 }); return summary; } var output = new TestOutputHelper(); var timer = new ExecutionTimer(); output.Initialize(bus, test); bus.QueueMessage(new TestStarting(test)); InvariantResult result; timer.Aggregate(() => result = Inv.Results(Config(output)).First()); var xresult = ToXunitResult(test, result, timer.Total); bus.QueueMessage(xresult.Item1); summary.Aggregate(xresult.Item2); return summary; }
private static Tuple<IMessageSinkMessage, RunSummary> ToXunitResult ( XunitTest test , InvariantResult ir , decimal time ) => ir.Passed ? Tuple.Create ( (IMessageSinkMessage) new TestPassed(test, time, ir.Message) , new RunSummary { Total = 1, Time = time } ) : ir.Exhausted ? Tuple.Create ( (IMessageSinkMessage) new TestPassed(test, time, ir.Message) , new RunSummary { Total = 1, Failed = 0, Time = time } ) : Tuple.Create ( (IMessageSinkMessage) new TestFailed(test, time, ir.Message, new Exception(ir.Message)) , new RunSummary { Total = 1, Failed = 1, Time = time } );
protected override Task<RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, DisplayName); return new PartialTrustTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, new ExceptionAggregator(Aggregator), CancellationTokenSource).RunAsync(); }
RunSummary RunTest_DataDiscoveryException() { var test = new XunitTest(TestCase, DisplayName); if (!MessageBus.QueueMessage(new TestStarting(test))) CancellationTokenSource.Cancel(); else if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, dataDiscoveryException.Unwrap()))) CancellationTokenSource.Cancel(); if (!MessageBus.QueueMessage(new TestFinished(test, 0, null))) CancellationTokenSource.Cancel(); return new RunSummary { Total = 1, Failed = 1 }; }
/// <inheritdoc/> protected override Task <RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, DisplayName); return(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, beforeAfterAttributes, new ExceptionAggregator(Aggregator), CancellationTokenSource).RunAsync()); }
/// <inheritdoc/> protected override async Task <RunSummary> RunTestAsync() { var testRunners = new List <XunitTestRunner>(); var toDispose = new List <IDisposable>(); try { var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(discovererType); foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method)) { toDispose.AddRange(dataRow.OfType <IDisposable>()); ITypeInfo[] resolvedTypes = null; var methodToRun = TestMethod; if (methodToRun.IsGenericMethodDefinition) { resolvedTypes = TypeUtility.ResolveGenericTypes(TestCase.TestMethod.Method, dataRow); methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray()); } var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray(); var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes); var theoryDisplayName = TypeUtility.GetDisplayNameWithArguments(TestCase.TestMethod.Method, DisplayName, convertedDataRow, resolvedTypes); var test = new XunitTest(TestCase, theoryDisplayName); testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource)); } } } catch (Exception ex) { var test = new XunitTest(TestCase, DisplayName); if (!MessageBus.QueueMessage(new TestStarting(test))) { CancellationTokenSource.Cancel(); } else { if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, ex.Unwrap()))) { CancellationTokenSource.Cancel(); } } if (!MessageBus.QueueMessage(new TestFinished(test, 0, null))) { CancellationTokenSource.Cancel(); } return(new RunSummary { Total = 1, Failed = 1 }); } var runSummary = new RunSummary(); foreach (var testRunner in testRunners) { runSummary.Aggregate(await testRunner.RunAsync()); } var timer = new ExecutionTimer(); var aggregator = new ExceptionAggregator(); // REVIEW: What should be done with these leftover errors? foreach (var disposable in toDispose) { timer.Aggregate(() => aggregator.Run(() => disposable.Dispose())); } runSummary.Time += timer.Total; return(runSummary); }