private async Task <Tuple <decimal, string> > RunTestCaseWithRetryAsync(RetryAttribute retryAttribute, ExceptionAggregator aggregator) { var totalTimeTaken = 0m; List <string> messages = new(); var numAttempts = Math.Max(1, retryAttribute.MaxRetries); for (var attempt = 1; attempt <= numAttempts; attempt++) { var result = await base.InvokeTestAsync(aggregator).ConfigureAwait(false); totalTimeTaken += result.Item1; messages.Add(result.Item2); if (!aggregator.HasExceptions) { break; } else if (attempt < numAttempts) { // We can't use the ITestOutputHelper here because there's no active test messages.Add($"[{TestCase.DisplayName}] Attempt {attempt} of {retryAttribute.MaxRetries} failed due to {aggregator.ToException()}"); await Task.Delay(5000).ConfigureAwait(false); aggregator.Clear(); } } return(new(totalTimeTaken, string.Join(Environment.NewLine, messages))); }
public async Task <RunSummary> RunAsync(VsixTestCase vsixTest, IMessageBus messageBus, ExceptionAggregator aggregator) { // We don't apply retry behavior when a debugger is attached, since that // typically means the developer is actually debugging a failing test. #if !DEBUG if (Debugger.IsAttached) { return(await RunAsyncCore(vsixTest, messageBus, aggregator)); } #endif var bufferBus = new InterceptingMessageBus(); var summary = await RunAsyncCore(vsixTest, bufferBus, aggregator); var shouldRecycle = vsixTest.RecycleOnFailure.GetValueOrDefault(); // Special case for MEF cache corruption, clear cache and restart the test. if (summary.Failed != 0 && ( (aggregator.HasExceptions && aggregator.ToException().GetType().FullName == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException") || (bufferBus.Messages.OfType <IFailureInformation>().Where(fail => fail.ExceptionTypes.Any(type => type == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException")).Any()) )) { shouldRecycle = true; try { var path = VsSetup.GetComponentModelCachePath(_devEnvPath, new Version(_visualStudioVersion), _rootSuffix); if (Directory.Exists(path)) { Directory.Delete(path, true); } } catch (IOException) { s_tracer.TraceEvent(TraceEventType.Warning, 0, "Failed to clear MEF cache after a failed test caused by an InvalidMEFCacheException."); } } if (summary.Failed != 0 && shouldRecycle) { Recycle(); aggregator.Clear(); summary = await RunAsyncCore(vsixTest, messageBus, aggregator); } else { // Dispatch messages from the first run to actual bus. foreach (var msg in bufferBus.Messages) { messageBus.QueueMessage(msg); } } return(summary); }
private async Task <decimal> InvokeTestMethodAsync(ExceptionAggregator aggregator, ITestOutputHelper output) { var retryAttribute = GetRetryAttribute(TestMethod); var collectDump = TestMethod.GetCustomAttribute <CollectDumpAttribute>() != null; if (!typeof(LoggedTestBase).IsAssignableFrom(TestClass) || retryAttribute == null) { return(await new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource, output, null, collectDump).RunAsync()); } var retryPredicateMethodName = retryAttribute.RetryPredicateName; var retryPredicateMethod = TestClass.GetMethod(retryPredicateMethodName, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static, null, new Type[] { typeof(Exception) }, null) ?? throw new InvalidOperationException($"No valid static retry predicate method {retryPredicateMethodName} was found on the type {TestClass.FullName}."); if (retryPredicateMethod.ReturnType != typeof(bool)) { throw new InvalidOperationException($"Retry predicate method {retryPredicateMethodName} on {TestClass.FullName} does not return bool."); } var retryContext = new RetryContext() { Limit = retryAttribute.RetryLimit, Reason = retryAttribute.RetryReason, }; var retryAggregator = new ExceptionAggregator(); var loggedTestInvoker = new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, retryAggregator, CancellationTokenSource, output, retryContext, collectDump); var totalTime = 0.0M; do { retryAggregator.Clear(); totalTime += await loggedTestInvoker.RunAsync(); retryContext.CurrentIteration++; }while (retryAggregator.HasExceptions && retryContext.CurrentIteration < retryContext.Limit && (retryPredicateMethod.IsStatic ? (bool)retryPredicateMethod.Invoke(null, new object[] { retryAggregator.ToException() }) : (bool)retryPredicateMethod.Invoke(retryContext.TestClassInstance, new object[] { retryAggregator.ToException() })) ); aggregator.Aggregate(retryAggregator); return(totalTime); }