private async Task <Tuple <decimal, string> > RunTestCaseWithRetryAsync(RetryAttribute retryAttribute, ExceptionAggregator aggregator)
{
var totalTimeTaken = 0m;
List <string> messages = new();
var numAttempts = Math.Max(1, retryAttribute.MaxRetries);
for (var attempt = 1; attempt <= numAttempts; attempt++)
{
var result = await base.InvokeTestAsync(aggregator).ConfigureAwait(false);
totalTimeTaken += result.Item1;
messages.Add(result.Item2);
if (!aggregator.HasExceptions)
{
break;
}
else if (attempt < numAttempts)
{
// We can't use the ITestOutputHelper here because there's no active test
messages.Add($"[{TestCase.DisplayName}] Attempt {attempt} of {retryAttribute.MaxRetries} failed due to {aggregator.ToException()}");
await Task.Delay(5000).ConfigureAwait(false);
aggregator.Clear();
}
}
return(new(totalTimeTaken, string.Join(Environment.NewLine, messages)));
}
public async Task <RunSummary> RunAsync(VsixTestCase vsixTest, IMessageBus messageBus, ExceptionAggregator aggregator)
{
// We don't apply retry behavior when a debugger is attached, since that
// typically means the developer is actually debugging a failing test.
#if !DEBUG
if (Debugger.IsAttached)
{
return(await RunAsyncCore(vsixTest, messageBus, aggregator));
}
#endif
var bufferBus = new InterceptingMessageBus();
var summary = await RunAsyncCore(vsixTest, bufferBus, aggregator);
var shouldRecycle = vsixTest.RecycleOnFailure.GetValueOrDefault();
// Special case for MEF cache corruption, clear cache and restart the test.
if (summary.Failed != 0 && (
(aggregator.HasExceptions && aggregator.ToException().GetType().FullName == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException") ||
(bufferBus.Messages.OfType <IFailureInformation>().Where(fail => fail.ExceptionTypes.Any(type => type == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException")).Any())
))
{
shouldRecycle = true;
try
{
var path = VsSetup.GetComponentModelCachePath(_devEnvPath, new Version(_visualStudioVersion), _rootSuffix);
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
}
catch (IOException)
{
s_tracer.TraceEvent(TraceEventType.Warning, 0, "Failed to clear MEF cache after a failed test caused by an InvalidMEFCacheException.");
}
}
if (summary.Failed != 0 && shouldRecycle)
{
Recycle();
aggregator.Clear();
summary = await RunAsyncCore(vsixTest, messageBus, aggregator);
}
else
{
// Dispatch messages from the first run to actual bus.
foreach (var msg in bufferBus.Messages)
{
messageBus.QueueMessage(msg);
}
}
return(summary);
}
private async Task <decimal> InvokeTestMethodAsync(ExceptionAggregator aggregator, ITestOutputHelper output)
{
var retryAttribute = GetRetryAttribute(TestMethod);
var collectDump = TestMethod.GetCustomAttribute <CollectDumpAttribute>() != null;
if (!typeof(LoggedTestBase).IsAssignableFrom(TestClass) || retryAttribute == null)
{
return(await new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource, output, null, collectDump).RunAsync());
}
var retryPredicateMethodName = retryAttribute.RetryPredicateName;
var retryPredicateMethod = TestClass.GetMethod(retryPredicateMethodName,
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static,
null,
new Type[] { typeof(Exception) },
null)
?? throw new InvalidOperationException($"No valid static retry predicate method {retryPredicateMethodName} was found on the type {TestClass.FullName}.");
if (retryPredicateMethod.ReturnType != typeof(bool))
{
throw new InvalidOperationException($"Retry predicate method {retryPredicateMethodName} on {TestClass.FullName} does not return bool.");
}
var retryContext = new RetryContext()
{
Limit = retryAttribute.RetryLimit,
Reason = retryAttribute.RetryReason,
};
var retryAggregator = new ExceptionAggregator();
var loggedTestInvoker = new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, retryAggregator, CancellationTokenSource, output, retryContext, collectDump);
var totalTime = 0.0M;
do
{
retryAggregator.Clear();
totalTime += await loggedTestInvoker.RunAsync();
retryContext.CurrentIteration++;
}while (retryAggregator.HasExceptions &&
retryContext.CurrentIteration < retryContext.Limit &&
(retryPredicateMethod.IsStatic
? (bool)retryPredicateMethod.Invoke(null, new object[] { retryAggregator.ToException() })
: (bool)retryPredicateMethod.Invoke(retryContext.TestClassInstance, new object[] { retryAggregator.ToException() }))
);
aggregator.Aggregate(retryAggregator);
return(totalTime);
}
