private bool EnsureStarted(VsixTestCase testCase, IMessageBus messageBus)
{
if (Process == null)
{
var sleep = _settings.RetrySleepInterval;
var retries = 0;
var started = false;
while (retries++ <= _settings.ProcessStartRetries && !(started = Start()))
{
Stop();
Thread.Sleep(sleep);
sleep = sleep * retries;
}
if (!started)
{
Stop();
s_tracer.TraceEvent(TraceEventType.Error, 0, Strings.VsClient.FailedToStart(_visualStudioVersion, _rootSuffix));
messageBus.QueueMessage(new TestFailed(new XunitTest(testCase, testCase.DisplayName), 0,
Strings.VsClient.FailedToStart(_visualStudioVersion, _rootSuffix),
new TimeoutException()));
return(false);
}
}
return(true);
}
private bool EnsureConnected(VsixTestCase testCase, IMessageBus messageBus)
{
if (!EnsureStarted(testCase, messageBus))
{
return(false);
}
if (_runner == null)
{
var hostUrl = RemotingUtil.GetHostUri(_pipeName);
var clientPipeName = Guid.NewGuid().ToString("N");
_clientChannel = RemotingUtil.CreateChannel(Constants.ClientChannelName + clientPipeName, clientPipeName);
_runner = (IVsRemoteRunner)RemotingServices.Connect(typeof(IVsRemoteRunner), hostUrl);
}
var retries = 0;
var connected = false;
var sleep = _settings.RetrySleepInterval;
while (retries++ <= _settings.RemoteConnectionRetries && !(connected = TryPing(_runner)))
{
Stop();
if (!EnsureStarted(testCase, messageBus))
{
return(false);
}
if (_runner == null)
{
var hostUrl = RemotingUtil.GetHostUri(_pipeName);
var clientPipeName = Guid.NewGuid().ToString("N");
_clientChannel = RemotingUtil.CreateChannel(Constants.ClientChannelName + clientPipeName, clientPipeName);
_runner = (IVsRemoteRunner)RemotingServices.Connect(typeof(IVsRemoteRunner), hostUrl);
}
Thread.Sleep(sleep);
sleep = sleep * retries;
}
if (!connected)
{
Stop();
var message = Strings.VsClient.FailedToConnect(testCase.VisualStudioVersion, testCase.RootSuffix);
messageBus.QueueMessage(new TestFailed(new XunitTest(testCase, testCase.DisplayName), 0, message, new InvalidOperationException(message)));
return(false);
}
var remoteVars = _runner.GetEnvironment();
Constants.Tracer.TraceEvent(TraceEventType.Verbose, 0, Strings.VsClient.RemoteEnvVars(string.Join(Environment.NewLine,
remoteVars.Select(x => " " + x[0] + "=" + x[1]))));
return(true);
}
public async Task <RunSummary> RunAsync(VsixTestCase vsixTest, IMessageBus messageBus, ExceptionAggregator aggregator)
{
// We don't apply retry behavior when a debugger is attached, since that
// typically means the developer is actually debugging a failing test.
#if !DEBUG
if (Debugger.IsAttached)
{
return(await RunAsyncCore(vsixTest, messageBus, aggregator));
}
#endif
var bufferBus = new InterceptingMessageBus();
var summary = await RunAsyncCore(vsixTest, bufferBus, aggregator);
var shouldRecycle = vsixTest.RecycleOnFailure.GetValueOrDefault();
// Special case for MEF cache corruption, clear cache and restart the test.
if (summary.Failed != 0 && (
(aggregator.HasExceptions && aggregator.ToException().GetType().FullName == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException") ||
(bufferBus.Messages.OfType <IFailureInformation>().Where(fail => fail.ExceptionTypes.Any(type => type == "Microsoft.VisualStudio.ExtensibilityHosting.InvalidMEFCacheException")).Any())
))
{
shouldRecycle = true;
try
{
var path = VsSetup.GetComponentModelCachePath(_devEnvPath, new Version(_visualStudioVersion), _rootSuffix);
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
}
catch (IOException)
{
s_tracer.TraceEvent(TraceEventType.Warning, 0, "Failed to clear MEF cache after a failed test caused by an InvalidMEFCacheException.");
}
}
if (summary.Failed != 0 && shouldRecycle)
{
Recycle();
aggregator.Clear();
summary = await RunAsyncCore(vsixTest, messageBus, aggregator);
}
else
{
// Dispatch messages from the first run to actual bus.
foreach (var msg in bufferBus.Messages)
{
messageBus.QueueMessage(msg);
}
}
return(summary);
}
public VsixRunSummary Run (VsixTestCase testCase, IMessageBus messageBus, object[] constructorArguments)
{
var aggregator = new ExceptionAggregator ();
var result = new XunitTestCaseRunner (
testCase, testCase.DisplayName, testCase.SkipReason, constructorArguments, testCase.TestMethodArguments, messageBus,
aggregator, new CancellationTokenSource ())
.RunAsync ()
.Result
.ToVsixRunSummary ();
if (aggregator.HasExceptions)
result.Exception = aggregator.ToException ();
return result;
}
private async Task <RunSummary> RunAsyncCore(VsixTestCase testCase, IMessageBus messageBus, ExceptionAggregator aggregator)
{
if (!EnsureConnected(testCase, messageBus))
{
return(new RunSummary
{
Failed = 1,
});
}
var xunitTest = new XunitTest(testCase, testCase.DisplayName);
try
{
var remoteBus = _remoteBuses.GetOrAdd(messageBus, bus =>
{
var instance = new RemoteMessageBus(bus);
_remoteObjects.Add(instance);
return(instance);
});
var outputBus = new TraceOutputMessageBus(remoteBus);
var summary = await Task.Run(
() => _runner.Run(testCase, outputBus))
.TimeoutAfter(testCase.TimeoutSeconds * 1000);
if (summary.Exception != null)
{
aggregator.Add(summary.Exception);
}
return(summary.ToRunSummary());
}
catch (Exception ex)
{
if (ex is RemotingException || ex is TimeoutException)
{
Stop();
}
aggregator.Add(ex);
messageBus.QueueMessage(new TestFailed(xunitTest, 0, ex.Message, ex));
return(new RunSummary
{
Failed = 1
});
}
}
public VsixRunSummary Run(VsixTestCase testCase, IMessageBus messageBus)
{
messageBus.QueueMessage(new DiagnosticMessage("Running {0}", testCase.DisplayName));
var aggregator = new ExceptionAggregator();
var runner = _collectionRunnerMap.GetOrAdd(testCase.TestMethod.TestClass.TestCollection, tc => new VsRemoteTestCollectionRunner(tc, _assemblyFixtureMappings, _collectionFixtureMappings));
if (SynchronizationContext.Current == null)
{
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
}
try
{
using (var bus = new TestMessageBus(messageBus))
{
var result = runner.RunAsync(testCase, bus, aggregator)
.Result
.ToVsixRunSummary();
if (aggregator.HasExceptions && result != null)
{
result.Exception = aggregator.ToException();
}
return(result);
}
}
catch (AggregateException aex)
{
return(new VsixRunSummary
{
Failed = 1,
Exception = aex.Flatten().InnerException
});
}
}
public async Task<RunSummary> RunAsync (VsixTestCase testCase, IMessageBus messageBus, ExceptionAggregator aggregator, object[] constructorArguments)
{
if (Process == null) {
if (!Start ()) {
Stop ();
if (!Start ()) {
Stop ();
messageBus.QueueMessage (new TestFailed (new XunitTest (testCase, testCase.DisplayName), 0,
string.Format ("Failed to start Visual Studio {0}{1}.", visualStudioVersion, rootSuffix),
new TimeoutException ()));
return new RunSummary {
Failed = 1,
};
}
}
}
if (runner == null) {
var hostUrl = RemotingUtil.GetHostUri (pipeName);
var clientPipeName = Guid.NewGuid ().ToString ("N");
clientChannel = RemotingUtil.CreateChannel (Constants.ClientChannelName + clientPipeName, clientPipeName);
try {
runner = (IVsRemoteRunner)RemotingServices.Connect (typeof (IVsRemoteRunner), hostUrl);
// We don't require restart anymore since we write to the registry directly the binding paths,
// rather than installing a VSIX
//if (runner.ShouldRestart ()) {
// Stop ();
// return await RunAsync (testCase, messageBus, aggregator, constructorArguments);
//}
if (Debugger.IsAttached) {
// Add default trace listeners to the remote process.
foreach (var listener in Trace.Listeners.OfType<TraceListener> ()) {
runner.AddListener (listener);
}
}
} catch (Exception ex) {
messageBus.QueueMessage (new TestFailed (new XunitTest (testCase, testCase.DisplayName), 0, ex.Message, ex));
return new RunSummary {
Failed = 1
};
}
}
var xunitTest = new XunitTest (testCase, testCase.DisplayName);
try {
var outputHelper = constructorArguments.OfType<TestOutputHelper> ().FirstOrDefault ();
if (outputHelper != null)
outputHelper.Initialize (messageBus, xunitTest);
// Special case for test output, since it's not MBR.
var args = constructorArguments.Select (arg => {
var helper = arg as ITestOutputHelper;
if (helper != null) {
var remoteHeper = new RemoteTestOutputHelper (helper);
remoteObjects.Add (remoteHeper);
return remoteHeper;
}
return arg;
}).ToArray ();
var remoteBus = new RemoteMessageBus (messageBus);
remoteObjects.Add (remoteBus);
var summary = await System.Threading.Tasks.Task.Run (
() => runner.Run (testCase, remoteBus, args))
.TimeoutAfter (testCase.TimeoutSeconds * 1000);
// Dump output only if a debugger is attached, meaning that most likely
// there is a single test being run/debugged.
if (Debugger.IsAttached && outputHelper != null && !string.IsNullOrEmpty (outputHelper.Output)) {
Trace.WriteLine (outputHelper.Output);
Debugger.Log (0, "", outputHelper.Output);
Console.WriteLine (outputHelper.Output);
}
if (summary.Exception != null)
aggregator.Add (summary.Exception);
return summary.ToRunSummary ();
} catch (Exception ex) {
aggregator.Add (ex);
messageBus.QueueMessage (new TestFailed (xunitTest, 0, ex.Message, ex));
return new RunSummary {
Failed = 1
};
} finally {
var outputHelper = constructorArguments.OfType<TestOutputHelper> ().FirstOrDefault ();
if (outputHelper != null)
outputHelper.Uninitialize ();
}
}