private void OnStatusChanged(object sender, bool started)
{
if (started)
{
return;
}
Contract.ThrowIfNull(_shutdownCancellationTokenSource);
if (_shutdownCancellationTokenSource.IsCancellationRequested)
{
lock (_gate)
{
// RemoteHost has been disabled
_remoteClientTask = null;
}
}
else
{
lock (_gate)
{
// save last remoteHostClient
s_lastRemoteClientTask = _remoteClientTask;
// save NoOpRemoteHostClient to remoteClient so that all RemoteHost call becomes
// No Op. this basically have same effect as disabling all RemoteHost features
_remoteClientTask = Task.FromResult <RemoteHostClient?>(new RemoteHostClient.NoOpClient(_workspace));
}
// s_lastRemoteClientTask info should be saved in the dump
// report NFW when connection is closed unless it is proper shutdown
FatalError.ReportWithoutCrash(new InvalidOperationException("Connection to remote host closed"));
RemoteHostCrashInfoBar.ShowInfoBar(_workspace);
}
}
public static async Task <Stream> RequestServiceAsync(
HostWorkspaceServices services,
HubClient client,
RemoteServiceName serviceName,
HostGroup hostGroup,
CancellationToken cancellationToken)
{
var is64bit = RemoteHostOptions.IsServiceHubProcess64Bit(services);
// Make sure we are on the thread pool to avoid UI thread dependencies if external code uses ConfigureAwait(true)
await TaskScheduler.Default;
var descriptor = new ServiceDescriptor(serviceName.ToString(is64bit))
{
HostGroup = hostGroup
};
try
{
return(await client.RequestServiceAsync(descriptor, cancellationToken).ConfigureAwait(false));
}
catch (Exception e) when(ReportNonFatalWatson(e, cancellationToken))
{
// TODO: Once https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1040692.
// ServiceHub may throw non-cancellation exceptions if it is called after VS started to shut down,
// even if our cancellation token is signaled. Cancel the operation and do not report an error in these cases.
//
// If ServiceHub did not throw non-cancellation exceptions when cancellation token is signaled,
// we can assume that these exceptions indicate a failure and should be reported to the user.
cancellationToken.ThrowIfCancellationRequested();
RemoteHostCrashInfoBar.ShowInfoBar(services, e);
// TODO: Propagate the original exception (see https://github.com/dotnet/roslyn/issues/40476)
throw new SoftCrashException("Unexpected exception from HubClient", e, cancellationToken);
}
private void OnUnexpectedExceptionThrown(Exception unexpectedException) => RemoteHostCrashInfoBar.ShowInfoBar(Workspace, unexpectedException);
public static async Task <RemoteHostClient?> CreateAsync(Workspace workspace, CancellationToken cancellationToken)
{
using (Logger.LogBlock(FunctionId.ServiceHubRemoteHostClient_CreateAsync, cancellationToken))
{
var timeout = TimeSpan.FromMilliseconds(workspace.Options.GetOption(RemoteHostOptions.RequestServiceTimeoutInMS));
var enableConnectionPool = workspace.Options.GetOption(RemoteHostOptions.EnableConnectionPool);
var maxConnection = workspace.Options.GetOption(RemoteHostOptions.MaxPoolConnection);
// let each client to have unique id so that we can distinguish different clients when service is restarted
var clientId = CreateClientId(Process.GetCurrentProcess().Id.ToString());
var hostGroup = new HostGroup(clientId);
var primary = new HubClient("ManagedLanguage.IDE.RemoteHostClient");
ServiceHubRemoteHostClient?client = null;
try
{
// Create the RemotableDataJsonRpc before we create the remote host: this call implicitly sets up the remote IExperimentationService so that will be available for later calls
var snapshotServiceStream = await Connections.RequestServiceAsync(workspace, primary, WellKnownServiceHubServices.SnapshotService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
var remoteHostStream = await Connections.RequestServiceAsync(workspace, primary, WellKnownRemoteHostServices.RemoteHostService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
var remotableDataRpc = new RemotableDataJsonRpc(workspace, primary.Logger, snapshotServiceStream);
var connectionManager = new ConnectionManager(primary, hostGroup, enableConnectionPool, maxConnection, timeout, new ReferenceCountedDisposable <RemotableDataJsonRpc>(remotableDataRpc));
client = new ServiceHubRemoteHostClient(workspace, primary.Logger, connectionManager, remoteHostStream);
var uiCultureLCID = CultureInfo.CurrentUICulture.LCID;
var cultureLCID = CultureInfo.CurrentCulture.LCID;
// make sure connection is done right
var host = await client._rpc.InvokeWithCancellationAsync <string>(
nameof(IRemoteHostService.Connect), new object[] { clientId, uiCultureLCID, cultureLCID, TelemetryService.DefaultSession.SerializeSettings() }, cancellationToken).ConfigureAwait(false);
client.Started();
return(client);
}
catch (ConnectionLostException ex)
{
RemoteHostCrashInfoBar.ShowInfoBar(workspace, ex);
Shutdown(ex);
// dont crash VS because OOP is failed to start. we will show info bar telling users to restart
// but never physically crash VS.
return(null);
}
catch (SoftCrashException ex)
{
Shutdown(ex);
// at this point, we should have shown info bar (RemoteHostCrashInfoBar.ShowInfoBar) to users
// returning null here will disable OOP for this VS session.
// * Note * this is not trying to recover the exception. but giving users to time
// to clean up before restart VS
return(null);
}
catch (Exception ex)
{
Shutdown(ex);
throw;
}
void Shutdown(Exception ex)
{
// make sure we shutdown client if initializing client has failed.
client?.Shutdown();
// translate to our own cancellation if it is raised.
cancellationToken.ThrowIfCancellationRequested();
// otherwise, report watson
ex.ReportServiceHubNFW("ServiceHub creation failed");
}
}
}
private void UnexpectedExceptionThrown(Exception exception) => RemoteHostCrashInfoBar.ShowInfoBar(_workspace, exception);
public static async Task <Stream> RequestServiceAsync(
Workspace workspace,
HubClient client,
string serviceName,
HostGroup hostGroup,
TimeSpan timeout,
CancellationToken cancellationToken)
{
const int max_retry = 10;
const int retry_delayInMS = 50;
RemoteInvocationException lastException = null;
var descriptor = new ServiceDescriptor(serviceName)
{
HostGroup = hostGroup
};
// call to get service can fail due to this bug - devdiv#288961 or more.
// until root cause is fixed, we decide to have retry rather than fail right away
for (var i = 0; i < max_retry; i++)
{
try
{
// we are wrapping HubClient.RequestServiceAsync since we can't control its internal timeout value ourselves.
// we have bug opened to track the issue.
// https://devdiv.visualstudio.com/DefaultCollection/DevDiv/Editor/_workitems?id=378757&fullScreen=false&_a=edit
// retry on cancellation token since HubClient will throw its own cancellation token
// when it couldn't connect to service hub service for some reasons
// (ex, OOP process GC blocked and not responding to request)
//
// we have double re-try here. we have these 2 seperated since 2 retries are for different problems.
// as noted by 2 different issues above at the start of each 2 different retries.
// first retry most likely deal with real issue on servicehub, second retry (cancellation) is to deal with
// by design servicehub behavior we don't want to use.
return(await RetryRemoteCallAsync <OperationCanceledException, Stream>(
workspace,
() => client.RequestServiceAsync(descriptor, cancellationToken),
timeout,
cancellationToken).ConfigureAwait(false));
}
catch (RemoteInvocationException ex)
{
// save info only if it failed with different issue than before.
if (lastException?.Message != ex.Message)
{
// RequestServiceAsync should never fail unless service itself is actually broken.
// So far, we catched multiple issues from this NFW. so we will keep this NFW.
ex.ReportServiceHubNFW("RequestServiceAsync Failed");
lastException = ex;
}
}
// wait for retry_delayInMS before next try
await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
}
RemoteHostCrashInfoBar.ShowInfoBar(workspace, lastException);
// raise soft crash exception rather than doing hard crash.
// we had enough feedback from users not to crash VS on servicehub failure
throw new SoftCrashException("RequestServiceAsync Failed", lastException, cancellationToken);
}
private void OnUnexpectedExceptionThrown(Exception unexpectedException) => RemoteHostCrashInfoBar.ShowInfoBar(_services, unexpectedException);
