private void OnStatusChanged(object sender, bool started)
{
if (started)
{
return;
}
if (_shutdownCancellationTokenSource.IsCancellationRequested)
{
lock (_gate)
{
// RemoteHost has been disabled
_remoteClientTask = null;
}
}
else
{
lock (_gate)
{
// save last remoteHostClient
s_lastRemoteClientTask = _remoteClientTask;
// save NoOpRemoteHostClient to remoteClient so that all RemoteHost call becomes
// No Op. this basically have same effect as disabling all RemoteHost features
_remoteClientTask = Task.FromResult <RemoteHostClient>(new RemoteHostClient.NoOpClient(_workspace));
}
// s_lastRemoteClientTask info should be saved in the dump
// report NFW when connection is closed unless it is proper shutdown
WatsonReporter.Report(new Exception("Connection to remote host closed"));
RemoteHostCrashInfoBar.ShowInfoBar(_workspace);
}
}
public static async Task <Stream> RequestServiceAsync(
Workspace workspace,
HubClient client,
string serviceName,
HostGroup hostGroup,
CancellationToken cancellationToken)
{
var descriptor = new ServiceDescriptor(serviceName)
{
HostGroup = hostGroup
};
try
{
return(await client.RequestServiceAsync(descriptor, cancellationToken).ConfigureAwait(false));
}
catch (Exception e) when(ReportNonFatalWatson(e, cancellationToken))
{
// TODO: Once https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1040692.
// ServiceHub may throw non-cancellation exceptions if it is called after VS started to shut down,
// even if our cancellation token is signaled. Cancel the operation and do not report an error in these cases.
//
// If ServiceHub did not throw non-cancellation exceptions when cancellation token is signaled,
// we can assume that these exceptions indicate a failure and should be reported to the user.
cancellationToken.ThrowIfCancellationRequested();
RemoteHostCrashInfoBar.ShowInfoBar(workspace, e);
// TODO: Propagate the original exception (see https://github.com/dotnet/roslyn/issues/40476)
throw new SoftCrashException("Unexpected exception from HubClient", e, cancellationToken);
}
public static async Task <Stream> RequestServiceAsync(
Workspace workspace,
HubClient client,
string serviceName,
HostGroup hostGroup,
TimeSpan timeout,
CancellationToken cancellationToken)
{
const int MaxRetryAttempts = 10;
var retryDelay = TimeSpan.FromMilliseconds(50);
Exception lastException = null;
var descriptor = new ServiceDescriptor(serviceName)
{
HostGroup = hostGroup
};
// Call to get service can fail due to this bug - devdiv#288961 or more.
// until root cause is fixed, we decide to have retry rather than fail right away
//
// We have double re-try here. We have these 2 separated since 2 retries are for different problems.
// First retry most likely deal with real issue on ServiceHub, second retry (cancellation) is to deal with
// ServiceHub behavior we don't want to use.
for (var i = 0; i < MaxRetryAttempts; i++)
{
try
{
return(await RequestServiceWithCancellationRetryAsync(
workspace,
client,
descriptor,
timeout,
cancellationToken).ConfigureAwait(false));
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
// save info only if it failed with different issue than before.
if (lastException?.Message != ex.Message)
{
// RequestServiceAsync should never fail unless service itself is actually broken.
// So far, we catched multiple issues from this NFW. so we will keep this NFW.
ex.ReportServiceHubNFW("RequestServiceAsync Failed");
lastException = ex;
}
}
// wait before next try
await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
}
RemoteHostCrashInfoBar.ShowInfoBar(workspace, lastException);
// raise soft crash exception rather than doing hard crash.
// we had enough feedback from users not to crash VS on servicehub failure
throw new SoftCrashException("RequestServiceAsync Failed", lastException, cancellationToken);
}
private void ThrowSoftCrashException(Exception ex, CancellationToken token)
{
RemoteHostCrashInfoBar.ShowInfoBar(Workspace);
// log disconnect information before throw
LogDisconnectInfo(_debuggingLastDisconnectReason, _debuggingLastDisconnectCallstack);
// throw soft crash exception
throw new SoftCrashException("remote host call failed", ex, token);
}
private static async Task <Stream> RequestServiceWithCancellationRetryAsync(
Workspace workspace,
HubClient client,
ServiceDescriptor descriptor,
TimeSpan timeout,
CancellationToken cancellationToken)
{
var retryDelay = TimeSpan.FromMilliseconds(50);
using (var pooledStopwatch = SharedPools.Default <Stopwatch>().GetPooledObject())
{
var watch = pooledStopwatch.Object;
watch.Start();
while (watch.Elapsed < timeout)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return(await client.RequestServiceAsync(descriptor, cancellationToken).ConfigureAwait(false));
}
catch (OperationCanceledException) when(!cancellationToken.IsCancellationRequested)
{
// Retry on cancellation that is not sourced by our cancellation token.
// Since HubClient will throw when it can't connect to service hub service (e.g. timeout, disposal).
}
// wait before next try
await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
// if we tried for too long and still couldn't connect, report non-fatal Watson
if (!s_timeoutReported && watch.Elapsed > s_reportTimeout)
{
s_timeoutReported = true;
// report service hub logs along with dump
new Exception("RequestServiceAsync Timeout").ReportServiceHubNFW("RequestServiceAsync Timeout");
}
}
}
// operation timed out, more than we are willing to wait
RemoteHostCrashInfoBar.ShowInfoBar(workspace);
// throw soft crash exception to minimize hard crash. it doesn't
// guarantee 100% hard crash free. but 99% it doesn't cause
// hard crash
throw new SoftCrashException("retry timed out", cancellationToken);
}
public static async Task <ServiceHubRemoteHostClient> CreateWorkerAsync(Workspace workspace, HubClient primary, TimeSpan timeout, CancellationToken cancellationToken)
{
ServiceHubRemoteHostClient client = null;
try
{
// let each client to have unique id so that we can distinguish different clients when service is restarted
var current = CreateClientId(Process.GetCurrentProcess().Id.ToString());
var hostGroup = new HostGroup(current);
// Create the RemotableDataJsonRpc before we create the remote host: this call implicitly sets up the remote IExperimentationService so that will be available for later calls
var remotableDataRpc = new RemotableDataJsonRpc(
workspace, primary.Logger,
await Connections.RequestServiceAsync(workspace, primary, WellKnownServiceHubServices.SnapshotService, hostGroup, timeout, cancellationToken).ConfigureAwait(false));
var remoteHostStream = await Connections.RequestServiceAsync(workspace, primary, WellKnownRemoteHostServices.RemoteHostService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
var enableConnectionPool = workspace.Options.GetOption(RemoteHostOptions.EnableConnectionPool);
var maxConnection = workspace.Options.GetOption(RemoteHostOptions.MaxPoolConnection);
var connectionManager = new ConnectionManager(primary, hostGroup, enableConnectionPool, maxConnection, timeout, new ReferenceCountedDisposable <RemotableDataJsonRpc>(remotableDataRpc));
client = new ServiceHubRemoteHostClient(workspace, primary.Logger, connectionManager, remoteHostStream);
var uiCultureLCID = CultureInfo.CurrentUICulture.LCID;
var cultureLCID = CultureInfo.CurrentCulture.LCID;
// make sure connection is done right
var host = await client._rpc.InvokeWithCancellationAsync <string>(
nameof(IRemoteHostService.Connect), new object[] { current, uiCultureLCID, cultureLCID, TelemetryService.DefaultSession.SerializeSettings() }, cancellationToken).ConfigureAwait(false);
return(client);
}
catch (ConnectionLostException ex)
{
RemoteHostCrashInfoBar.ShowInfoBar(workspace, ex);
Shutdown(client, ex, cancellationToken);
// dont crash VS because OOP is failed to start. we will show info bar telling users to restart
// but never physically crash VS.
throw new SoftCrashException("Connection Lost", ex, cancellationToken);
}
catch (Exception ex)
{
Shutdown(client, ex, cancellationToken);
throw;
}
/// <summary>
/// call <paramref name="funcAsync"/> and retry up to <paramref name="timeout"/> if the call throws
/// <typeparamref name="TException"/>. any other exception from the call won't be handled here.
/// </summary>
private static async Task <TResult> RetryRemoteCallAsync <TException, TResult>(
Func <Task <TResult> > funcAsync,
TimeSpan timeout,
CancellationToken cancellationToken) where TException : Exception
{
const int retry_delayInMS = 50;
using (var pooledStopwatch = SharedPools.Default <Stopwatch>().GetPooledObject())
{
var watch = pooledStopwatch.Object;
watch.Start();
while (watch.Elapsed < timeout)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return(await funcAsync().ConfigureAwait(false));
}
catch (TException)
{
// throw cancellation token if operation is cancelled
cancellationToken.ThrowIfCancellationRequested();
}
// wait for retry_delayInMS before next try
await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
ReportTimeout(watch);
}
}
// operation timed out, more than we are willing to wait
RemoteHostCrashInfoBar.ShowInfoBar();
// user didn't ask for cancellation, but we can't fullfill this request. so we
// create our own cancellation token and then throw it. this doesn't guarantee
// 100% that we won't crash, but this is at least safest way we know until user
// restart VS (with info bar)
using (var ownCancellationSource = new CancellationTokenSource())
{
ownCancellationSource.Cancel();
ownCancellationSource.Token.ThrowIfCancellationRequested();
}
throw ExceptionUtilities.Unreachable;
}
private SoftCrashException CreateSoftCrashException(Exception ex, CancellationToken cancellationToken)
{
// we are getting unexpected exception from service hub. rather than doing hard crash on unexpected exception,
// we decided to do soft crash where we show info bar to users saying "VS got corrupted and users should save
// their works and close VS"
cancellationToken.ThrowIfCancellationRequested();
LogError($"exception: {ex.ToString()}");
RemoteHostCrashInfoBar.ShowInfoBar(Workspace, ex);
// log disconnect information before throw
LogDisconnectInfo(_debuggingLastDisconnectReason, _debuggingLastDisconnectCallstack);
// throw soft crash exception
return(new SoftCrashException("remote host call failed", ex, cancellationToken));
}
private async Task RpcInvokeAsync(string targetName, params object[] arguments)
{
// handle exception gracefully. don't crash VS due to this.
// especially on shutdown time. because of pending async BG work such as
// OnGlobalOperationStarted and more, we can get into a situation where either
// we are in the middle of call when we are disconnected, or we runs
// after shutdown.
try
{
await _rpc.InvokeWithCancellationAsync(targetName, arguments?.AsArray(), _shutdownCancellationTokenSource.Token).ConfigureAwait(false);
}
catch (Exception ex) when(ReportUnlessCanceled(ex))
{
if (!_shutdownCancellationTokenSource.IsCancellationRequested)
{
RemoteHostCrashInfoBar.ShowInfoBar(Workspace, ex);
}
}
}
/// <summary>
/// call <paramref name="funcAsync"/> and retry up to <paramref name="timeout"/> if the call throws
/// <typeparamref name="TException"/>. any other exception from the call won't be handled here.
/// </summary>
public static async Task <TResult> RetryRemoteCallAsync <TException, TResult>(
Workspace workspace,
Func <Task <TResult> > funcAsync,
TimeSpan timeout,
CancellationToken cancellationToken) where TException : Exception
{
const int retry_delayInMS = 50;
using (var pooledStopwatch = SharedPools.Default <Stopwatch>().GetPooledObject())
{
var watch = pooledStopwatch.Object;
watch.Start();
while (watch.Elapsed < timeout)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
return(await funcAsync().ConfigureAwait(false));
}
catch (TException)
{
// throw cancellation token if operation is cancelled
cancellationToken.ThrowIfCancellationRequested();
}
// wait for retry_delayInMS before next try
await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
ReportTimeout(watch);
}
}
// operation timed out, more than we are willing to wait
RemoteHostCrashInfoBar.ShowInfoBar(workspace);
// throw soft crash exception to minimize hard crash. it doesn't
// gurantee 100% hard crash free. but 99% it doesn't cause
// hard crash
throw new SoftCrashException("retry timed out", cancellationToken);
}
public static async Task <Stream> RequestServiceAsync(
HostWorkspaceServices services,
HubClient client,
RemoteServiceName serviceName,
HostGroup hostGroup,
CancellationToken cancellationToken)
{
var is64bit = RemoteHostOptions.IsServiceHubProcess64Bit(services);
// Make sure we are on the thread pool to avoid UI thread dependencies if external code uses ConfigureAwait(true)
await TaskScheduler.Default;
var descriptor = new ServiceDescriptor(serviceName.ToString(is64bit))
{
HostGroup = hostGroup
};
try
{
return(await client.RequestServiceAsync(descriptor, cancellationToken).ConfigureAwait(false));
}
catch (Exception e) when(ReportNonFatalWatson(e, cancellationToken))
{
// TODO: Once https://devdiv.visualstudio.com/DevDiv/_workitems/edit/1040692.
// ServiceHub may throw non-cancellation exceptions if it is called after VS started to shut down,
// even if our cancellation token is signaled. Cancel the operation and do not report an error in these cases.
//
// If ServiceHub did not throw non-cancellation exceptions when cancellation token is signaled,
// we can assume that these exceptions indicate a failure and should be reported to the user.
cancellationToken.ThrowIfCancellationRequested();
RemoteHostCrashInfoBar.ShowInfoBar(services, e);
// TODO: Propagate the original exception (see https://github.com/dotnet/roslyn/issues/40476)
throw new SoftCrashException("Unexpected exception from HubClient", e, cancellationToken);
}
private void OnUnexpectedExceptionThrown(Exception unexpectedException) => RemoteHostCrashInfoBar.ShowInfoBar(Workspace, unexpectedException);
public static async Task <RemoteHostClient?> CreateAsync(Workspace workspace, CancellationToken cancellationToken)
{
using (Logger.LogBlock(FunctionId.ServiceHubRemoteHostClient_CreateAsync, cancellationToken))
{
var timeout = TimeSpan.FromMilliseconds(workspace.Options.GetOption(RemoteHostOptions.RequestServiceTimeoutInMS));
var enableConnectionPool = workspace.Options.GetOption(RemoteHostOptions.EnableConnectionPool);
var maxConnection = workspace.Options.GetOption(RemoteHostOptions.MaxPoolConnection);
// let each client to have unique id so that we can distinguish different clients when service is restarted
var clientId = CreateClientId(Process.GetCurrentProcess().Id.ToString());
var hostGroup = new HostGroup(clientId);
var primary = new HubClient("ManagedLanguage.IDE.RemoteHostClient");
ServiceHubRemoteHostClient?client = null;
try
{
// Create the RemotableDataJsonRpc before we create the remote host: this call implicitly sets up the remote IExperimentationService so that will be available for later calls
var snapshotServiceStream = await Connections.RequestServiceAsync(workspace, primary, WellKnownServiceHubServices.SnapshotService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
var remoteHostStream = await Connections.RequestServiceAsync(workspace, primary, WellKnownRemoteHostServices.RemoteHostService, hostGroup, timeout, cancellationToken).ConfigureAwait(false);
var remotableDataRpc = new RemotableDataJsonRpc(workspace, primary.Logger, snapshotServiceStream);
var connectionManager = new ConnectionManager(primary, hostGroup, enableConnectionPool, maxConnection, timeout, new ReferenceCountedDisposable <RemotableDataJsonRpc>(remotableDataRpc));
client = new ServiceHubRemoteHostClient(workspace, primary.Logger, connectionManager, remoteHostStream);
var uiCultureLCID = CultureInfo.CurrentUICulture.LCID;
var cultureLCID = CultureInfo.CurrentCulture.LCID;
// make sure connection is done right
var host = await client._rpc.InvokeWithCancellationAsync <string>(
nameof(IRemoteHostService.Connect), new object[] { clientId, uiCultureLCID, cultureLCID, TelemetryService.DefaultSession.SerializeSettings() }, cancellationToken).ConfigureAwait(false);
client.Started();
return(client);
}
catch (ConnectionLostException ex)
{
RemoteHostCrashInfoBar.ShowInfoBar(workspace, ex);
Shutdown(ex);
// dont crash VS because OOP is failed to start. we will show info bar telling users to restart
// but never physically crash VS.
return(null);
}
catch (SoftCrashException ex)
{
Shutdown(ex);
// at this point, we should have shown info bar (RemoteHostCrashInfoBar.ShowInfoBar) to users
// returning null here will disable OOP for this VS session.
// * Note * this is not trying to recover the exception. but giving users to time
// to clean up before restart VS
return(null);
}
catch (Exception ex)
{
Shutdown(ex);
throw;
}
void Shutdown(Exception ex)
{
// make sure we shutdown client if initializing client has failed.
client?.Shutdown();
// translate to our own cancellation if it is raised.
cancellationToken.ThrowIfCancellationRequested();
// otherwise, report watson
ex.ReportServiceHubNFW("ServiceHub creation failed");
}
}
}
private void UnexpectedExceptionThrown(Exception exception) => RemoteHostCrashInfoBar.ShowInfoBar(_workspace, exception);
public static async Task <Stream> RequestServiceAsync(
Workspace workspace,
HubClient client,
string serviceName,
HostGroup hostGroup,
TimeSpan timeout,
CancellationToken cancellationToken)
{
const int max_retry = 10;
const int retry_delayInMS = 50;
RemoteInvocationException lastException = null;
var descriptor = new ServiceDescriptor(serviceName)
{
HostGroup = hostGroup
};
// call to get service can fail due to this bug - devdiv#288961 or more.
// until root cause is fixed, we decide to have retry rather than fail right away
for (var i = 0; i < max_retry; i++)
{
try
{
// we are wrapping HubClient.RequestServiceAsync since we can't control its internal timeout value ourselves.
// we have bug opened to track the issue.
// https://devdiv.visualstudio.com/DefaultCollection/DevDiv/Editor/_workitems?id=378757&fullScreen=false&_a=edit
// retry on cancellation token since HubClient will throw its own cancellation token
// when it couldn't connect to service hub service for some reasons
// (ex, OOP process GC blocked and not responding to request)
//
// we have double re-try here. we have these 2 seperated since 2 retries are for different problems.
// as noted by 2 different issues above at the start of each 2 different retries.
// first retry most likely deal with real issue on servicehub, second retry (cancellation) is to deal with
// by design servicehub behavior we don't want to use.
return(await RetryRemoteCallAsync <OperationCanceledException, Stream>(
workspace,
() => client.RequestServiceAsync(descriptor, cancellationToken),
timeout,
cancellationToken).ConfigureAwait(false));
}
catch (RemoteInvocationException ex)
{
// save info only if it failed with different issue than before.
if (lastException?.Message != ex.Message)
{
// RequestServiceAsync should never fail unless service itself is actually broken.
// So far, we catched multiple issues from this NFW. so we will keep this NFW.
ex.ReportServiceHubNFW("RequestServiceAsync Failed");
lastException = ex;
}
}
// wait for retry_delayInMS before next try
await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
}
RemoteHostCrashInfoBar.ShowInfoBar(workspace, lastException);
// raise soft crash exception rather than doing hard crash.
// we had enough feedback from users not to crash VS on servicehub failure
throw new SoftCrashException("RequestServiceAsync Failed", lastException, cancellationToken);
}
private void OnUnexpectedExceptionThrown(Exception unexpectedException) => RemoteHostCrashInfoBar.ShowInfoBar(_services, unexpectedException);
