public async Task <bool> ConnectAndPinAsync(BondCallTracker callTracker, CancellationToken cancellationToken)
{
using (await m_connectionSemaphore.AcquireAsync(cancellationToken))
{
var connection = m_connection;
if (connection == null || ShouldRecreate)
{
if (m_isDisposed)
{
return(false);
}
callTracker.OnStateChanged(BondCallState.RecreateConnection);
connection = await RecreateConnection(connection);
lock (this)
{
if (m_isDisposed)
{
connection.Dispose();
return(false);
}
m_connection = connection;
}
}
}
return(true);
}
private async Task <RpcCallResult <T> > Call <T>(
Func <TrackedConnection, BondCallTracker, Task <T> > callAsync,
CancellationToken cancellationToken,
string functionName,
BondCallTracker callTracker,
bool allowInactive = false,
Func <T, bool> shouldRetry = null,
uint maxTryCount = 0)
{
using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_cancellationTokenSource.Token))
{
return(await CallCore(callAsync, cancellationTokenSource.Token, functionName, callTracker, allowInactive, shouldRetry, maxTryCount));
}
}
/// <summary>
/// Creates a task for a bond proxy method call
/// </summary>
/// <typeparam name="TInput">the input type</typeparam>
/// <typeparam name="T">the return type</typeparam>
/// <param name="connection">the connection containing the active bond proxy</param>
/// <param name="input">the input value</param>
/// <param name="callTracker">the call tracker containing data about the call state</param>
/// <param name="cancellationToken">cancellation token used to cancel the call</param>
/// <returns>a task representing the result of the call</returns>
private async Task <T> CreateTaskForProxyCall <TInput, T>(
TrackedConnection connection,
TInput input,
BondCallTracker callTracker,
CancellationToken cancellationToken = default(CancellationToken))
where TInput : RpcMessageBase, IBondSerializable, new()
where T : IBondSerializable, new()
{
using (var bufferProvider = m_bufferManager.GetBufferProvider())
{
IBondAdaptable adaptable = input as IBondAdaptable;
if (adaptable != null)
{
callTracker.OnStateChanged(BondCallState.Converting);
adaptable.Adapt(bufferProvider);
callTracker.OnStateChanged(BondCallState.Converted);
}
// Create a cancellation token source which can be disposed after call completes.
// Need to unregister cancellation when call completes but this is difficult to do with
// the CancellationTokenRegistration returned by CancellationToken.Register since it
// is created inside the begin method delegate
using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
input.SenderName = m_thisMachineName;
input.SenderId = m_thisMachineId;
input.BuildId = m_services.BuildId;
var message = new Message <TInput>(input);
// Assign checksum so messages can be verified on receipt
m_services.AssignChecksum(message);
message.Context.PacketHeaders.m_nettrace.m_callID.SetFromSystemGuid(callTracker.CallId);
Func <AsyncCallback, object, IAsyncResult> beginMethod = (callback, state) =>
{
var asyncResult = connection.Proxy.BeginRequest(callTracker.FunctionName, message, callback, bufferProvider.Allocator);
callTracker.OnStateChanged(BondCallState.InitiatedRequest);
cancellationTokenSource.Token.Register(() => connection.Proxy.CancelRequest(callTracker.FunctionName, asyncResult), useSynchronizationContext: false);
return(asyncResult);
};
// Use the overload which takes an async callback to circumvent need to wait on wait handle in thread pool
var resultMessage = await Task.Factory.FromAsync(beginMethod, asyncResult => connection.Proxy.EndRequest <T>(callTracker.FunctionName, asyncResult), state : null);
return(resultMessage.Payload.Value);
}
}
}
/// <summary>
/// Creates a task for a bond proxy method call
/// </summary>
/// <typeparam name="TInput">the input type</typeparam>
/// <typeparam name="T">the return type</typeparam>
/// <param name="input">the input value</param>
/// <param name="callTracker">the call tracker containing data about the call state</param>
/// <param name="cancellationToken">cancellation token used to cancel the call</param>
/// <param name="maxTryCount">the maximum number of times to try the call (0 to use default)</param>
/// <returns>a task representing the result of the call</returns>
public Task <RpcCallResult <T> > Call <TInput, T>(
TInput input,
BondCallTracker callTracker,
CancellationToken cancellationToken = default(CancellationToken),
uint maxTryCount = 0)
where TInput : RpcMessageBase, IBondSerializable, new()
where T : IBondSerializable, new()
{
return(Call(
callAsync: (connection, callTracker0) =>
CreateTaskForProxyCall <TInput, T>(connection, input, callTracker0, cancellationToken),
cancellationToken: cancellationToken,
functionName: callTracker.FunctionName,
callTracker: callTracker,
maxTryCount: maxTryCount));
}
private Task <RpcCallResult <Void> > HeartbeatAsync(BondCallTracker heartbeatCallTracker, CancellationToken cancellationToken = default(CancellationToken))
{
var input = new RpcMessageBase();
return(Call(
callAsync: (connection, callTracker) =>
CreateTaskForProxyCall <RpcMessageBase, Void>(
connection,
input,
callTracker: callTracker,
cancellationToken: cancellationToken),
cancellationToken: cancellationToken,
functionName: "Heartbeat",
// Heartbeats should not wait for the connection to become active because the
// heartbeat is what determines the active state
allowInactive: true,
callTracker: heartbeatCallTracker,
// Only try once in the loop. The retry is implemented via the timer rather than in the call.
maxTryCount: 1));
}
/// <summary>
/// Waits for the connection to become active via a successful heartbeat. Heartbeats pass allowInactive to skip actually waiting.
/// </summary>
private async Task <TrackedConnectionScope> WaitForConnectionAsync(BondCallTracker callTracker, bool allowInactive, CancellationToken cancellationToken)
{
if (m_exceededInactivityTimeout)
{
return(default(TrackedConnectionScope));
}
if (!allowInactive)
{
bool isActive = await WaitForActive(cancellationToken);
if (!isActive)
{
return(default(TrackedConnectionScope));
}
}
// TODO: Should the amount of concurrency be limited?
await m_proxySemaphore.WaitAsync(cancellationToken);
// Cycle through the connections to distribute the load of the calls
var takeIndex = Interlocked.Increment(ref m_takeIndex) % m_maxConnectionConcurrency;
var connection = m_connections[takeIndex];
Contract.Assert(connection != null);
bool connected = await connection.ConnectAndPinAsync(callTracker, cancellationToken);
if (!connected)
{
return(default(TrackedConnectionScope));
}
return(new TrackedConnectionScope(connection));
}
private async Task <RpcCallResult <T> > CallCore <T>(
Func <TrackedConnection, BondCallTracker, Task <T> > callAsync,
CancellationToken cancellationToken,
string functionName,
BondCallTracker callTracker,
bool allowInactive = false,
Func <T, bool> shouldRetry = null,
uint maxTryCount = 0)
{
Contract.Requires(functionName != null);
callTracker = callTracker ?? CreateLoggingCallTracker(functionName);
TimeSpan waitForConnectionDuration = default(TimeSpan);
Failure lastFailure = null;
m_outstandingCalls.AddOrUpdate(functionName, 1, (k, i) => i + 1);
// For heartbeat only try once
if (maxTryCount == 0)
{
maxTryCount = DefaultMaxRetryCount;
}
for (uint retryCount = 0; retryCount < maxTryCount; retryCount++)
{
callTracker.TryCount = retryCount;
if (retryCount != 0)
{
// For retries, log a call start with the call tracker's updated retry count
callTracker.OnStateChanged(BondCallState.Started);
// Yield after first iteration to ensure
// we don't overflow the stack with async continuations
await Task.Yield();
}
TrackedConnection connection = null;
try
{
var startWaitForConnection = m_stopwatch.Elapsed;
callTracker.OnStateChanged(BondCallState.WaitingForConnection);
// Wait for a connection to become active via the a successful heartbeat
using (var connectionScope = await WaitForConnectionAsync(callTracker, allowInactive, cancellationToken))
{
// Log wait for connection success
var iterationWaitForConnectionDuration = GetElapsed(startWaitForConnection);
waitForConnectionDuration += iterationWaitForConnectionDuration;
callTracker.OnStateChanged(BondCallState.CompletedWaitForConnection);
// connection is not returned in the case that the proxy is shutting down or timed out
// other case is that this is a failed heartbeat call. In which case, just continue.
if (connectionScope.Connection == null)
{
if (m_isShuttingDown || m_exceededInactivityTimeout)
{
// Log the failure
lastFailure = new RecoverableExceptionFailure(new BuildXLException(m_isShuttingDown ?
"Bond RPC Call failure: Proxy is shutting down" :
"Bond RPC Call failure: Proxy timed out"));
callTracker.LogMessage("Could not retrieve connection. Failure={0}", lastFailure.DescribeIncludingInnerFailures());
callTracker.OnStateChanged(BondCallState.Failed);
return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration, lastFailure));
}
continue;
}
connection = connectionScope.Connection;
// Make the actual call
var result = await callAsync(connection, callTracker);
// Check if call should be retried
if (shouldRetry != null && shouldRetry(result))
{
continue;
}
// Log the call completion
callTracker.OnStateChanged(BondCallState.Succeeded);
m_proxyLogger.LogSuccessfulCall(m_loggingContext, functionName, retryCount);
connectionScope.MarkSucceeded();
m_services.Counters.AddToCounter(DistributionCounter.SendPipBuildRequestCallDurationMs, (long)callTracker.TotalDuration.TotalMilliseconds);
return(new RpcCallResult <T>(result, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration));
}
}
catch (OperationCanceledException)
{
callTracker.OnStateChanged(BondCallState.Canceled);
return(new RpcCallResult <T>(RpcCallResultState.Cancelled, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration));
}
catch (Exception ex)
{
// If shutting down just return the failed result
if (ex is ObjectDisposedException && m_isShuttingDown)
{
lastFailure = new RecoverableExceptionFailure(new BuildXLException("Bond RPC Call failure: Proxy is shutting down", ex));
callTracker.LogMessage("{0}", lastFailure.DescribeIncludingInnerFailures());
callTracker.OnStateChanged(BondCallState.Failed);
return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration));
}
if (DistributionServices.IsBuildIdMismatchException(ex))
{
m_proxyLogger.LogCallException(m_loggingContext, functionName, retryCount, ex);
// If a message with different build is received, it means that the sender has participated in a different distributed build.
// Then, we need to lose the connection with the sender.
OnConnectionTimeOut?.Invoke(this, EventArgs.Empty);
return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration));
}
// If not a transient exception, log and throw
if (!DistributionHelpers.IsTransientBondException(ex, m_services.Counters) && !m_services.IsChecksumMismatchException(ex))
{
m_proxyLogger.LogCallException(m_loggingContext, functionName, retryCount, ex);
throw;
}
// Otherwise, the exception is transient, so log exception and try again
lastFailure = new RecoverableExceptionFailure(new BuildXLException("Failed Bond RPC call", ex));
callTracker.LogMessage("{0}", lastFailure.DescribeIncludingInnerFailures());
// Deactivate connection so subsequent calls on the proxy will wait for heartbeat before trying to make call.
DeactivateConnection(connection);
m_services.Counters.AddToCounter(DistributionCounter.FailedSendPipBuildRequestCallDurationMs, (long)callTracker.TotalDuration.TotalMilliseconds);
m_services.Counters.IncrementCounter(DistributionCounter.FailedSendPipBuildRequestCount);
m_proxyLogger.LogFailedCall(m_loggingContext, functionName, retryCount, lastFailure);
}
finally
{
m_outstandingCalls.AddOrUpdate(functionName, 0, (k, i) => i - 1);
}
}
// Exceeded retry count.
callTracker.LogMessage("Call failed and exhausted allowed retries. LastFailure={0}", lastFailure?.DescribeIncludingInnerFailures() ?? string.Empty);
callTracker.OnStateChanged(BondCallState.Failed);
return(new RpcCallResult <T>(RpcCallResultState.Failed, DefaultMaxRetryCount, callTracker.TotalDuration, waitForConnectionDuration, lastFailure));
}
/// <summary>
/// Logs child call invocation
/// </summary>
public void LogChildTracker(BondCallTracker childTracker)
{
LogMessage("Child call initiated: #{0}", childTracker.CallId);
OnStateChanged(BondCallState.WaitingForChildCall);
}