public async Task <bool> ConnectAndPinAsync(BondCallTracker callTracker, CancellationToken cancellationToken) { using (await m_connectionSemaphore.AcquireAsync(cancellationToken)) { var connection = m_connection; if (connection == null || ShouldRecreate) { if (m_isDisposed) { return(false); } callTracker.OnStateChanged(BondCallState.RecreateConnection); connection = await RecreateConnection(connection); lock (this) { if (m_isDisposed) { connection.Dispose(); return(false); } m_connection = connection; } } } return(true); }
private async Task <RpcCallResult <T> > Call <T>( Func <TrackedConnection, BondCallTracker, Task <T> > callAsync, CancellationToken cancellationToken, string functionName, BondCallTracker callTracker, bool allowInactive = false, Func <T, bool> shouldRetry = null, uint maxTryCount = 0) { using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_cancellationTokenSource.Token)) { return(await CallCore(callAsync, cancellationTokenSource.Token, functionName, callTracker, allowInactive, shouldRetry, maxTryCount)); } }
/// <summary> /// Creates a task for a bond proxy method call /// </summary> /// <typeparam name="TInput">the input type</typeparam> /// <typeparam name="T">the return type</typeparam> /// <param name="connection">the connection containing the active bond proxy</param> /// <param name="input">the input value</param> /// <param name="callTracker">the call tracker containing data about the call state</param> /// <param name="cancellationToken">cancellation token used to cancel the call</param> /// <returns>a task representing the result of the call</returns> private async Task <T> CreateTaskForProxyCall <TInput, T>( TrackedConnection connection, TInput input, BondCallTracker callTracker, CancellationToken cancellationToken = default(CancellationToken)) where TInput : RpcMessageBase, IBondSerializable, new() where T : IBondSerializable, new() { using (var bufferProvider = m_bufferManager.GetBufferProvider()) { IBondAdaptable adaptable = input as IBondAdaptable; if (adaptable != null) { callTracker.OnStateChanged(BondCallState.Converting); adaptable.Adapt(bufferProvider); callTracker.OnStateChanged(BondCallState.Converted); } // Create a cancellation token source which can be disposed after call completes. // Need to unregister cancellation when call completes but this is difficult to do with // the CancellationTokenRegistration returned by CancellationToken.Register since it // is created inside the begin method delegate using (var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { input.SenderName = m_thisMachineName; input.SenderId = m_thisMachineId; input.BuildId = m_services.BuildId; var message = new Message <TInput>(input); // Assign checksum so messages can be verified on receipt m_services.AssignChecksum(message); message.Context.PacketHeaders.m_nettrace.m_callID.SetFromSystemGuid(callTracker.CallId); Func <AsyncCallback, object, IAsyncResult> beginMethod = (callback, state) => { var asyncResult = connection.Proxy.BeginRequest(callTracker.FunctionName, message, callback, bufferProvider.Allocator); callTracker.OnStateChanged(BondCallState.InitiatedRequest); cancellationTokenSource.Token.Register(() => connection.Proxy.CancelRequest(callTracker.FunctionName, asyncResult), useSynchronizationContext: false); return(asyncResult); }; // Use the overload which takes an async callback to circumvent need to wait on wait handle in thread pool var resultMessage = await Task.Factory.FromAsync(beginMethod, asyncResult => connection.Proxy.EndRequest <T>(callTracker.FunctionName, asyncResult), state : null); return(resultMessage.Payload.Value); } } }
/// <summary> /// Creates a task for a bond proxy method call /// </summary> /// <typeparam name="TInput">the input type</typeparam> /// <typeparam name="T">the return type</typeparam> /// <param name="input">the input value</param> /// <param name="callTracker">the call tracker containing data about the call state</param> /// <param name="cancellationToken">cancellation token used to cancel the call</param> /// <param name="maxTryCount">the maximum number of times to try the call (0 to use default)</param> /// <returns>a task representing the result of the call</returns> public Task <RpcCallResult <T> > Call <TInput, T>( TInput input, BondCallTracker callTracker, CancellationToken cancellationToken = default(CancellationToken), uint maxTryCount = 0) where TInput : RpcMessageBase, IBondSerializable, new() where T : IBondSerializable, new() { return(Call( callAsync: (connection, callTracker0) => CreateTaskForProxyCall <TInput, T>(connection, input, callTracker0, cancellationToken), cancellationToken: cancellationToken, functionName: callTracker.FunctionName, callTracker: callTracker, maxTryCount: maxTryCount)); }
private Task <RpcCallResult <Void> > HeartbeatAsync(BondCallTracker heartbeatCallTracker, CancellationToken cancellationToken = default(CancellationToken)) { var input = new RpcMessageBase(); return(Call( callAsync: (connection, callTracker) => CreateTaskForProxyCall <RpcMessageBase, Void>( connection, input, callTracker: callTracker, cancellationToken: cancellationToken), cancellationToken: cancellationToken, functionName: "Heartbeat", // Heartbeats should not wait for the connection to become active because the // heartbeat is what determines the active state allowInactive: true, callTracker: heartbeatCallTracker, // Only try once in the loop. The retry is implemented via the timer rather than in the call. maxTryCount: 1)); }
/// <summary> /// Waits for the connection to become active via a successful heartbeat. Heartbeats pass allowInactive to skip actually waiting. /// </summary> private async Task <TrackedConnectionScope> WaitForConnectionAsync(BondCallTracker callTracker, bool allowInactive, CancellationToken cancellationToken) { if (m_exceededInactivityTimeout) { return(default(TrackedConnectionScope)); } if (!allowInactive) { bool isActive = await WaitForActive(cancellationToken); if (!isActive) { return(default(TrackedConnectionScope)); } } // TODO: Should the amount of concurrency be limited? await m_proxySemaphore.WaitAsync(cancellationToken); // Cycle through the connections to distribute the load of the calls var takeIndex = Interlocked.Increment(ref m_takeIndex) % m_maxConnectionConcurrency; var connection = m_connections[takeIndex]; Contract.Assert(connection != null); bool connected = await connection.ConnectAndPinAsync(callTracker, cancellationToken); if (!connected) { return(default(TrackedConnectionScope)); } return(new TrackedConnectionScope(connection)); }
private async Task <RpcCallResult <T> > CallCore <T>( Func <TrackedConnection, BondCallTracker, Task <T> > callAsync, CancellationToken cancellationToken, string functionName, BondCallTracker callTracker, bool allowInactive = false, Func <T, bool> shouldRetry = null, uint maxTryCount = 0) { Contract.Requires(functionName != null); callTracker = callTracker ?? CreateLoggingCallTracker(functionName); TimeSpan waitForConnectionDuration = default(TimeSpan); Failure lastFailure = null; m_outstandingCalls.AddOrUpdate(functionName, 1, (k, i) => i + 1); // For heartbeat only try once if (maxTryCount == 0) { maxTryCount = DefaultMaxRetryCount; } for (uint retryCount = 0; retryCount < maxTryCount; retryCount++) { callTracker.TryCount = retryCount; if (retryCount != 0) { // For retries, log a call start with the call tracker's updated retry count callTracker.OnStateChanged(BondCallState.Started); // Yield after first iteration to ensure // we don't overflow the stack with async continuations await Task.Yield(); } TrackedConnection connection = null; try { var startWaitForConnection = m_stopwatch.Elapsed; callTracker.OnStateChanged(BondCallState.WaitingForConnection); // Wait for a connection to become active via the a successful heartbeat using (var connectionScope = await WaitForConnectionAsync(callTracker, allowInactive, cancellationToken)) { // Log wait for connection success var iterationWaitForConnectionDuration = GetElapsed(startWaitForConnection); waitForConnectionDuration += iterationWaitForConnectionDuration; callTracker.OnStateChanged(BondCallState.CompletedWaitForConnection); // connection is not returned in the case that the proxy is shutting down or timed out // other case is that this is a failed heartbeat call. In which case, just continue. if (connectionScope.Connection == null) { if (m_isShuttingDown || m_exceededInactivityTimeout) { // Log the failure lastFailure = new RecoverableExceptionFailure(new BuildXLException(m_isShuttingDown ? "Bond RPC Call failure: Proxy is shutting down" : "Bond RPC Call failure: Proxy timed out")); callTracker.LogMessage("Could not retrieve connection. Failure={0}", lastFailure.DescribeIncludingInnerFailures()); callTracker.OnStateChanged(BondCallState.Failed); return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration, lastFailure)); } continue; } connection = connectionScope.Connection; // Make the actual call var result = await callAsync(connection, callTracker); // Check if call should be retried if (shouldRetry != null && shouldRetry(result)) { continue; } // Log the call completion callTracker.OnStateChanged(BondCallState.Succeeded); m_proxyLogger.LogSuccessfulCall(m_loggingContext, functionName, retryCount); connectionScope.MarkSucceeded(); m_services.Counters.AddToCounter(DistributionCounter.SendPipBuildRequestCallDurationMs, (long)callTracker.TotalDuration.TotalMilliseconds); return(new RpcCallResult <T>(result, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration)); } } catch (OperationCanceledException) { callTracker.OnStateChanged(BondCallState.Canceled); return(new RpcCallResult <T>(RpcCallResultState.Cancelled, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration)); } catch (Exception ex) { // If shutting down just return the failed result if (ex is ObjectDisposedException && m_isShuttingDown) { lastFailure = new RecoverableExceptionFailure(new BuildXLException("Bond RPC Call failure: Proxy is shutting down", ex)); callTracker.LogMessage("{0}", lastFailure.DescribeIncludingInnerFailures()); callTracker.OnStateChanged(BondCallState.Failed); return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration)); } if (DistributionServices.IsBuildIdMismatchException(ex)) { m_proxyLogger.LogCallException(m_loggingContext, functionName, retryCount, ex); // If a message with different build is received, it means that the sender has participated in a different distributed build. // Then, we need to lose the connection with the sender. OnConnectionTimeOut?.Invoke(this, EventArgs.Empty); return(new RpcCallResult <T>(RpcCallResultState.Failed, retryCount + 1, callTracker.TotalDuration, waitForConnectionDuration)); } // If not a transient exception, log and throw if (!DistributionHelpers.IsTransientBondException(ex, m_services.Counters) && !m_services.IsChecksumMismatchException(ex)) { m_proxyLogger.LogCallException(m_loggingContext, functionName, retryCount, ex); throw; } // Otherwise, the exception is transient, so log exception and try again lastFailure = new RecoverableExceptionFailure(new BuildXLException("Failed Bond RPC call", ex)); callTracker.LogMessage("{0}", lastFailure.DescribeIncludingInnerFailures()); // Deactivate connection so subsequent calls on the proxy will wait for heartbeat before trying to make call. DeactivateConnection(connection); m_services.Counters.AddToCounter(DistributionCounter.FailedSendPipBuildRequestCallDurationMs, (long)callTracker.TotalDuration.TotalMilliseconds); m_services.Counters.IncrementCounter(DistributionCounter.FailedSendPipBuildRequestCount); m_proxyLogger.LogFailedCall(m_loggingContext, functionName, retryCount, lastFailure); } finally { m_outstandingCalls.AddOrUpdate(functionName, 0, (k, i) => i - 1); } } // Exceeded retry count. callTracker.LogMessage("Call failed and exhausted allowed retries. LastFailure={0}", lastFailure?.DescribeIncludingInnerFailures() ?? string.Empty); callTracker.OnStateChanged(BondCallState.Failed); return(new RpcCallResult <T>(RpcCallResultState.Failed, DefaultMaxRetryCount, callTracker.TotalDuration, waitForConnectionDuration, lastFailure)); }
/// <summary> /// Logs child call invocation /// </summary> public void LogChildTracker(BondCallTracker childTracker) { LogMessage("Child call initiated: #{0}", childTracker.CallId); OnStateChanged(BondCallState.WaitingForChildCall); }