/// <summary> /// Class constructor /// </summary> public RpcCallResult(T value, uint attempts, TimeSpan duration, TimeSpan waitForConnectionDuration) { m_value = value; State = RpcCallResultState.Succeeded; Attempts = attempts; Duration = duration; WaitForConnectionDuration = waitForConnectionDuration; }
/// <summary> /// Class constructor /// </summary> public RpcCallResult(RpcCallResultState state, uint attempts, TimeSpan duration, TimeSpan waitForConnectionDuration, Failure lastFailure = null) { Contract.Requires(state != RpcCallResultState.Succeeded); Contract.Requires(lastFailure == null || state == RpcCallResultState.Failed); State = state; Attempts = attempts; Duration = duration; m_lastFailure = lastFailure; WaitForConnectionDuration = waitForConnectionDuration; }
public async Task <RpcCallResult <Unit> > CallAsync( Func <CallOptions, AsyncUnaryCall <RpcResponse> > func, string operation, CancellationToken cancellationToken = default(CancellationToken), bool waitForConnection = false) { var watch = Stopwatch.StartNew(); TimeSpan waitForConnectionDuration = TimeSpan.Zero; TimeSpan totalCallDuration = TimeSpan.Zero; if (waitForConnection) { try { Logger.Log.GrpcTrace(m_loggingContext, $"Attempt to connect to {Channel.Target}. ChannelState {Channel.State}. Operation {operation}"); await Channel.ConnectAsync(DateTime.UtcNow.Add(GrpcSettings.InactiveTimeout)); Logger.Log.GrpcTrace(m_loggingContext, $"Connected to {Channel.Target}. Duration {watch.ElapsedMilliseconds}ms"); } catch (OperationCanceledException e) { Logger.Log.GrpcTrace(m_loggingContext, $"Failed to connect to {Channel.Target}. Duration {watch.ElapsedMilliseconds}ms. Failure {e.Message}"); return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration: watch.Elapsed)); } waitForConnectionDuration = watch.Elapsed; } Guid traceId = Guid.NewGuid(); var headers = new Metadata(); headers.Add(GrpcSettings.TraceIdKey, traceId.ToByteArray()); headers.Add(GrpcSettings.BuildIdKey, m_buildId); RpcCallResultState state = RpcCallResultState.Succeeded; Failure failure = null; uint numTry = 0; while (numTry < GrpcSettings.MaxRetry) { numTry++; watch.Restart(); try { var callOptions = new CallOptions( deadline: DateTime.UtcNow.Add(GrpcSettings.CallTimeout), cancellationToken: cancellationToken, headers: headers); Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Call", numTry, operation)); await func(callOptions); Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Sent", numTry, $"Duration: {watch.ElapsedMilliseconds}ms")); state = RpcCallResultState.Succeeded; break; } catch (RpcException e) { state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed; Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Fail", numTry, $"Duration: {watch.ElapsedMilliseconds}ms. Failure: {e.Message}")); failure = state == RpcCallResultState.Failed ? new RecoverableExceptionFailure(new BuildXLException(e.Message, e)) : null; // If the call is NOT cancelled, retry the call. if (state == RpcCallResultState.Cancelled) { break; } } finally { totalCallDuration += watch.Elapsed; } } if (state == RpcCallResultState.Succeeded) { return(new RpcCallResult <Unit>(Unit.Void, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration)); } return(new RpcCallResult <Unit>( state, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration, lastFailure: failure)); }
public async Task <RpcCallResult <Unit> > CallAsync( Func <CallOptions, AsyncUnaryCall <RpcResponse> > func, string operation, CancellationToken cancellationToken = default(CancellationToken), bool waitForConnection = false) { var watch = Stopwatch.StartNew(); TimeSpan waitForConnectionDuration = TimeSpan.Zero; TimeSpan totalCallDuration = TimeSpan.Zero; if (waitForConnection) { bool connectionSucceeded = await TryConnectChannelAsync(GrpcSettings.InactiveTimeout, operation, watch); waitForConnectionDuration = watch.Elapsed; if (!connectionSucceeded) { return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration)); } } Guid traceId = Guid.NewGuid(); var headers = new Metadata(); headers.Add(GrpcSettings.TraceIdKey, traceId.ToByteArray()); headers.Add(GrpcSettings.BuildIdKey, m_buildId); headers.Add(GrpcSettings.SenderKey, DistributionHelpers.MachineName); RpcCallResultState state = RpcCallResultState.Succeeded; Failure failure = null; uint numTry = 0; while (numTry < GrpcSettings.MaxRetry) { numTry++; watch.Restart(); try { var callOptions = new CallOptions( deadline: DateTime.UtcNow.Add(GrpcSettings.CallTimeout), cancellationToken: cancellationToken, headers: headers).WithWaitForReady(); Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Call", numTry, operation)); await func(callOptions); Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Sent", numTry, $"Duration: {watch.ElapsedMilliseconds}ms")); state = RpcCallResultState.Succeeded; break; } catch (RpcException e) { state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed; failure = state == RpcCallResultState.Failed ? new RecoverableExceptionFailure(new BuildXLException(e.Message)) : null; Logger.Log.GrpcTrace(m_loggingContext, GenerateFailLog(traceId.ToString(), numTry, watch.ElapsedMilliseconds, e.Message)); // If the call is cancelled or channel is shutdown, then do not retry the call. if (state == RpcCallResultState.Cancelled || m_isShutdownInitiated) { break; } if (numTry == GrpcSettings.MaxRetry - 1) { // If this is the last retry, try to attempt reconnecting. If the connection fails, do not attempt to retry the call. bool connectionSucceeded = await TryConnectChannelAsync(GrpcSettings.CallTimeout, operation); if (!connectionSucceeded) { break; } } } catch (ObjectDisposedException e) { state = RpcCallResultState.Failed; failure = new RecoverableExceptionFailure(new BuildXLException(e.Message)); Logger.Log.GrpcTrace(m_loggingContext, GenerateFailLog(traceId.ToString(), numTry, watch.ElapsedMilliseconds, e.Message)); // If stream is already disposed, we cannot retry call. break; } finally { totalCallDuration += watch.Elapsed; } } if (state == RpcCallResultState.Succeeded) { return(new RpcCallResult <Unit>(Unit.Void, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration)); } return(new RpcCallResult <Unit>( state, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration, lastFailure: failure)); }
public async Task <RpcCallResult <Unit> > CallAsync( Func <CallOptions, AsyncUnaryCall <RpcResponse> > func, string operationName, CancellationToken cancellationToken = default(CancellationToken), bool waitForConnection = false) { TimeSpan waitForConnectionDuration = TimeSpan.Zero; if (waitForConnection) { var waitForConnectionSw = new StopwatchVar(); using (waitForConnectionSw.Start()) { try { Logger.Log.DistributionTrace(m_loggingContext, $"Attempt to connect to '{Channel.Target}' for operation {operationName}. ChannelState '{Channel.State}'"); await Channel.ConnectAsync(DateTime.UtcNow.Add(InactiveTimeout)); Logger.Log.DistributionTrace(m_loggingContext, $"Connected to '{Channel.Target}' for operation {operationName}. Duration {waitForConnectionSw.TotalElapsed.TotalMilliseconds}ms."); } catch (OperationCanceledException e) { var duration = waitForConnectionSw.TotalElapsed; Logger.Log.DistributionTrace(m_loggingContext, $"Failed to connect to '{Channel.Target}' for operation {operationName}. Duration {duration.TotalMilliseconds}ms. Failure {e.Message}"); return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration: duration)); } } waitForConnectionDuration = waitForConnectionSw.TotalElapsed; } var callDurationSw = new StopwatchVar(); using (callDurationSw.Start()) { try { var callOptions = new CallOptions( deadline: DateTime.UtcNow.Add(CallTimeout), cancellationToken: cancellationToken); Logger.Log.DistributionTrace(m_loggingContext, $"Attempt to call '{Channel.Target}' for operation {operationName}. ChannelState '{Channel.State}'"); await func(callOptions); Logger.Log.DistributionTrace(m_loggingContext, $"Called '{Channel.Target}' for operation {operationName}. Duration {callDurationSw.TotalElapsed.TotalMilliseconds}ms."); } catch (RpcException e) { RpcCallResultState state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed; Logger.Log.DistributionTrace(m_loggingContext, $"Failed to call '{Channel.Target}' for operation {operationName}. Duration {callDurationSw.TotalElapsed.TotalMilliseconds}ms. Failure {e.Message}"); return(new RpcCallResult <Unit>( state, attempts: 1, duration: callDurationSw.TotalElapsed, waitForConnectionDuration: waitForConnectionDuration, lastFailure: state == RpcCallResultState.Failed ? new RecoverableExceptionFailure(new BuildXLException(e.Message, e)) : null)); } } return(new RpcCallResult <Unit>(Unit.Void, attempts: 1, duration: callDurationSw.TotalElapsed, waitForConnectionDuration: waitForConnectionDuration)); }