/// <summary>
/// Class constructor
/// </summary>
public RpcCallResult(T value, uint attempts, TimeSpan duration, TimeSpan waitForConnectionDuration)
{
m_value = value;
State = RpcCallResultState.Succeeded;
Attempts = attempts;
Duration = duration;
WaitForConnectionDuration = waitForConnectionDuration;
}
/// <summary>
/// Class constructor
/// </summary>
public RpcCallResult(RpcCallResultState state, uint attempts, TimeSpan duration, TimeSpan waitForConnectionDuration, Failure lastFailure = null)
{
Contract.Requires(state != RpcCallResultState.Succeeded);
Contract.Requires(lastFailure == null || state == RpcCallResultState.Failed);
State = state;
Attempts = attempts;
Duration = duration;
m_lastFailure = lastFailure;
WaitForConnectionDuration = waitForConnectionDuration;
}
public async Task <RpcCallResult <Unit> > CallAsync(
Func <CallOptions, AsyncUnaryCall <RpcResponse> > func,
string operation,
CancellationToken cancellationToken = default(CancellationToken),
bool waitForConnection = false)
{
var watch = Stopwatch.StartNew();
TimeSpan waitForConnectionDuration = TimeSpan.Zero;
TimeSpan totalCallDuration = TimeSpan.Zero;
if (waitForConnection)
{
try
{
Logger.Log.GrpcTrace(m_loggingContext, $"Attempt to connect to {Channel.Target}. ChannelState {Channel.State}. Operation {operation}");
await Channel.ConnectAsync(DateTime.UtcNow.Add(GrpcSettings.InactiveTimeout));
Logger.Log.GrpcTrace(m_loggingContext, $"Connected to {Channel.Target}. Duration {watch.ElapsedMilliseconds}ms");
}
catch (OperationCanceledException e)
{
Logger.Log.GrpcTrace(m_loggingContext, $"Failed to connect to {Channel.Target}. Duration {watch.ElapsedMilliseconds}ms. Failure {e.Message}");
return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration: watch.Elapsed));
}
waitForConnectionDuration = watch.Elapsed;
}
Guid traceId = Guid.NewGuid();
var headers = new Metadata();
headers.Add(GrpcSettings.TraceIdKey, traceId.ToByteArray());
headers.Add(GrpcSettings.BuildIdKey, m_buildId);
RpcCallResultState state = RpcCallResultState.Succeeded;
Failure failure = null;
uint numTry = 0;
while (numTry < GrpcSettings.MaxRetry)
{
numTry++;
watch.Restart();
try
{
var callOptions = new CallOptions(
deadline: DateTime.UtcNow.Add(GrpcSettings.CallTimeout),
cancellationToken: cancellationToken,
headers: headers);
Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Call", numTry, operation));
await func(callOptions);
Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Sent", numTry, $"Duration: {watch.ElapsedMilliseconds}ms"));
state = RpcCallResultState.Succeeded;
break;
}
catch (RpcException e)
{
state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed;
Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Fail", numTry, $"Duration: {watch.ElapsedMilliseconds}ms. Failure: {e.Message}"));
failure = state == RpcCallResultState.Failed ? new RecoverableExceptionFailure(new BuildXLException(e.Message, e)) : null;
// If the call is NOT cancelled, retry the call.
if (state == RpcCallResultState.Cancelled)
{
break;
}
}
finally
{
totalCallDuration += watch.Elapsed;
}
}
if (state == RpcCallResultState.Succeeded)
{
return(new RpcCallResult <Unit>(Unit.Void, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration));
}
return(new RpcCallResult <Unit>(
state,
attempts: numTry,
duration: totalCallDuration,
waitForConnectionDuration: waitForConnectionDuration,
lastFailure: failure));
}
public async Task <RpcCallResult <Unit> > CallAsync(
Func <CallOptions, AsyncUnaryCall <RpcResponse> > func,
string operation,
CancellationToken cancellationToken = default(CancellationToken),
bool waitForConnection = false)
{
var watch = Stopwatch.StartNew();
TimeSpan waitForConnectionDuration = TimeSpan.Zero;
TimeSpan totalCallDuration = TimeSpan.Zero;
if (waitForConnection)
{
bool connectionSucceeded = await TryConnectChannelAsync(GrpcSettings.InactiveTimeout, operation, watch);
waitForConnectionDuration = watch.Elapsed;
if (!connectionSucceeded)
{
return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration));
}
}
Guid traceId = Guid.NewGuid();
var headers = new Metadata();
headers.Add(GrpcSettings.TraceIdKey, traceId.ToByteArray());
headers.Add(GrpcSettings.BuildIdKey, m_buildId);
headers.Add(GrpcSettings.SenderKey, DistributionHelpers.MachineName);
RpcCallResultState state = RpcCallResultState.Succeeded;
Failure failure = null;
uint numTry = 0;
while (numTry < GrpcSettings.MaxRetry)
{
numTry++;
watch.Restart();
try
{
var callOptions = new CallOptions(
deadline: DateTime.UtcNow.Add(GrpcSettings.CallTimeout),
cancellationToken: cancellationToken,
headers: headers).WithWaitForReady();
Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Call", numTry, operation));
await func(callOptions);
Logger.Log.GrpcTrace(m_loggingContext, GenerateLog(traceId.ToString(), "Sent", numTry, $"Duration: {watch.ElapsedMilliseconds}ms"));
state = RpcCallResultState.Succeeded;
break;
}
catch (RpcException e)
{
state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed;
failure = state == RpcCallResultState.Failed ? new RecoverableExceptionFailure(new BuildXLException(e.Message)) : null;
Logger.Log.GrpcTrace(m_loggingContext, GenerateFailLog(traceId.ToString(), numTry, watch.ElapsedMilliseconds, e.Message));
// If the call is cancelled or channel is shutdown, then do not retry the call.
if (state == RpcCallResultState.Cancelled || m_isShutdownInitiated)
{
break;
}
if (numTry == GrpcSettings.MaxRetry - 1)
{
// If this is the last retry, try to attempt reconnecting. If the connection fails, do not attempt to retry the call.
bool connectionSucceeded = await TryConnectChannelAsync(GrpcSettings.CallTimeout, operation);
if (!connectionSucceeded)
{
break;
}
}
}
catch (ObjectDisposedException e)
{
state = RpcCallResultState.Failed;
failure = new RecoverableExceptionFailure(new BuildXLException(e.Message));
Logger.Log.GrpcTrace(m_loggingContext, GenerateFailLog(traceId.ToString(), numTry, watch.ElapsedMilliseconds, e.Message));
// If stream is already disposed, we cannot retry call.
break;
}
finally
{
totalCallDuration += watch.Elapsed;
}
}
if (state == RpcCallResultState.Succeeded)
{
return(new RpcCallResult <Unit>(Unit.Void, attempts: numTry, duration: totalCallDuration, waitForConnectionDuration: waitForConnectionDuration));
}
return(new RpcCallResult <Unit>(
state,
attempts: numTry,
duration: totalCallDuration,
waitForConnectionDuration: waitForConnectionDuration,
lastFailure: failure));
}
public async Task <RpcCallResult <Unit> > CallAsync(
Func <CallOptions, AsyncUnaryCall <RpcResponse> > func,
string operationName,
CancellationToken cancellationToken = default(CancellationToken),
bool waitForConnection = false)
{
TimeSpan waitForConnectionDuration = TimeSpan.Zero;
if (waitForConnection)
{
var waitForConnectionSw = new StopwatchVar();
using (waitForConnectionSw.Start())
{
try
{
Logger.Log.DistributionTrace(m_loggingContext, $"Attempt to connect to '{Channel.Target}' for operation {operationName}. ChannelState '{Channel.State}'");
await Channel.ConnectAsync(DateTime.UtcNow.Add(InactiveTimeout));
Logger.Log.DistributionTrace(m_loggingContext, $"Connected to '{Channel.Target}' for operation {operationName}. Duration {waitForConnectionSw.TotalElapsed.TotalMilliseconds}ms.");
}
catch (OperationCanceledException e)
{
var duration = waitForConnectionSw.TotalElapsed;
Logger.Log.DistributionTrace(m_loggingContext, $"Failed to connect to '{Channel.Target}' for operation {operationName}. Duration {duration.TotalMilliseconds}ms. Failure {e.Message}");
return(new RpcCallResult <Unit>(RpcCallResultState.Cancelled, attempts: 1, duration: TimeSpan.Zero, waitForConnectionDuration: duration));
}
}
waitForConnectionDuration = waitForConnectionSw.TotalElapsed;
}
var callDurationSw = new StopwatchVar();
using (callDurationSw.Start())
{
try
{
var callOptions = new CallOptions(
deadline: DateTime.UtcNow.Add(CallTimeout),
cancellationToken: cancellationToken);
Logger.Log.DistributionTrace(m_loggingContext, $"Attempt to call '{Channel.Target}' for operation {operationName}. ChannelState '{Channel.State}'");
await func(callOptions);
Logger.Log.DistributionTrace(m_loggingContext, $"Called '{Channel.Target}' for operation {operationName}. Duration {callDurationSw.TotalElapsed.TotalMilliseconds}ms.");
}
catch (RpcException e)
{
RpcCallResultState state = e.Status.StatusCode == StatusCode.Cancelled ? RpcCallResultState.Cancelled : RpcCallResultState.Failed;
Logger.Log.DistributionTrace(m_loggingContext, $"Failed to call '{Channel.Target}' for operation {operationName}. Duration {callDurationSw.TotalElapsed.TotalMilliseconds}ms. Failure {e.Message}");
return(new RpcCallResult <Unit>(
state,
attempts: 1,
duration: callDurationSw.TotalElapsed,
waitForConnectionDuration: waitForConnectionDuration,
lastFailure: state == RpcCallResultState.Failed
? new RecoverableExceptionFailure(new BuildXLException(e.Message, e))
: null));
}
}
return(new RpcCallResult <Unit>(Unit.Void, attempts: 1, duration: callDurationSw.TotalElapsed, waitForConnectionDuration: waitForConnectionDuration));
}
