private async Task<AsyncOperationResult<T>> TryClusterOperationAsync<T>(OperationMetadata node, Func<OperationMetadata, Task<T>> operation, bool avoidThrowing, CancellationToken token)
{
Debug.Assert(node != null);
token.ThrowIfCancellationRequested();
var shouldRetry = false;
var operationResult = new AsyncOperationResult<T>();
try
{
operationResult.Result = await operation(node).ConfigureAwait(false);
operationResult.Success = true;
}
catch (Exception e)
{
bool wasTimeout;
if (HttpConnectionHelper.IsServerDown(e, out wasTimeout))
{
shouldRetry = true;
operationResult.WasTimeout = wasTimeout;
}
else
{
var ae = e as AggregateException;
ErrorResponseException errorResponseException;
if (ae != null)
errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException;
else
errorResponseException = e as ErrorResponseException;
if (errorResponseException != null && (errorResponseException.StatusCode == HttpStatusCode.Redirect || errorResponseException.StatusCode == HttpStatusCode.ExpectationFailed))
shouldRetry = true;
}
if (shouldRetry == false && avoidThrowing == false)
throw;
}
if (operationResult.Success)
FailureCounters.ResetFailureCount(node.Url);
return operationResult;
}
private async Task <AsyncOperationResult <T> > TryClusterOperationAsync <T>(OperationMetadata node, Func <OperationMetadata, IRequestTimeMetric, Task <T> > operation, bool avoidThrowing, CancellationToken token)
{
Debug.Assert(node != null);
token.ThrowIfCancellationRequested();
var shouldRetry = false;
var operationResult = new AsyncOperationResult <T>();
try
{
operationResult.Result = await operation(node, null).ConfigureAwait(false);
operationResult.Success = true;
}
catch (Exception e)
{
bool wasTimeout;
if (HttpConnectionHelper.IsServerDown(e, out wasTimeout))
{
shouldRetry = true;
operationResult.WasTimeout = wasTimeout;
if (Log.IsDebugEnabled)
{
Log.Debug($"Operation failed because server {node.Url} is down.");
}
}
else
{
var ae = e as AggregateException;
ErrorResponseException errorResponseException;
if (ae != null)
{
errorResponseException = ae.ExtractSingleInnerException() as ErrorResponseException;
}
else
{
errorResponseException = e as ErrorResponseException;
}
if (errorResponseException != null)
{
if (errorResponseException.StatusCode == HttpStatusCode.Redirect)
{
IEnumerable <string> values;
if (errorResponseException.Response.Headers.TryGetValues("Raven-Leader-Redirect", out values) == false &&
values.Contains("true") == false)
{
throw new InvalidOperationException("Got 302 Redirect, but without Raven-Leader-Redirect: true header, maybe there is a proxy in the middle", e);
}
var redirectUrl = errorResponseException.Response.Headers.Location.ToString();
var newLeaderNode = Nodes.FirstOrDefault(n => n.Url.Equals(redirectUrl)) ?? new OperationMetadata(redirectUrl, node.Credentials, node.ClusterInformation);
SetLeaderNodeToKnownLeader(newLeaderNode);
if (Log.IsDebugEnabled)
{
Log.Debug($"Redirecting to {redirectUrl} because {node.Url} responded with 302-redirect.");
}
return(await TryClusterOperationAsync(newLeaderNode, operation, avoidThrowing, token).ConfigureAwait(false));
}
if (errorResponseException.StatusCode == HttpStatusCode.ExpectationFailed)
{
if (Log.IsDebugEnabled)
{
Log.Debug($"Operation failed with status code {HttpStatusCode.ExpectationFailed}, will retry.");
}
shouldRetry = true;
}
}
}
if (shouldRetry == false && avoidThrowing == false)
{
throw;
}
operationResult.Error = e;
}
if (operationResult.Success)
{
FailureCounters.ResetFailureCount(node.Url);
}
return(operationResult);
}
