private async Task HandleServerDown <TResult>(ChoosenNode choosenNode, JsonOperationContext context, RavenCommand <TResult> command,
HttpRequestException e)
{
if (command.FailedNodes == null)
{
command.FailedNodes = new HashSet <ServerNode>();
}
choosenNode.Node.IsFailed = true;
command.FailedNodes.Add(choosenNode.Node);
var failoverNode = ChooseNodeForRequest(command, e);
await ExecuteAsync(failoverNode, context, command);
}
private ChoosenNode ChooseNodeForRequest <T>(RavenCommand <T> command, HttpRequestException exception = null)
{
var topology = _topology;
var leaderNode = topology.LeaderNode;
if (command.IsReadRequest)
{
if (topology.ReadBehavior == ReadBehavior.LeaderOnly)
{
if (command.IsFailedWithNode(leaderNode) == false)
{
return new ChoosenNode {
Node = leaderNode
}
}
;
throw new HttpRequestException("Leader not was failed to make this request. The current ReadBehavior is set to Leader to we won't failover to a differnt node.", exception);
}
if (topology.ReadBehavior == ReadBehavior.RoundRobin)
{
if (leaderNode.IsFailed == false && command.IsFailedWithNode(leaderNode) == false)
{
return new ChoosenNode {
Node = leaderNode
}
}
;
// TODO: Should we choose nodes here by rate value as for SLA?
var choosenNode = new ChoosenNode {
SkippedNodes = new List <ServerNode>()
};
foreach (var node in topology.Nodes)
{
if (node.IsFailed == false && command.IsFailedWithNode(node) == false)
{
choosenNode.Node = node;
return(choosenNode);
}
choosenNode.SkippedNodes.Add(node);
}
throw new HttpRequestException("Tried all nodes in the cluster but failed getting a response", exception);
}
if (topology.ReadBehavior == ReadBehavior.LeaderWithFailoverWhenRequestTimeSlaThresholdIsReached)
{
if (leaderNode.IsFailed == false && command.IsFailedWithNode(leaderNode) == false && leaderNode.IsRateSurpassed(topology.SLA.RequestTimeThresholdInMilliseconds))
{
return new ChoosenNode {
Node = leaderNode
}
}
;
var nodesWithLeader = topology.Nodes
.Union(new[] { leaderNode })
.OrderBy(node => node.Rate())
.ToList();
var fastestNode = nodesWithLeader.FirstOrDefault(node => node.IsFailed == false && command.IsFailedWithNode(node) == false);
nodesWithLeader.Remove(fastestNode);
var choosenNode = new ChoosenNode
{
Node = fastestNode,
SkippedNodes = nodesWithLeader
};
if (choosenNode.Node != null)
{
return(choosenNode);
}
throw new HttpRequestException("Tried all nodes in the cluster but failed getting a response", exception);
}
throw new InvalidOperationException($"Invalid ReadBehaviour value: {topology.ReadBehavior}");
}
if (topology.WriteBehavior == WriteBehavior.LeaderOnly)
{
if (command.IsFailedWithNode(leaderNode) == false)
{
return new ChoosenNode {
Node = leaderNode
}
}
;
throw new HttpRequestException("Leader not was failed to make this request. The current WriteBehavior is set to Leader to we won't failover to a differnt node.", exception);
}
if (topology.WriteBehavior == WriteBehavior.LeaderWithFailover)
{
if (leaderNode.IsFailed == false && command.IsFailedWithNode(leaderNode) == false)
{
return new ChoosenNode {
Node = leaderNode
}
}
;
// TODO: Should we choose nodes here by rate value as for SLA?
foreach (var node in topology.Nodes)
{
if (node.IsFailed == false && command.IsFailedWithNode(node) == false)
{
return new ChoosenNode {
Node = node
}
}
;
}
throw new HttpRequestException("Tried all nodes in the cluster but failed getting a response", exception);
}
throw new InvalidOperationException($"Invalid WriteBehavior value: {topology.WriteBehavior}");
}
private async Task <bool> HandleUnsuccessfulResponse <TResult>(ChoosenNode choosenNode, JsonOperationContext context, RavenCommand <TResult> command,
HttpResponseMessage response, string url)
{
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
command.SetResponse(null);
return(true);
case HttpStatusCode.Unauthorized:
case HttpStatusCode.PreconditionFailed:
if (string.IsNullOrEmpty(choosenNode.Node.ApiKey))
{
throw new UnauthorizedAccessException(
$"Got unauthorized response exception for {url}. Please specify an API Key.");
}
if (++command.AuthenticationRetries > 1)
{
throw new UnauthorizedAccessException(
$"Got unauthorized response exception for {url} after trying to authenticate using ApiKey.");
}
var oauthSource = response.Headers.GetFirstValue("OAuth-Source");
#if DEBUG && FIDDLER
// Make sure to avoid a cross DNS security issue, when running with Fiddler
if (string.IsNullOrEmpty(oauthSource) == false)
{
oauthSource = oauthSource.Replace("localhost:", "localhost.fiddler:");
}
#endif
await HandleUnauthorized(oauthSource, choosenNode.Node, context).ConfigureAwait(false);
await ExecuteAsync(choosenNode, context, command).ConfigureAwait(false);
return(true);
case HttpStatusCode.Forbidden:
throw new UnauthorizedAccessException(
$"Forbidan access to {url}. Make sure you're using the correct ApiKey.");
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
await HandleServerDown(choosenNode, context, command, null);
break;
case HttpStatusCode.Conflict:
// TODO: Conflict resolution
// current implementation is temporary
object message;
using (var stream = await response.Content.ReadAsStreamAsync())
{
var blittableJsonReaderObject = await context.ReadForMemoryAsync(stream, "PutResult");
blittableJsonReaderObject.TryGetMember("Message", out message);
}
throw new ConcurrencyException(message.ToString());
default:
await ThrowServerError(context, response);
break;
}
return(false);
}
public async Task ExecuteAsync <TResult>(ChoosenNode choosenNode, JsonOperationContext context, RavenCommand <TResult> command, CancellationToken token = default(CancellationToken))
{
string url;
var request = CreateRequest(choosenNode.Node, command, out url);
long cachedEtag;
BlittableJsonReaderObject cachedValue;
HttpCache.ReleaseCacheItem cachedItem;
using (cachedItem = GetFromCache(context, command, request, url, out cachedEtag, out cachedValue))
{
if (cachedEtag != 0)
{
var aggresiveCacheOptions = AggressiveCaching.Value;
if (aggresiveCacheOptions != null && cachedItem.Age < aggresiveCacheOptions.Duration)
{
command.SetResponse(cachedValue);
return;
}
request.Headers.IfNoneMatch.Add(new EntityTagHeaderValue("\"" + cachedEtag + "\""));
}
var sp = Stopwatch.StartNew();
HttpResponseMessage response;
try
{
response = await _httpClient.SendAsync(request, token).ConfigureAwait(false);
sp.Stop();
}
catch (HttpRequestException e) // server down, network down
{
sp.Stop();
await HandleServerDown(choosenNode, context, command, e);
return;
}
finally
{
var requestTimeInMilliseconds = sp.ElapsedMilliseconds;
choosenNode.Node.UpdateRequestTime(requestTimeInMilliseconds);
if (choosenNode.SkippedNodes != null)
{
foreach (var skippedNode in choosenNode.SkippedNodes)
{
skippedNode.DecreaseRate(requestTimeInMilliseconds);
}
}
}
if (response.StatusCode == HttpStatusCode.NotModified)
{
cachedItem.NotModified();
command.SetResponse(cachedValue);
return;
}
if (response.IsSuccessStatusCode == false)
{
if (await HandleUnsuccessfulResponse(choosenNode, context, command, response, url))
{
return;
}
}
if (response.Content.Headers.ContentLength.HasValue && response.Content.Headers.ContentLength == 0)
{
return;
}
await command.ProcessResponse(context, _cache, response, url);
}
}
