public void DecreasingTimeMetricDecreasedRequestTimeMetricAfterEachRequest()
{
var conventions = new DocumentConvention
{
RequestTimeSlaThresholdInMilliseconds = 100 // switch back threshold will be 75
};
var metric = new RequestTimeMetric();
UpdateMetric(metric, 500, 60); // 500
Assert.True(metric.RateSurpassed(conventions));
var decreasingMetric = new DecreasingTimeMetric(metric);
UpdateMetric(decreasingMetric, 500, 30); // 77+
Assert.True(metric.RateSurpassed(conventions));
UpdateMetric(decreasingMetric, 500, 1); // 73
Assert.False(metric.RateSurpassed(conventions));
}
public void RequestTimeMetricRequiresLowerThresholdToSwitchBackIfItHasSurpassed()
{
var conventions = new DocumentConvention
{
RequestTimeSlaThresholdInMilliseconds = 100 // switch back threshold will be 75
};
var metric = new RequestTimeMetric();
Assert.False(metric.RateSurpassed(conventions));
UpdateMetric(metric, 500, 60); // 500
Assert.True(metric.RateSurpassed(conventions));
UpdateMetric(metric, 15, 24); // 75+
Assert.True(metric.RateSurpassed(conventions));
UpdateMetric(metric, 15, 1); // 70
Assert.False(metric.RateSurpassed(conventions));
}
public async Task <T> ExecuteWithReplicationAsync <T>(HttpMethod method,
string primaryUrl,
OperationCredentials primaryCredentials,
RequestTimeMetric primaryRequestTimeMetric,
int currentRequest,
int currentReadStripingBase,
Func <OperationMetadata, Task <T> > operation,
CancellationToken token = default(CancellationToken))
{
Debug.Assert(typeof(T).FullName.Contains("Task") == false);
var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy
var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials, null);
var operationResult = new AsyncOperationResult <T>();
var shouldReadFromAllServers = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers);
var allowReadFromSecondariesWhenRequestTimeThresholdIsPassed = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.AllowReadFromSecondariesWhenRequestTimeThresholdIsSurpassed);
if (method == HttpMethods.Get && (shouldReadFromAllServers || allowReadFromSecondariesWhenRequestTimeThresholdIsPassed))
{
var replicationIndex = -1;
if (allowReadFromSecondariesWhenRequestTimeThresholdIsPassed && primaryRequestTimeMetric != null && primaryRequestTimeMetric.RateSurpassed(Conventions))
{
replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count);
}
else if (shouldReadFromAllServers)
{
replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1);
}
// if replicationIndex == destinations count, then we want to use the master
// if replicationIndex < 0, then we were explicitly instructed to use the master
if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0)
{
// if it is failing, ignore that, and move to the master or any of the replicas
if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, method, false, null, token))
{
operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
}
}
}
if (ShouldExecuteUsing(primaryOperation, primaryOperation, method, true, null, token))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0, token)
.ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(primaryOperation.Url);
if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(primaryOperation.Url))
{
operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(primaryOperation.Url);
}
}
for (var i = 0; i < localReplicationDestinations.Count; i++)
{
token.ThrowCancellationIfNotDefault();
var replicationDestination = localReplicationDestinations[i];
if (ShouldExecuteUsing(replicationDestination, primaryOperation, method, false, operationResult.Error, token) == false)
{
continue;
}
var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1;
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(replicationDestination.Url);
if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(replicationDestination.Url))
{
operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations, token).ConfigureAwait(false);
if (operationResult.Success)
{
return(operationResult.Result);
}
FailureCounters.IncrementFailureCount(replicationDestination.Url);
}
}
// this should not be thrown, but since I know the value of should...
throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up.
There is a high probability of a network problem preventing access to all the replicas.
Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances.");
}
public ReplicationAwareRequestExecuter(IDocumentStoreReplicationInformer replicationInformer, RequestTimeMetric requestTimeMetric)
{
this.replicationInformer = replicationInformer;
this.requestTimeMetric = requestTimeMetric;
}
