public void DecreasingTimeMetricDecreasedRequestTimeMetricAfterEachRequest() { var conventions = new DocumentConvention { RequestTimeSlaThresholdInMilliseconds = 100 // switch back threshold will be 75 }; var metric = new RequestTimeMetric(); UpdateMetric(metric, 500, 60); // 500 Assert.True(metric.RateSurpassed(conventions)); var decreasingMetric = new DecreasingTimeMetric(metric); UpdateMetric(decreasingMetric, 500, 30); // 77+ Assert.True(metric.RateSurpassed(conventions)); UpdateMetric(decreasingMetric, 500, 1); // 73 Assert.False(metric.RateSurpassed(conventions)); }
public void RequestTimeMetricRequiresLowerThresholdToSwitchBackIfItHasSurpassed() { var conventions = new DocumentConvention { RequestTimeSlaThresholdInMilliseconds = 100 // switch back threshold will be 75 }; var metric = new RequestTimeMetric(); Assert.False(metric.RateSurpassed(conventions)); UpdateMetric(metric, 500, 60); // 500 Assert.True(metric.RateSurpassed(conventions)); UpdateMetric(metric, 15, 24); // 75+ Assert.True(metric.RateSurpassed(conventions)); UpdateMetric(metric, 15, 1); // 70 Assert.False(metric.RateSurpassed(conventions)); }
public async Task <T> ExecuteWithReplicationAsync <T>(HttpMethod method, string primaryUrl, OperationCredentials primaryCredentials, RequestTimeMetric primaryRequestTimeMetric, int currentRequest, int currentReadStripingBase, Func <OperationMetadata, Task <T> > operation, CancellationToken token = default(CancellationToken)) { Debug.Assert(typeof(T).FullName.Contains("Task") == false); var localReplicationDestinations = ReplicationDestinationsUrls; // thread safe copy var primaryOperation = new OperationMetadata(primaryUrl, primaryCredentials, null); var operationResult = new AsyncOperationResult <T>(); var shouldReadFromAllServers = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.ReadFromAllServers); var allowReadFromSecondariesWhenRequestTimeThresholdIsPassed = Conventions.FailoverBehavior.HasFlag(FailoverBehavior.AllowReadFromSecondariesWhenRequestTimeThresholdIsSurpassed); if (method == HttpMethods.Get && (shouldReadFromAllServers || allowReadFromSecondariesWhenRequestTimeThresholdIsPassed)) { var replicationIndex = -1; if (allowReadFromSecondariesWhenRequestTimeThresholdIsPassed && primaryRequestTimeMetric != null && primaryRequestTimeMetric.RateSurpassed(Conventions)) { replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count); } else if (shouldReadFromAllServers) { replicationIndex = currentReadStripingBase % (localReplicationDestinations.Count + 1); } // if replicationIndex == destinations count, then we want to use the master // if replicationIndex < 0, then we were explicitly instructed to use the master if (replicationIndex < localReplicationDestinations.Count && replicationIndex >= 0) { // if it is failing, ignore that, and move to the master or any of the replicas if (ShouldExecuteUsing(localReplicationDestinations[replicationIndex], primaryOperation, method, false, null, token)) { operationResult = await TryOperationAsync(operation, localReplicationDestinations[replicationIndex], primaryOperation, true, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } } } } if (ShouldExecuteUsing(primaryOperation, primaryOperation, method, true, null, token)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, !operationResult.WasTimeout && localReplicationDestinations.Count > 0, token) .ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(primaryOperation.Url); if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(primaryOperation.Url)) { operationResult = await TryOperationAsync(operation, primaryOperation, null, localReplicationDestinations.Count > 0, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(primaryOperation.Url); } } for (var i = 0; i < localReplicationDestinations.Count; i++) { token.ThrowCancellationIfNotDefault(); var replicationDestination = localReplicationDestinations[i]; if (ShouldExecuteUsing(replicationDestination, primaryOperation, method, false, operationResult.Error, token) == false) { continue; } var hasMoreReplicationDestinations = localReplicationDestinations.Count > i + 1; operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, !operationResult.WasTimeout && hasMoreReplicationDestinations, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(replicationDestination.Url); if (!operationResult.WasTimeout && FailureCounters.IsFirstFailure(replicationDestination.Url)) { operationResult = await TryOperationAsync(operation, replicationDestination, primaryOperation, hasMoreReplicationDestinations, token).ConfigureAwait(false); if (operationResult.Success) { return(operationResult.Result); } FailureCounters.IncrementFailureCount(replicationDestination.Url); } } // this should not be thrown, but since I know the value of should... throw new InvalidOperationException(@"Attempted to connect to master and all replicas have failed, giving up. There is a high probability of a network problem preventing access to all the replicas. Failed to get in touch with any of the " + (1 + localReplicationDestinations.Count) + " Raven instances."); }
public ReplicationAwareRequestExecuter(IDocumentStoreReplicationInformer replicationInformer, RequestTimeMetric requestTimeMetric) { this.replicationInformer = replicationInformer; this.requestTimeMetric = requestTimeMetric; }