public void GetNextDelay_WhenExecutionFailedAgainEnoughTimes_ReturnsMaximumInternal() { // Arrange TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123); TimeSpan maximumInterval = TimeSpan.FromSeconds(4); IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval); double randomizationMinimum = 1 - RandomizedExponentialBackoffStrategy.RandomizationFactor; TimeSpan minimumDeltaInterval = new TimeSpan((long)(minimumInterval.Ticks * randomizationMinimum)); // minimumBackoffInterval = minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1) // when is minimumBackOffInterval first >= maximumInterval? // maximumInterval <= minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1) // minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1) >= maximumInterval // minimumDeltaInterval * 2 ^ (deltaIteration - 1) >= maximumInterval - minimumInterval // 2 ^ (deltaIteration - 1) >= (maximumInterval - minimumInterval) / minimumDeltaInterval // deltaIteration - 1 >= log2(maximumInterval - minimumInterval) / minimumDeltaInterval // deltaIteration >= (log2(maximumInterval - minimumInterval) / minimumDeltaInterval) + 1 int deltaIterationsNeededForMaximumInterval = (int)Math.Ceiling(Math.Log( (maximumInterval - minimumInterval).Ticks / minimumDeltaInterval.Ticks, 2)) + 1; // Add one for initial minimumInterval interation (before deltaIterations start). int iterationsNeededForMaximumInterval = deltaIterationsNeededForMaximumInterval + 1; for (int iteration = 0; iteration < iterationsNeededForMaximumInterval - 1; iteration++) { product.GetNextDelay(executionSucceeded: false); } // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false); // Assert Assert.Equal(maximumInterval, nextDelay); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { TimeSpan delay; try { _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Renewing Singleton lock ({0})", _lockId), source: TraceSource.Execution); AccessCondition condition = new AccessCondition { LeaseId = _leaseId }; await _leaseBlob.RenewLeaseAsync(condition, null, null, cancellationToken); // The next execution should occur after a normal delay. delay = _speedupStrategy.GetNextDelay(executionSucceeded: true); } catch (StorageException exception) { if (exception.IsServerSideError()) { // The next execution should occur more quickly (try to renew the lease before it expires). delay = _speedupStrategy.GetNextDelay(executionSucceeded: false); } else { // If we've lost the lease or cannot restablish it, we want to fail any // in progress function execution throw; } } return(new TaskSeriesCommandResult(wait: Task.Delay(delay))); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { TimeSpan delay; try { AccessCondition condition = new AccessCondition { LeaseId = _leaseId }; DateTimeOffset requestStart = DateTimeOffset.UtcNow; await _leaseBlob.RenewLeaseAsync(condition, null, null, cancellationToken); _lastRenewal = DateTime.UtcNow; _lastRenewalLatency = _lastRenewal - requestStart; // The next execution should occur after a normal delay. delay = _speedupStrategy.GetNextDelay(executionSucceeded: true); } catch (StorageException exception) { if (exception.IsServerSideError()) { // The next execution should occur more quickly (try to renew the lease before it expires). delay = _speedupStrategy.GetNextDelay(executionSucceeded: false); string msg = string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. Retry renewal in {2} milliseconds.", _lockId, FormatErrorCode(exception), delay.TotalMilliseconds); _trace.Warning(msg, source: TraceSource.Execution); _logger?.LogWarning(msg); } else { // Log the details we've been accumulating to help with debugging this scenario int leasePeriodMilliseconds = (int)_leasePeriod.TotalMilliseconds; string lastRenewalFormatted = _lastRenewal.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ", CultureInfo.InvariantCulture); int millisecondsSinceLastSuccess = (int)(DateTime.UtcNow - _lastRenewal).TotalMilliseconds; int lastRenewalMilliseconds = (int)_lastRenewalLatency.TotalMilliseconds; string msg = string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. The last successful renewal completed at {2} ({3} milliseconds ago) with a duration of {4} milliseconds. The lease period was {5} milliseconds.", _lockId, FormatErrorCode(exception), lastRenewalFormatted, millisecondsSinceLastSuccess, lastRenewalMilliseconds, leasePeriodMilliseconds); _trace.Error(msg); _logger?.LogError(msg); // If we've lost the lease or cannot re-establish it, we want to fail any // in progress function execution throw; } } return(new TaskSeriesCommandResult(wait: Task.Delay(delay))); }
public void GetNextDelay_WhenSecondExecutionSucceededAfterFailing_ReturnsNormalInterval() { // Arrange TimeSpan normalInterval = TimeSpan.FromMilliseconds(123); IDelayStrategy product = CreateProductUnderTest(normalInterval, TimeSpan.Zero, 2); product.GetNextDelay(executionSucceeded: false); // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: true); // Assert Assert.Equal(normalInterval, nextDelay); }
public void GetNextDelay_WhenSecondExecutionFailedAgain_ReturnsApproximatelyDoubleMinimumInterval() { // Arrange TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123); TimeSpan maximumInterval = TimeSpan.FromSeconds(4); IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval); product.GetNextDelay(executionSucceeded: false); // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false); // Assert AssertInRandomizationRange(minimumInterval, 1, nextDelay); }
public void GetNextDelay_WhenExecutionSuccededAfterPreviouslyFailing_ReturnsMinimumInterval() { // Arrange TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123); TimeSpan maximumInterval = TimeSpan.FromSeconds(4); IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval); product.GetNextDelay(executionSucceeded: false); // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: true); // Assert Assert.Equal(minimumInterval, nextDelay); }
private Task CreateDelayWithNotificationTask() { Task normalDelay = Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded: _foundMessageSinceLastDelay)); _foundMessageSinceLastDelay = false; return(Task.WhenAny(_stopWaitingTaskSource.Task, normalDelay)); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { bool succeeded = await _innerCommand.TryExecuteAsync(cancellationToken).ConfigureAwait(false); Task wait = Task.Delay(_delayStrategy.GetNextDelay(succeeded), cancellationToken); return(new TaskSeriesCommandResult(wait)); }
public void GetNextDelay_WhenExecutionFailedAgainEnoughTimes_ReturnsMinimumInternal() { // Arrange TimeSpan normalInterval = TimeSpan.FromMilliseconds(100); TimeSpan minimumInternal = TimeSpan.FromMilliseconds(30); int failureDivisor = 2; IDelayStrategy product = CreateProductUnderTest(normalInterval, minimumInternal, failureDivisor); product.GetNextDelay(executionSucceeded: false); product.GetNextDelay(executionSucceeded: false); // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false); // Assert Assert.Equal(minimumInternal, nextDelay); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { // Exceptions wil propagate bool executionSucceeded = await _lockManager.RenewAsync(_lock, cancellationToken); TimeSpan delay = _speedupStrategy.GetNextDelay(executionSucceeded: true); return(new TaskSeriesCommandResult(wait: Task.Delay(delay))); }
public void GetNextDelay_WhenSecondExecutionFailedAgain_ReturnsNormalIntervalDividedByDivisorTwice() { // Arrange TimeSpan normalInterval = TimeSpan.FromMilliseconds(123); int failureDivisor = 2; IDelayStrategy product = CreateProductUnderTest(normalInterval, TimeSpan.Zero, failureDivisor); product.GetNextDelay(executionSucceeded: false); // Act TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false); // Assert const int executeCalls = 2; TimeSpan expectedNextDelay = new TimeSpan(normalInterval.Ticks / (failureDivisor * executeCalls)); Assert.Equal(expectedNextDelay, nextDelay); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { // Exceptions will propagate bool renewalSucceeded = await _lockManager.RenewAsync(_lock, cancellationToken); TimeSpan delay = _delayStrategy.GetNextDelay(renewalSucceeded); return(new TaskSeriesCommandResult(Task.Delay(delay))); }
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken) { TimeSpan delay; try { UpdateReceipt updateReceipt = await _queue.UpdateMessageAsync(_message.MessageId, _message.PopReceipt, visibilityTimeout : _visibilityTimeout, cancellationToken : cancellationToken).ConfigureAwait(false); _message = _message.Update(updateReceipt); _onUpdateReceipt?.Invoke(updateReceipt); // The next execution should occur after a normal delay. delay = _speedupStrategy.GetNextDelay(executionSucceeded: true); } catch (RequestFailedException exception) { // For consistency, the exceptions handled here should match PollQueueCommand.DeleteMessageAsync. if (exception.IsServerSideError()) { // The next execution should occur more quickly (try to update the visibility before it expires). delay = _speedupStrategy.GetNextDelay(executionSucceeded: false); } if (exception.IsBadRequestPopReceiptMismatch()) { // There's no point to executing again. Once the pop receipt doesn't match, we've permanently lost // ownership. delay = Timeout.InfiniteTimeSpan; } else if (exception.IsNotFoundMessageOrQueueNotFound() || exception.IsConflictQueueBeingDeletedOrDisabled()) { // There's no point to executing again. Once the message or queue is deleted, we've permanently lost // ownership. // For queue disabled, in theory it's possible the queue could be re-enabled, but the scenarios here // are currently unclear. delay = Timeout.InfiniteTimeSpan; } else { throw; } } return(new TaskSeriesCommandResult(wait: Task.Delay(delay, cancellationToken))); }
public override Task RunAsync(CancellationToken cancellationToken) { return(Task.Run(async() => { try { while (!cancellationToken.IsCancellationRequested) { IQueueMessage message = null; bool executionSucceeded = false; try { do { message = await _queueReader.GetMessageAsync(); if (message == null) { break; } var context = new QueueTriggeringContext(message.InsertionTime); var p = new List <object>() { message.Value(_parameterType) }; if (_hasSecondParameter) { p.Add(_useTriggeringContext ? context : (object)message.InsertionTime); } await Invoke(_serviceProvider, _method, p.ToArray()); await ProcessCompletedMessage(message, context); executionSucceeded = true; } while (!cancellationToken.IsCancellationRequested); } catch (Exception ex) { await LogError("QueueTriggerBinding", "RunAsync", ex); await ProcessFailedMessage(message); executionSucceeded = false; } finally { await Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded), cancellationToken); } } } finally { await _log.WriteInfoAsync("QueueTriggerBinding", "RunAsync", _queueName, "Process ended"); } }, cancellationToken)); }
private Task CreateDelayWithNotificationTask() { TimeSpan nextDelay = _delayStrategy.GetNextDelay(executionSucceeded: _foundMessageSinceLastDelay); Task normalDelay = Task.Delay(nextDelay); _foundMessageSinceLastDelay = false; Logger.BackoffDelay(_logger, _functionDescriptor.LogName, _queue.Name, nextDelay.TotalMilliseconds); return(Task.WhenAny(_stopWaitingTaskSource.Task, normalDelay)); }
public override Task RunAsync(CancellationToken cancellationToken) { return(Task.Factory.StartNew(async() => { Exception globalEx = null; try { while (!cancellationToken.IsCancellationRequested) { IQueueMessage message = null; bool executionSucceeded = false; try { do { message = await _queueReader.GetMessageAsync(); if (message == null) { break; } var context = new QueueTriggeringContext(message.InsertionTime); var p = new List <object>() { message.Value(_parameterType) }; if (_hasSecondParameter) { p.Add(_useTriggeringContext ? context : (object)message.InsertionTime); } await Invoke(_serviceProvider, _method, p.ToArray()); await ProcessCompletedMessage(message, context); executionSucceeded = true; } while (!cancellationToken.IsCancellationRequested); } catch (Exception ex) { await LogError("QueueTriggerBinding", "RunAsync", ex); await ProcessFailedMessage(message); executionSucceeded = false; } finally { await Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded), cancellationToken); } } } catch (Exception ex) { globalEx = ex; } finally { var msg = $"Process ended. Exception={globalEx?.Message + globalEx?.StackTrace}. Token.IsCancellationRequested={cancellationToken.IsCancellationRequested}"; await _log.WriteInfoAsync("QueueTriggerBinding", "RunAsync", _queueName, msg); } }, cancellationToken, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default).Unwrap()); }