public void GetNextDelay_WhenExecutionFailedAgainEnoughTimes_ReturnsMaximumInternal()
{
// Arrange
TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123);
TimeSpan maximumInterval = TimeSpan.FromSeconds(4);
IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval);
double randomizationMinimum = 1 - RandomizedExponentialBackoffStrategy.RandomizationFactor;
TimeSpan minimumDeltaInterval = new TimeSpan((long)(minimumInterval.Ticks * randomizationMinimum));
// minimumBackoffInterval = minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1)
// when is minimumBackOffInterval first >= maximumInterval?
// maximumInterval <= minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1)
// minimumInterval + minimumDeltaInterval * 2 ^ (deltaIteration - 1) >= maximumInterval
// minimumDeltaInterval * 2 ^ (deltaIteration - 1) >= maximumInterval - minimumInterval
// 2 ^ (deltaIteration - 1) >= (maximumInterval - minimumInterval) / minimumDeltaInterval
// deltaIteration - 1 >= log2(maximumInterval - minimumInterval) / minimumDeltaInterval
// deltaIteration >= (log2(maximumInterval - minimumInterval) / minimumDeltaInterval) + 1
int deltaIterationsNeededForMaximumInterval = (int)Math.Ceiling(Math.Log(
(maximumInterval - minimumInterval).Ticks / minimumDeltaInterval.Ticks, 2)) + 1;
// Add one for initial minimumInterval interation (before deltaIterations start).
int iterationsNeededForMaximumInterval = deltaIterationsNeededForMaximumInterval + 1;
for (int iteration = 0; iteration < iterationsNeededForMaximumInterval - 1; iteration++)
{
product.GetNextDelay(executionSucceeded: false);
}
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false);
// Assert
Assert.Equal(maximumInterval, nextDelay);
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
TimeSpan delay;
try
{
_trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Renewing Singleton lock ({0})", _lockId), source: TraceSource.Execution);
AccessCondition condition = new AccessCondition
{
LeaseId = _leaseId
};
await _leaseBlob.RenewLeaseAsync(condition, null, null, cancellationToken);
// The next execution should occur after a normal delay.
delay = _speedupStrategy.GetNextDelay(executionSucceeded: true);
}
catch (StorageException exception)
{
if (exception.IsServerSideError())
{
// The next execution should occur more quickly (try to renew the lease before it expires).
delay = _speedupStrategy.GetNextDelay(executionSucceeded: false);
}
else
{
// If we've lost the lease or cannot restablish it, we want to fail any
// in progress function execution
throw;
}
}
return(new TaskSeriesCommandResult(wait: Task.Delay(delay)));
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
TimeSpan delay;
try
{
AccessCondition condition = new AccessCondition
{
LeaseId = _leaseId
};
DateTimeOffset requestStart = DateTimeOffset.UtcNow;
await _leaseBlob.RenewLeaseAsync(condition, null, null, cancellationToken);
_lastRenewal = DateTime.UtcNow;
_lastRenewalLatency = _lastRenewal - requestStart;
// The next execution should occur after a normal delay.
delay = _speedupStrategy.GetNextDelay(executionSucceeded: true);
}
catch (StorageException exception)
{
if (exception.IsServerSideError())
{
// The next execution should occur more quickly (try to renew the lease before it expires).
delay = _speedupStrategy.GetNextDelay(executionSucceeded: false);
string msg = string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. Retry renewal in {2} milliseconds.",
_lockId, FormatErrorCode(exception), delay.TotalMilliseconds);
_trace.Warning(msg, source: TraceSource.Execution);
_logger?.LogWarning(msg);
}
else
{
// Log the details we've been accumulating to help with debugging this scenario
int leasePeriodMilliseconds = (int)_leasePeriod.TotalMilliseconds;
string lastRenewalFormatted = _lastRenewal.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ", CultureInfo.InvariantCulture);
int millisecondsSinceLastSuccess = (int)(DateTime.UtcNow - _lastRenewal).TotalMilliseconds;
int lastRenewalMilliseconds = (int)_lastRenewalLatency.TotalMilliseconds;
string msg = string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. The last successful renewal completed at {2} ({3} milliseconds ago) with a duration of {4} milliseconds. The lease period was {5} milliseconds.",
_lockId, FormatErrorCode(exception), lastRenewalFormatted, millisecondsSinceLastSuccess, lastRenewalMilliseconds, leasePeriodMilliseconds);
_trace.Error(msg);
_logger?.LogError(msg);
// If we've lost the lease or cannot re-establish it, we want to fail any
// in progress function execution
throw;
}
}
return(new TaskSeriesCommandResult(wait: Task.Delay(delay)));
}
public void GetNextDelay_WhenSecondExecutionSucceededAfterFailing_ReturnsNormalInterval()
{
// Arrange
TimeSpan normalInterval = TimeSpan.FromMilliseconds(123);
IDelayStrategy product = CreateProductUnderTest(normalInterval, TimeSpan.Zero, 2);
product.GetNextDelay(executionSucceeded: false);
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: true);
// Assert
Assert.Equal(normalInterval, nextDelay);
}
public void GetNextDelay_WhenSecondExecutionFailedAgain_ReturnsApproximatelyDoubleMinimumInterval()
{
// Arrange
TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123);
TimeSpan maximumInterval = TimeSpan.FromSeconds(4);
IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval);
product.GetNextDelay(executionSucceeded: false);
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false);
// Assert
AssertInRandomizationRange(minimumInterval, 1, nextDelay);
}
public void GetNextDelay_WhenExecutionSuccededAfterPreviouslyFailing_ReturnsMinimumInterval()
{
// Arrange
TimeSpan minimumInterval = TimeSpan.FromMilliseconds(123);
TimeSpan maximumInterval = TimeSpan.FromSeconds(4);
IDelayStrategy product = CreateProductUnderTest(minimumInterval, maximumInterval);
product.GetNextDelay(executionSucceeded: false);
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: true);
// Assert
Assert.Equal(minimumInterval, nextDelay);
}
private Task CreateDelayWithNotificationTask()
{
Task normalDelay = Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded: _foundMessageSinceLastDelay));
_foundMessageSinceLastDelay = false;
return(Task.WhenAny(_stopWaitingTaskSource.Task, normalDelay));
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
bool succeeded = await _innerCommand.TryExecuteAsync(cancellationToken).ConfigureAwait(false);
Task wait = Task.Delay(_delayStrategy.GetNextDelay(succeeded), cancellationToken);
return(new TaskSeriesCommandResult(wait));
}
public void GetNextDelay_WhenExecutionFailedAgainEnoughTimes_ReturnsMinimumInternal()
{
// Arrange
TimeSpan normalInterval = TimeSpan.FromMilliseconds(100);
TimeSpan minimumInternal = TimeSpan.FromMilliseconds(30);
int failureDivisor = 2;
IDelayStrategy product = CreateProductUnderTest(normalInterval, minimumInternal, failureDivisor);
product.GetNextDelay(executionSucceeded: false);
product.GetNextDelay(executionSucceeded: false);
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false);
// Assert
Assert.Equal(minimumInternal, nextDelay);
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
// Exceptions wil propagate
bool executionSucceeded = await _lockManager.RenewAsync(_lock, cancellationToken);
TimeSpan delay = _speedupStrategy.GetNextDelay(executionSucceeded: true);
return(new TaskSeriesCommandResult(wait: Task.Delay(delay)));
}
public void GetNextDelay_WhenSecondExecutionFailedAgain_ReturnsNormalIntervalDividedByDivisorTwice()
{
// Arrange
TimeSpan normalInterval = TimeSpan.FromMilliseconds(123);
int failureDivisor = 2;
IDelayStrategy product = CreateProductUnderTest(normalInterval, TimeSpan.Zero, failureDivisor);
product.GetNextDelay(executionSucceeded: false);
// Act
TimeSpan nextDelay = product.GetNextDelay(executionSucceeded: false);
// Assert
const int executeCalls = 2;
TimeSpan expectedNextDelay = new TimeSpan(normalInterval.Ticks / (failureDivisor * executeCalls));
Assert.Equal(expectedNextDelay, nextDelay);
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
// Exceptions will propagate
bool renewalSucceeded = await _lockManager.RenewAsync(_lock, cancellationToken);
TimeSpan delay = _delayStrategy.GetNextDelay(renewalSucceeded);
return(new TaskSeriesCommandResult(Task.Delay(delay)));
}
public async Task <TaskSeriesCommandResult> ExecuteAsync(CancellationToken cancellationToken)
{
TimeSpan delay;
try
{
UpdateReceipt updateReceipt = await _queue.UpdateMessageAsync(_message.MessageId, _message.PopReceipt, visibilityTimeout : _visibilityTimeout, cancellationToken : cancellationToken).ConfigureAwait(false);
_message = _message.Update(updateReceipt);
_onUpdateReceipt?.Invoke(updateReceipt);
// The next execution should occur after a normal delay.
delay = _speedupStrategy.GetNextDelay(executionSucceeded: true);
}
catch (RequestFailedException exception)
{
// For consistency, the exceptions handled here should match PollQueueCommand.DeleteMessageAsync.
if (exception.IsServerSideError())
{
// The next execution should occur more quickly (try to update the visibility before it expires).
delay = _speedupStrategy.GetNextDelay(executionSucceeded: false);
}
if (exception.IsBadRequestPopReceiptMismatch())
{
// There's no point to executing again. Once the pop receipt doesn't match, we've permanently lost
// ownership.
delay = Timeout.InfiniteTimeSpan;
}
else if (exception.IsNotFoundMessageOrQueueNotFound() ||
exception.IsConflictQueueBeingDeletedOrDisabled())
{
// There's no point to executing again. Once the message or queue is deleted, we've permanently lost
// ownership.
// For queue disabled, in theory it's possible the queue could be re-enabled, but the scenarios here
// are currently unclear.
delay = Timeout.InfiniteTimeSpan;
}
else
{
throw;
}
}
return(new TaskSeriesCommandResult(wait: Task.Delay(delay, cancellationToken)));
}
public override Task RunAsync(CancellationToken cancellationToken)
{
return(Task.Run(async() =>
{
try
{
while (!cancellationToken.IsCancellationRequested)
{
IQueueMessage message = null;
bool executionSucceeded = false;
try
{
do
{
message = await _queueReader.GetMessageAsync();
if (message == null)
{
break;
}
var context = new QueueTriggeringContext(message.InsertionTime);
var p = new List <object>()
{
message.Value(_parameterType)
};
if (_hasSecondParameter)
{
p.Add(_useTriggeringContext ? context : (object)message.InsertionTime);
}
await Invoke(_serviceProvider, _method, p.ToArray());
await ProcessCompletedMessage(message, context);
executionSucceeded = true;
} while (!cancellationToken.IsCancellationRequested);
}
catch (Exception ex)
{
await LogError("QueueTriggerBinding", "RunAsync", ex);
await ProcessFailedMessage(message);
executionSucceeded = false;
}
finally
{
await Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded), cancellationToken);
}
}
}
finally
{
await _log.WriteInfoAsync("QueueTriggerBinding", "RunAsync", _queueName, "Process ended");
}
}, cancellationToken));
}
private Task CreateDelayWithNotificationTask()
{
TimeSpan nextDelay = _delayStrategy.GetNextDelay(executionSucceeded: _foundMessageSinceLastDelay);
Task normalDelay = Task.Delay(nextDelay);
_foundMessageSinceLastDelay = false;
Logger.BackoffDelay(_logger, _functionDescriptor.LogName, _queue.Name, nextDelay.TotalMilliseconds);
return(Task.WhenAny(_stopWaitingTaskSource.Task, normalDelay));
}
public override Task RunAsync(CancellationToken cancellationToken)
{
return(Task.Factory.StartNew(async() =>
{
Exception globalEx = null;
try
{
while (!cancellationToken.IsCancellationRequested)
{
IQueueMessage message = null;
bool executionSucceeded = false;
try
{
do
{
message = await _queueReader.GetMessageAsync();
if (message == null)
{
break;
}
var context = new QueueTriggeringContext(message.InsertionTime);
var p = new List <object>()
{
message.Value(_parameterType)
};
if (_hasSecondParameter)
{
p.Add(_useTriggeringContext ? context : (object)message.InsertionTime);
}
await Invoke(_serviceProvider, _method, p.ToArray());
await ProcessCompletedMessage(message, context);
executionSucceeded = true;
} while (!cancellationToken.IsCancellationRequested);
}
catch (Exception ex)
{
await LogError("QueueTriggerBinding", "RunAsync", ex);
await ProcessFailedMessage(message);
executionSucceeded = false;
}
finally
{
await Task.Delay(_delayStrategy.GetNextDelay(executionSucceeded), cancellationToken);
}
}
}
catch (Exception ex)
{
globalEx = ex;
}
finally
{
var msg =
$"Process ended. Exception={globalEx?.Message + globalEx?.StackTrace}. Token.IsCancellationRequested={cancellationToken.IsCancellationRequested}";
await _log.WriteInfoAsync("QueueTriggerBinding", "RunAsync", _queueName, msg);
}
}, cancellationToken, TaskCreationOptions.LongRunning | TaskCreationOptions.DenyChildAttach, TaskScheduler.Default).Unwrap());
}
