public RetryStrategy(ExpectedPollException ex, RetryStrategy lastRetryStategy)
{
this.ErrorCategory = ex.ErrorCategory;
if (this.ErrorCategory != lastRetryStategy?.ErrorCategory)
{
this.RetryCount = 0;
}
else
{
this.RetryCount = (lastRetryStategy?.RetryCount ?? 0) + 1;
}
switch (this.ErrorCategory)
{
case ExpectedErrorCategory.Unauthorised401:
case ExpectedErrorCategory.DuplicateMessageDetected:
this.DropImmediately = true;
break;
case ExpectedErrorCategory.InvalidRPDEPage:
case ExpectedErrorCategory.PageFetchError:
case ExpectedErrorCategory.UnexpectedErrorDuringDatabaseWrite:
case ExpectedErrorCategory.UnexpectedError:
// Exponential backoff
if (this.RetryCount > 15)
{
this.DeadLetter = true;
}
else
{
this.DelaySeconds = (int)BigInteger.Pow(2, this.RetryCount);
}
break;
case ExpectedErrorCategory.ForceClearProxyCache:
this.DeadLetter = true;
break;
case ExpectedErrorCategory.SqlTransientError:
this.DelaySeconds = SqlUtils.SqlRetrySecondsRecommendation;
break;
default:
throw new InvalidOperationException("Unknown Retry Strategy");
}
}
public static async Task Run([ServiceBusTrigger(Utils.FEED_STATE_QUEUE_NAME, Connection = "ServiceBusConnection")] Message message, MessageReceiver messageReceiver, string lockToken,
[ServiceBus(Utils.FEED_STATE_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> queueCollector,
ILogger log)
{
var feedStateItem = FeedState.DecodeFromMessage(message);
log.LogInformation($"PollQueueHandler queue trigger function processed message: {feedStateItem?.nextUrl}");
// Increment poll requests before anything else
feedStateItem.totalPollRequests++;
feedStateItem.dateModified = DateTime.Now;
SourcePage sourcePage;
try
{
sourcePage = await ExecutePoll(feedStateItem.name, feedStateItem.nextUrl, feedStateItem.lastPageReads == 0, feedStateItem.deletedItemDaysToLive, log);
}
catch (Exception ex)
{
var expectedException = ExpectedPollException.ExpectTheUnexpected(ex);
feedStateItem.retryStategy = new RetryStrategy(expectedException, feedStateItem?.retryStategy);
feedStateItem.totalErrors++;
if (feedStateItem.retryStategy.DeadLetter)
{
log.LogError(expectedException.RenderMessageWithFullContext(feedStateItem, $"DEAD-LETTERING: '{feedStateItem.name}'."));
await messageReceiver.DeadLetterAsync(lockToken);
}
if (feedStateItem.retryStategy.DropImmediately)
{
log.LogWarning(expectedException.RenderMessageWithFullContext(feedStateItem, $"Dropped message for '{feedStateItem.name}'."));
await messageReceiver.CompleteAsync(lockToken);
}
else
{
feedStateItem.lastError = expectedException.RenderMessageWithFullContext(feedStateItem, $"Retrying '{feedStateItem.name}' attempt {feedStateItem.retryStategy.RetryCount} in {feedStateItem.retryStategy.DelaySeconds} seconds.");
log.LogWarning(feedStateItem.lastError);
var retryMsg = feedStateItem.EncodeToMessage(feedStateItem.retryStategy.DelaySeconds);
// Check lock exists, as close to a transaction as we can get
if (await messageReceiver.RenewLockAsync(lockToken) != null)
{
await messageReceiver.CompleteAsync(lockToken);
await queueCollector.AddAsync(retryMsg);
}
}
return;
}
// Update counters on success
if (sourcePage.IsLastPage)
{
feedStateItem.lastPageReads++;
}
else
{
feedStateItem.lastPageReads = 0;
}
feedStateItem.retryStategy = null;
feedStateItem.lastError = null;
feedStateItem.totalPagesRead++;
feedStateItem.totalItemsRead += sourcePage.Content.items.Count;
feedStateItem.nextUrl = sourcePage.Content.next;
Message newMessage;
// If immediate poll is specified for last page, respect any Expires header provided for throttling
if (sourcePage.IsLastPage)
{
if (sourcePage.LastPageDetails?.Expires != null)
{
newMessage = feedStateItem.EncodeToMessage(sourcePage.LastPageDetails.Expires);
}
else if (sourcePage.LastPageDetails?.MaxAge != null)
{
newMessage = feedStateItem.EncodeToMessage((int)sourcePage.LastPageDetails.MaxAge?.TotalSeconds);
}
else
{
// Default last page polling interval
newMessage = feedStateItem.EncodeToMessage(Utils.DEFAULT_POLL_INTERVAL);
}
}
else
{
// If not last page, get the next page immediately
newMessage = feedStateItem.EncodeToMessage(0);
}
// These two operations should be in a transaction, but to save cost they ordered so that a failure will result in the polling stopping,
// and the ResyncDroppedFeeds timer trigger will hence be required to ensure the system is still robust
// using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
// {
// await messageReceiver.CompleteAsync(lockToken);
// await queueCollector.AddAsync(newMessage);
// scope.Complete(); // declare the transaction done
// }
// Check lock exists, as close to a transaction as we can get
if (await messageReceiver.RenewLockAsync(lockToken) != null)
{
await messageReceiver.CompleteAsync(lockToken);
await queueCollector.AddAsync(newMessage);
}
}
