public virtual async Task <QueueServiceBatchResult> FetchAndProcessDataAsync(
Func <TData, Task <TResult> > processFunc,
long totalCount)
{
var batchResult = new QueueServiceBatchResult();
foreach (var fetchTask in DequeueMany(totalCount, Configuration.BatchSize))
{
IQueueMessage <TData> message = null;
try
{
message = await GetQueueExceptionPolicy()
.ExecuteAsync(async() => await fetchTask.ConfigureAwait(false))
.ConfigureAwait(false);
if (!Validate(message))
{
_requestValidator.HandleInvalidMessage(message, batchResult, nameof(FetchAndProcessDataAsync));
continue;
}
var result = await Policy.Handle <MessageFetchAndProcessException>()
.WaitAndRetryAsync(GetRetrySleepDurations(Configuration.RetryInMessageProcess))
.ExecuteAsync(async() => await QueueServiceMessageHelpers
.ProcessMessageAsync(message, processFunc)
.ConfigureAwait(false))
.ConfigureAwait(false);
_logger.LogDebug(result.ItemDescription);
batchResult.SuccessCount++;
}
catch (MessageQueueException e)
{
_logger.LogError("Error while fetching queue item. Item will remain in the queue.", e);
batchResult.FailureCount++;
}
catch (MessageFetchAndProcessException e)
{
_logger.LogError(
"Error while processing message data. Adding the data back to the message queue for the next run.", e);
batchResult.FailureCount++;
if (message == null)
{
_logger.LogWarning("Queue message is null, skipping.");
continue;
}
await MessageQueue.EnqueueAsync(message.Data, message.Id).ConfigureAwait(false);
}
}
return(batchResult);
}
public virtual async Task <QueueServiceBatchResult> FetchAndProcessDataManyAsync(
Func <IEnumerable <TData>, Task <int> > processFunc,
long totalCount)
{
var batchResult = new QueueServiceBatchResult();
var bag = new ConcurrentBag <IQueueMessage <TData> >();
foreach (var fetchTask in DequeueMany(totalCount, Configuration.BatchSize))
{
try
{
var message = await GetQueueExceptionPolicy()
.ExecuteAsync(async() => await fetchTask.ConfigureAwait(false));
if (!Validate(message))
{
_requestValidator.HandleInvalidMessage(message, batchResult, nameof(FetchAndProcessDataManyAsync));
continue;
}
bag.Add(message);
}
catch (MessageQueueException e)
{
_logger.LogError("Error while fetching queue item. Item will remain in the queue.", e);
batchResult.FailureCount++;
}
}
foreach (var processBatch in bag.Partition(Configuration.ProcessBatchSize))
{
try
{
var count = await QueueServiceMessageHelpers
.ProcessMessagesAsync(processBatch, processFunc).ConfigureAwait(false);
batchResult.SuccessCount += Math.Max(0, count);
}
catch (MessageFetchAndProcessException e)
{
_logger.LogError(
"Error while processing messages. Adding the data back to the message queue for the next run.", e);
batchResult.FailureCount += processBatch.Count;
processBatch.ForEach(async message =>
await AddDataAsync(message.Data, message.Id).ConfigureAwait(false));
}
}
return(batchResult);
}
