private void EnqueueBatchesIfNoFailures(
CollectorHttpClient client,
JToken context,
SortedDictionary <DateTime, CommitBatchTasks> commitBatchTasksMap,
Queue <CatalogItemBatch> unprocessedBatches,
Queue <BatchTask> processingBatches,
CatalogItemBatch lastBatch,
CancellationToken cancellationToken)
{
var hasAnyBatchFailed = processingBatches.Any(batch => batch.Task.IsFaulted || batch.Task.IsCanceled);
if (hasAnyBatchFailed)
{
return;
}
var batchesToEnqueue = Math.Min(
_maxConcurrentBatches - processingBatches.Count(batch => !batch.Task.IsCompleted),
unprocessedBatches.Count);
for (var i = 0; i < batchesToEnqueue; ++i)
{
var batch = unprocessedBatches.Dequeue();
var batchItem = batch.Items.First();
var packageId = GetKey(batchItem.Value);
var batchTask = commitBatchTasksMap[batchItem.CommitTimeStamp].BatchTasks
.Single(bt => bt.PackageId == packageId);
batchTask.Task = ProcessBatchAsync(client, context, packageId, batch, lastBatch, cancellationToken);
processingBatches.Enqueue(batchTask);
}
}
private async Task ProcessBatchAsync(
CollectorHttpClient client,
JToken context,
string packageId,
CatalogItemBatch batch,
CatalogItemBatch lastBatch,
CancellationToken cancellationToken)
{
await Task.Yield();
using (_telemetryService.TrackDuration(
TelemetryConstants.ProcessBatchSeconds,
new Dictionary <string, string>()
{
{ TelemetryConstants.Id, packageId },
{ TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() }
}))
{
await OnProcessBatchAsync(
client,
batch.Items.Select(item => item.Value),
context,
batch.CommitTimeStamp,
batch.CommitTimeStamp == lastBatch.CommitTimeStamp,
cancellationToken);
}
}
// Summary:
//
// 1. Process one catalog page at a time.
// 2. Within a given catalog page, batch catalog commit entries by lower-cased package ID.
// 3. Process up to `n` batches in parallel. Note that the batches may span multiple catalog commits.
// 4. Cease processing new batches if a failure has been observed. This job will eventually retry
// batches on its next outermost job loop.
// 5. If a failure has been observed, wait for all existing tasks to complete. Avoid task cancellation
// as that could lead to the entirety of a package registration being in an inconsistent state.
// To be fair, a well-timed exception could have the same result, but registration updates have never
// been transactional. Actively cancelling tasks would make an inconsistent registration more likely.
// 6. Update the cursor if and only if all preceding commits and the current (oldest) commit have been
// fully and successfully processed.
protected override async Task <bool> FetchAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
CancellationToken cancellationToken)
{
IEnumerable <CatalogItem> catalogItems = await FetchCatalogItemsAsync(client, front, cancellationToken);
var hasAnyBatchFailed = false;
var hasAnyBatchBeenProcessed = false;
foreach (CatalogItem catalogItem in catalogItems)
{
JObject page = await client.GetJObjectAsync(catalogItem.Uri, cancellationToken);
JToken context = page["@context"];
CatalogItemBatch[] batches = await CreateBatchesAsync(front, back, page);
SortedDictionary <DateTime, CommitBatchTasks> commitBatchTasksMap = CreateCommitBatchTasksMap(batches);
var unprocessedBatches = new Queue <CatalogItemBatch>(batches);
var processingBatches = new Queue <BatchTask>();
CatalogItemBatch lastBatch = unprocessedBatches.LastOrDefault();
var exceptions = new List <Exception>();
EnqueueBatchesIfNoFailures(
client,
context,
commitBatchTasksMap,
unprocessedBatches,
processingBatches,
lastBatch,
cancellationToken);
while (processingBatches.Any())
{
var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted)
.Select(batch => batch.Task)
.DefaultIfEmpty(CompletedTask);
await Task.WhenAny(activeTasks);
while (!hasAnyBatchFailed && commitBatchTasksMap.Any())
{
var commitBatchTasks = commitBatchTasksMap.First().Value;
var isCommitFullyProcessed = commitBatchTasks.BatchTasks.All(batch => batch.Task != null && batch.Task.IsCompleted);
if (!isCommitFullyProcessed)
{
break;
}
var isCommitSuccessfullyProcessed = commitBatchTasks.BatchTasks.All(batch => batch.Task.Status == TaskStatus.RanToCompletion);
if (isCommitSuccessfullyProcessed)
{
var commitTimeStamp = commitBatchTasks.CommitTimeStamp;
front.Value = commitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation($"{nameof(RegistrationCollector)}.{nameof(FetchAsync)} {nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front);
DequeueBatchesWhileMatches(processingBatches, batch => batch.CommitTimeStamp == commitTimeStamp);
commitBatchTasksMap.Remove(commitTimeStamp);
}
else // Canceled or Failed
{
hasAnyBatchFailed = true;
exceptions.AddRange(
commitBatchTasks.BatchTasks
.Select(batch => batch.Task)
.Where(task => (task.IsFaulted || task.IsCanceled) && task.Exception != null)
.Select(task => task.Exception));
}
}
if (hasAnyBatchFailed)
{
DequeueBatchesWhileMatches(processingBatches, batch => batch.Task.IsCompleted);
}
hasAnyBatchBeenProcessed = true;
EnqueueBatchesIfNoFailures(
client,
context,
commitBatchTasksMap,
unprocessedBatches,
processingBatches,
lastBatch,
cancellationToken);
}
if (hasAnyBatchFailed)
{
var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions);
throw new BatchProcessingException(innerException);
}
}
return(hasAnyBatchBeenProcessed);
}
