private static async Task <CatalogCommitItemBatch[]> CreateBatchesForAllAvailableItemsInPageAsync(
ReadWriteCursor front,
ReadCursor back,
JObject page,
JObject context,
CreateCommitItemBatchesAsync createCommitItemBatchesAsync)
{
IEnumerable <CatalogCommitItem> commitItems = page["items"]
.Select(item => CatalogCommitItem.Create(context, (JObject)item))
.Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value);
IEnumerable <CatalogCommitItemBatch> batches = await createCommitItemBatchesAsync(commitItems);
return(batches
.OrderBy(batch => batch.CommitTimeStamp)
.ToArray());
}
internal static async Task <bool> ProcessCatalogCommitsAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
FetchCatalogCommitsAsync fetchCatalogCommitsAsync,
CreateCommitItemBatchesAsync createCommitItemBatchesAsync,
ProcessCommitItemBatchAsync processCommitItemBatchAsync,
int maxConcurrentBatches,
ILogger logger,
CancellationToken cancellationToken)
{
var rootItems = await fetchCatalogCommitsAsync(client, front, back, cancellationToken);
var hasAnyBatchFailed = false;
var hasAnyBatchBeenProcessed = false;
foreach (CatalogCommit rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
var context = (JObject)page["@context"];
CatalogCommitItemBatch[] batches = await CreateBatchesForAllAvailableItemsInPageAsync(
front,
back,
page,
context,
createCommitItemBatchesAsync);
if (!batches.Any())
{
continue;
}
hasAnyBatchBeenProcessed = true;
DateTime maxCommitTimeStamp = GetMaxCommitTimeStamp(batches);
var unprocessedBatches = batches.ToList();
var processingBatches = new List <CatalogCommitItemBatchTask>();
var exceptions = new List <Exception>();
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
while (processingBatches.Any())
{
var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted)
.Select(batch => batch.Task)
.DefaultIfEmpty(Task.CompletedTask);
await Task.WhenAny(activeTasks);
for (var i = 0; i < processingBatches.Count; ++i)
{
var batch = processingBatches[i];
if (batch.Task.IsFaulted || batch.Task.IsCanceled)
{
hasAnyBatchFailed = true;
if (batch.Task.Exception != null)
{
var exception = ExceptionUtilities.Unwrap(batch.Task.Exception);
exceptions.Add(exception);
}
}
if (batch.Task.IsCompleted)
{
processingBatches.RemoveAt(i);
--i;
}
}
if (!hasAnyBatchFailed)
{
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
}
}
if (hasAnyBatchFailed)
{
foreach (var exception in exceptions)
{
logger.LogError(_eventId, exception, Strings.BatchProcessingFailure);
}
var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions);
throw new BatchProcessingException(innerException);
}
front.Value = maxCommitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation($"{nameof(CatalogCommitUtilities)}.{nameof(ProcessCatalogCommitsAsync)} " +
$"{nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front);
}
return(hasAnyBatchBeenProcessed);
}
