public void StartProcessingBatchesIfNoFailures_WhenProcessCommitItemBatchAsyncIsNull_Throws()
{
const ProcessCommitItemBatchAsync processCommitItemBatchAsync = null;
var exception = Assert.Throws <ArgumentNullException>(
() => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures(
new CollectorHttpClient(),
new JObject(),
new List <CatalogCommitItemBatch>(),
new List <CatalogCommitItemBatchTask>(),
_maxConcurrentBatches,
processCommitItemBatchAsync,
CancellationToken.None));
Assert.Equal("processCommitItemBatchAsync", exception.ParamName);
}
public static void StartProcessingBatchesIfNoFailures(
CollectorHttpClient client,
JToken context,
List <CatalogCommitItemBatch> unprocessedBatches,
List <CatalogCommitItemBatchTask> processingBatches,
int maxConcurrentBatches,
ProcessCommitItemBatchAsync processCommitItemBatchAsync,
CancellationToken cancellationToken)
{
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
if (unprocessedBatches == null)
{
throw new ArgumentNullException(nameof(unprocessedBatches));
}
if (processingBatches == null)
{
throw new ArgumentNullException(nameof(processingBatches));
}
if (maxConcurrentBatches < 1)
{
throw new ArgumentOutOfRangeException(
nameof(maxConcurrentBatches),
maxConcurrentBatches,
string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue));
}
if (processCommitItemBatchAsync == null)
{
throw new ArgumentNullException(nameof(processCommitItemBatchAsync));
}
var hasAnyBatchFailed = processingBatches.Any(batch => batch.Task.IsFaulted || batch.Task.IsCanceled);
if (hasAnyBatchFailed)
{
return;
}
var batchesToEnqueue = Math.Min(
maxConcurrentBatches - processingBatches.Count(batch => !batch.Task.IsCompleted),
unprocessedBatches.Count);
for (var i = 0; i < batchesToEnqueue; ++i)
{
var batch = unprocessedBatches[0];
unprocessedBatches.RemoveAt(0);
var task = processCommitItemBatchAsync(
client,
context,
batch.Key,
batch,
lastBatch: null,
cancellationToken: cancellationToken);
var batchTask = new CatalogCommitItemBatchTask(batch, task);
processingBatches.Add(batchTask);
}
}
internal static async Task <bool> ProcessCatalogCommitsAsync(
CollectorHttpClient client,
ReadWriteCursor front,
ReadCursor back,
FetchCatalogCommitsAsync fetchCatalogCommitsAsync,
CreateCommitItemBatchesAsync createCommitItemBatchesAsync,
ProcessCommitItemBatchAsync processCommitItemBatchAsync,
int maxConcurrentBatches,
ILogger logger,
CancellationToken cancellationToken)
{
var rootItems = await fetchCatalogCommitsAsync(client, front, back, cancellationToken);
var hasAnyBatchFailed = false;
var hasAnyBatchBeenProcessed = false;
foreach (CatalogCommit rootItem in rootItems)
{
JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken);
var context = (JObject)page["@context"];
CatalogCommitItemBatch[] batches = await CreateBatchesForAllAvailableItemsInPageAsync(
front,
back,
page,
context,
createCommitItemBatchesAsync);
if (!batches.Any())
{
continue;
}
hasAnyBatchBeenProcessed = true;
DateTime maxCommitTimeStamp = GetMaxCommitTimeStamp(batches);
var unprocessedBatches = batches.ToList();
var processingBatches = new List <CatalogCommitItemBatchTask>();
var exceptions = new List <Exception>();
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
while (processingBatches.Any())
{
var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted)
.Select(batch => batch.Task)
.DefaultIfEmpty(Task.CompletedTask);
await Task.WhenAny(activeTasks);
for (var i = 0; i < processingBatches.Count; ++i)
{
var batch = processingBatches[i];
if (batch.Task.IsFaulted || batch.Task.IsCanceled)
{
hasAnyBatchFailed = true;
if (batch.Task.Exception != null)
{
var exception = ExceptionUtilities.Unwrap(batch.Task.Exception);
exceptions.Add(exception);
}
}
if (batch.Task.IsCompleted)
{
processingBatches.RemoveAt(i);
--i;
}
}
if (!hasAnyBatchFailed)
{
StartProcessingBatchesIfNoFailures(
client,
context,
unprocessedBatches,
processingBatches,
maxConcurrentBatches,
processCommitItemBatchAsync,
cancellationToken);
}
}
if (hasAnyBatchFailed)
{
foreach (var exception in exceptions)
{
logger.LogError(_eventId, exception, Strings.BatchProcessingFailure);
}
var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions);
throw new BatchProcessingException(innerException);
}
front.Value = maxCommitTimeStamp;
await front.SaveAsync(cancellationToken);
Trace.TraceInformation($"{nameof(CatalogCommitUtilities)}.{nameof(ProcessCatalogCommitsAsync)} " +
$"{nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front);
}
return(hasAnyBatchBeenProcessed);
}
