private async Task ConsumeExecutionResultTask(Queue <Task <IEnumerable <string> > > executionTasks, IProgress <BatchAnonymizeProgressDetail> progress)
{
IEnumerable <string> resultContents = await executionTasks.Dequeue().ConfigureAwait(false);
int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false);
progress?.Report(new BatchAnonymizeProgressDetail()
{
ConsumeCompleted = consumeCount
});
}
public async Task ExecuteAsync(CancellationToken cancellationToken, IProgress <BatchAnonymizeProgressDetail> progress = null)
{
List <Task <IEnumerable <TResult> > > executionTasks = new List <Task <IEnumerable <TResult> > >();
List <TSource> batchData = new List <TSource>();
TSource content;
while ((content = await RawDataReader.NextAsync().ConfigureAwait(false)) != null)
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException();
}
batchData.Add(content);
if (batchData.Count < BatchSize)
{
continue;
}
executionTasks.Add(AnonymizeAsync(batchData, progress, cancellationToken));
batchData = new List <TSource>();
if (executionTasks.Count < PartitionCount)
{
continue;
}
await ConsumeExecutionResultTask(executionTasks, progress).ConfigureAwait(false);
}
if (batchData.Count > 0)
{
executionTasks.Add(AnonymizeAsync(batchData, progress, cancellationToken));
}
while (executionTasks.Count > 0)
{
await ConsumeExecutionResultTask(executionTasks, progress).ConfigureAwait(false);
}
if (AnonymizedDataConsumer != null)
{
await AnonymizedDataConsumer.CompleteAsync().ConfigureAwait(false);
}
}
private async Task ConsumeExecutionResultTask(List <Task <IEnumerable <TResult> > > executionTasks, IProgress <BatchAnonymizeProgressDetail> progress)
{
if (KeepOrder)
{
IEnumerable <TResult> resultContents = await executionTasks.First().ConfigureAwait(false);
executionTasks.RemoveAt(0);
if (AnonymizedDataConsumer != null)
{
int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false);
progress?.Report(new BatchAnonymizeProgressDetail()
{
ConsumeCompleted = consumeCount, CurrentThreadId = Thread.CurrentThread.ManagedThreadId
});
}
}
else
{
await Task.WhenAny(executionTasks).ConfigureAwait(false);
for (int index = 0; index < executionTasks.Count; ++index)
{
if (executionTasks[index].IsCompleted)
{
IEnumerable <TResult> resultContents = await executionTasks[index].ConfigureAwait(false);
executionTasks.RemoveAt(index);
if (AnonymizedDataConsumer != null)
{
int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false);
progress?.Report(new BatchAnonymizeProgressDetail()
{
ConsumeCompleted = consumeCount, CurrentThreadId = Thread.CurrentThread.ManagedThreadId
});
}
break; // Only consume 1 result from completed task
}
}
}
}
