private async Task ConsumeExecutionResultTask(Queue <Task <IEnumerable <string> > > executionTasks, IProgress <BatchAnonymizeProgressDetail> progress) { IEnumerable <string> resultContents = await executionTasks.Dequeue().ConfigureAwait(false); int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false); progress?.Report(new BatchAnonymizeProgressDetail() { ConsumeCompleted = consumeCount }); }
public async Task ExecuteAsync(CancellationToken cancellationToken, IProgress <BatchAnonymizeProgressDetail> progress = null) { List <Task <IEnumerable <TResult> > > executionTasks = new List <Task <IEnumerable <TResult> > >(); List <TSource> batchData = new List <TSource>(); TSource content; while ((content = await RawDataReader.NextAsync().ConfigureAwait(false)) != null) { if (cancellationToken.IsCancellationRequested) { throw new OperationCanceledException(); } batchData.Add(content); if (batchData.Count < BatchSize) { continue; } executionTasks.Add(AnonymizeAsync(batchData, progress, cancellationToken)); batchData = new List <TSource>(); if (executionTasks.Count < PartitionCount) { continue; } await ConsumeExecutionResultTask(executionTasks, progress).ConfigureAwait(false); } if (batchData.Count > 0) { executionTasks.Add(AnonymizeAsync(batchData, progress, cancellationToken)); } while (executionTasks.Count > 0) { await ConsumeExecutionResultTask(executionTasks, progress).ConfigureAwait(false); } if (AnonymizedDataConsumer != null) { await AnonymizedDataConsumer.CompleteAsync().ConfigureAwait(false); } }
private async Task ConsumeExecutionResultTask(List <Task <IEnumerable <TResult> > > executionTasks, IProgress <BatchAnonymizeProgressDetail> progress) { if (KeepOrder) { IEnumerable <TResult> resultContents = await executionTasks.First().ConfigureAwait(false); executionTasks.RemoveAt(0); if (AnonymizedDataConsumer != null) { int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false); progress?.Report(new BatchAnonymizeProgressDetail() { ConsumeCompleted = consumeCount, CurrentThreadId = Thread.CurrentThread.ManagedThreadId }); } } else { await Task.WhenAny(executionTasks).ConfigureAwait(false); for (int index = 0; index < executionTasks.Count; ++index) { if (executionTasks[index].IsCompleted) { IEnumerable <TResult> resultContents = await executionTasks[index].ConfigureAwait(false); executionTasks.RemoveAt(index); if (AnonymizedDataConsumer != null) { int consumeCount = await AnonymizedDataConsumer.ConsumeAsync(resultContents).ConfigureAwait(false); progress?.Report(new BatchAnonymizeProgressDetail() { ConsumeCompleted = consumeCount, CurrentThreadId = Thread.CurrentThread.ManagedThreadId }); } break; // Only consume 1 result from completed task } } } }