public async Task Collect(CollectTask collectTask)
{
var blockFactory = new BlockFactory(collectTask);
var blockController = new BlockController();
blockController.Trace = DataCollector.Trace;
blockFactory.Trace = DataCollector.Trace;
// Create repo for write results
var inputRepo = getRepository(collectTask.Input);
var outputRepo = getRepository(collectTask.Output);
var inputBufferBlock = blockFactory.InputBuffer();
var bacthBlock = blockFactory.Batch<string>();
var processBlock = blockFactory.Process();
var outputBufferBlock = blockFactory.OutputBuffer();
var outputBlock = blockFactory.WriteResults(outputRepo);
blockController.LinkWithCompletion(inputBufferBlock, bacthBlock);
blockController.LinkWithCompletion(bacthBlock, processBlock);
blockController.LinkWithCompletion(processBlock, outputBufferBlock);
blockController.LinkWithCompletion(outputBufferBlock, outputBlock);
// Read data and send to blocks
foreach (var item in inputRepo.GetInputData())
{
await inputBufferBlock.SendAsync(item);
collectTask.AllItems++;
if (collectTask.AllItems % 524288 == 0)
Thread.Sleep(TimeSpan.FromMinutes(15));
if (collectTask.CancellationSource.IsCancellationRequested)
break;
}
inputBufferBlock.Complete();
await outputBlock.Completion;
inputRepo.Dispose();
outputRepo.Dispose();
}
