コード例 #1
0
		public async Task Collect(CollectTask collectTask)
		{
			var blockFactory = new BlockFactory(collectTask);
			var blockController = new BlockController();
			blockController.Trace = DataCollector.Trace;
			blockFactory.Trace = DataCollector.Trace;

			// Create repo for write results
			var inputRepo = getRepository(collectTask.Input);
			var outputRepo = getRepository(collectTask.Output);

			var inputBufferBlock = blockFactory.InputBuffer();
			var bacthBlock = blockFactory.Batch<string>();
			var processBlock = blockFactory.Process();
			var outputBufferBlock = blockFactory.OutputBuffer();
			var outputBlock = blockFactory.WriteResults(outputRepo);

			blockController.LinkWithCompletion(inputBufferBlock, bacthBlock);
			blockController.LinkWithCompletion(bacthBlock, processBlock);
			blockController.LinkWithCompletion(processBlock, outputBufferBlock);
			blockController.LinkWithCompletion(outputBufferBlock, outputBlock);

			// Read data and send to blocks
			foreach (var item in inputRepo.GetInputData())
			{
				await inputBufferBlock.SendAsync(item);
				collectTask.AllItems++;

                if (collectTask.AllItems % 524288 == 0)
                    Thread.Sleep(TimeSpan.FromMinutes(15));

				if (collectTask.CancellationSource.IsCancellationRequested)
					break;
			}
			inputBufferBlock.Complete();

			await outputBlock.Completion;

			inputRepo.Dispose();
			outputRepo.Dispose();
		}