/// <summary> /// Starts reading all the sets that are available. /// Returns whenever the whole pipeline is complete. /// </summary> public Task <HarvesterResult> Run(CancellationToken?cancellationToken = null) { var cToken = cancellationToken ?? CancellationToken.None; //Destination.ConsumeAsync(cToken); var parallelOptions = new ParallelOptions() { MaxDegreeOfParallelism = (int)ThreadCount }; parallelOptions.CancellationToken = cToken; ResetStopwatch(); _stopwatch.Start(); int shardsUsed; var flowCompletion = Destination.FlowCompletion(); var totalItemsUsed = ProcessInputShards(parallelOptions, out shardsUsed); //Let the dest know that we're finished passing data to it, so that it could complete. Destination.Complete(); return(flowCompletion.ContinueWith(continuationFunction: (t) => { _stopwatch.Stop(); var output = new HarvesterResult(shardsUsed, totalItemsUsed); return output; }, cancellationToken: cToken)); }