/// <summary>
/// Starts reading all the sets that are available.
/// Returns whenever the whole pipeline is complete.
/// </summary>
public Task <HarvesterResult> Run(CancellationToken?cancellationToken = null)
{
var cToken = cancellationToken ?? CancellationToken.None;
//Destination.ConsumeAsync(cToken);
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)ThreadCount
};
parallelOptions.CancellationToken = cToken;
ResetStopwatch();
_stopwatch.Start();
int shardsUsed;
var flowCompletion = Destination.FlowCompletion();
var totalItemsUsed = ProcessInputShards(parallelOptions, out shardsUsed);
//Let the dest know that we're finished passing data to it, so that it could complete.
Destination.Complete();
return(flowCompletion.ContinueWith(continuationFunction: (t) =>
{
_stopwatch.Stop();
var output = new HarvesterResult(shardsUsed, totalItemsUsed);
return output;
}, cancellationToken: cToken));
}
/// <summary>
/// Reads all values from the source, in raw means without any pre-processing.
/// </summary>
/// <param name="target">A target block to which to post all input.</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task <HarvesterResult> ReadAll(
ITargetBlock <ExpandoObject> target,
CancellationToken?cancellationToken = null)
{
var cToken = cancellationToken ?? CancellationToken.None;
var parallelOptions = new ParallelOptions()
{
MaxDegreeOfParallelism = (int)ThreadCount
};
parallelOptions.CancellationToken = cToken;
ResetStopwatch();
_stopwatch.Start();
int shardsUsed;
var totalItemsUsed = ProcessInputShardsRaw(target, parallelOptions, out shardsUsed);
target.Complete();
return(target.Completion.ContinueWith(x =>
{
_stopwatch.Stop();
var output = new HarvesterResult(shardsUsed, totalItemsUsed);
return output;
}));
}
