private void ScheduleWorkers(
JobCancellationUnit cancellationUnit,
BlockingCollection <T> stream)
{
var indexTasks = new List <Task>();
try
{
while (stream.PossiblyMoreInStream())
{
T currentItem;
if (stream.TryTake(out currentItem, 5 * 1000, cancellationUnit.Token))
{
var sinkWorkerTask = new Task(this.GetWorker(currentItem));
sinkWorkerTask.Start();
indexTasks.Add(sinkWorkerTask);
}
}
Task.WaitAll(indexTasks.ToArray());
}
catch (Exception)
{
// logging?
// would be done anyway by TPL, but to be explicit about things...
cancellationUnit.Cancel();
throw;
}
}
protected override void Transform(
JobCancellationUnit cancellationUnit,
BlockingCollection <EsDocument> source,
BlockingCollection <List <EsDocument> > transformations)
{
var currentBatch = new List <EsDocument>(_batchSize);
while (source.PossiblyMoreInStream())
{
// close the current batch if its full
if (currentBatch.Count() == _batchSize)
{
transformations.Add(currentBatch);
currentBatch = new List <EsDocument>(_batchSize);
}
// add another doc to the current batch
EsDocument currentDoc;
if (source.TryTake(out currentDoc, 5 * 1000, cancellationUnit.Token))
{
currentBatch.Add(currentDoc);
}
this.ThrowIfSuccessorCancelled(cancellationUnit, source);
}
// ensure all docs are pushed when total number is not exact multiple
// of batch size.
if (currentBatch.Any())
{
transformations.Add(currentBatch);
}
}
private void ScheduleWorkers(
JobCancellationUnit cancellationUnit,
BlockingCollection <T> stream)
{
try
{
// single task / synchronous workers pattern...
this.ThrowIfSuccessorCancelled(cancellationUnit);
GetWorker()
(stream,
() => this.ThrowIfSuccessorCancelled(cancellationUnit));
}
catch (TaskCanceledException)
{
// our sucessors should have stopped, but lets dot the i's...
throw;
}
catch (Exception ex)
{
// because we're the first in the job, don't cause a cancel if we
// fail, just say we're finished. This gives everything further down
// stream a chance to process all the data we've gathered so far.
// TODO: logging
throw; // currently throwing. though I think this will cause a cancel
// on the token. but we do want to report the exceptioned status of this
// task, but just not cause everything else to stop immediately.
}
finally
{
stream.CompleteAdding();
}
}
public Task StartDrainingAsync(
JobCancellationUnit cancellationUnit,
BlockingCollection <T> stream)
{
var indexingTask = new Task(
() => this.ScheduleWorkers(cancellationUnit, stream));
return(indexingTask);
}
private void ThrowIfSuccessorCancelled(
JobCancellationUnit cancellationUnit)
{
// if something cancelled the job, and we have no predecessor, so we can only
// assume that is must have been a successor, so we must end.
if (cancellationUnit.IsCancellationRequested)
{
cancellationUnit.ThrowIfCancelled();
}
}
public BlockingCollection <T> StartFlowingToEnd(
JobCancellationUnit cancellationUnit)
{
var stream = new BlockingCollection <T>();
var streamTask = new Task(
() => this.ScheduleWorkers(cancellationUnit, stream),
cancellationUnit.Token);
streamTask.Start();
return(stream);
}
