private void ScheduleWorkers(
JobCancellationUnit cancellationUnit,
BlockingCollection <T> stream)
{
var indexTasks = new List <Task>();
try
{
while (stream.PossiblyMoreInStream())
{
T currentItem;
if (stream.TryTake(out currentItem, 5 * 1000, cancellationUnit.Token))
{
var sinkWorkerTask = new Task(this.GetWorker(currentItem));
sinkWorkerTask.Start();
indexTasks.Add(sinkWorkerTask);
}
}
Task.WaitAll(indexTasks.ToArray());
}
catch (Exception)
{
// logging?
// would be done anyway by TPL, but to be explicit about things...
cancellationUnit.Cancel();
throw;
}
}
protected override void Transform(
JobCancellationUnit cancellationUnit,
BlockingCollection <EsDocument> source,
BlockingCollection <List <EsDocument> > transformations)
{
var currentBatch = new List <EsDocument>(_batchSize);
while (source.PossiblyMoreInStream())
{
// close the current batch if its full
if (currentBatch.Count() == _batchSize)
{
transformations.Add(currentBatch);
currentBatch = new List <EsDocument>(_batchSize);
}
// add another doc to the current batch
EsDocument currentDoc;
if (source.TryTake(out currentDoc, 5 * 1000, cancellationUnit.Token))
{
currentBatch.Add(currentDoc);
}
this.ThrowIfSuccessorCancelled(cancellationUnit, source);
}
// ensure all docs are pushed when total number is not exact multiple
// of batch size.
if (currentBatch.Any())
{
transformations.Add(currentBatch);
}
}
