private TikaAsyncWorker pushPop(PipelineContext ctx, IDatasourceSink sink, TikaAsyncWorker newElt)
{
try
{
return((TikaAsyncWorker)((newElt == null) ? workerQueue.Pop() : workerQueue.PushAndOptionalPop(newElt)));
}
catch (Exception e)
{
ctx.HandleException(e);
return(null);
}
}
public override int CallNextPostProcessor(PipelineContext ctx)
{
ctx.PostProcessor = this;
ReportStart(ctx);
if (mapper != null)
{
ctx.ImportLog.Log(_LogType.ltTimerStart, "Reduce phase maxparallel={0}, fanout={1}.", readMaxParallel, fanOut);
AsyncRequestQueue q = (readMaxParallel == 0 || fanOut <= 1) ? null : AsyncRequestQueue.Create(readMaxParallel);
MappedObjectEnumerator e;
int i;
if (q == null)
{
for (i = 0; true; i++)
{
e = mapper.GetObjectEnumerator(i);
if (e == null)
{
break;
}
enumeratePartialAndClose(ctx, e, i);
}
}
else
{
//Push enum requests into the Q and process the results
for (i = 0; true; i++)
{
var x = q.PushAndOptionalPop(new AsyncRequestElement(i, getEnum));
if (x == null)
{
continue;
}
e = (MappedObjectEnumerator)x.Result;
if (e == null)
{
break;
}
enumeratePartialAndClose(ctx, e, i);
}
//Pop all existing from the Q and process them
while (true)
{
var x = q.Pop();
if (x == null)
{
break;
}
e = (MappedObjectEnumerator)x.Result;
if (e == null)
{
continue;
}
;
enumeratePartialAndClose(ctx, e, i++);
}
}
}
ReportEnd(ctx);
ctx.ImportLog.Log(_LogType.ltTimerStop, "Reduce phase ended.");
Utils.FreeAndNil(ref mapper);
return(base.CallNextPostProcessor(ctx));
}