public void FlushCache()
{
if (buffer.Count == 0)
{
return;
}
asyncQ.PushAndOptionalPop(new AsyncRequestElement(buffer, asyncAdd));
buffer = new List <JObject>();
}
private TikaAsyncWorker pushPop(PipelineContext ctx, IDatasourceSink sink, TikaAsyncWorker newElt)
{
try
{
return((TikaAsyncWorker)((newElt == null) ? workerQueue.Pop() : workerQueue.PushAndOptionalPop(newElt)));
}
catch (Exception e)
{
ctx.HandleException(e);
return(null);
}
}
public override void Add(PipelineContext ctx)
{
if ((ctx.ImportFlags & _ImportFlags.TraceValues) != 0)
{
ctx.DebugLog.Log("Add: accumulator.Count={0}", accumulator.Count);
}
if (accumulator.Count == 0)
{
return;
}
OptLogAdd();
if (DocType.AutoTimestampFieldName != null)
{
accumulator[DocType.AutoTimestampFieldName] = DateTime.UtcNow;
}
if (cache == null)
{
if (asyncQ == null)
{
Connection.Post(DocType.GetUrlForAdd(accumulator), accumulator, _recordSerializer).ThrowIfError();
}
else
{
asyncQ.PushAndOptionalPop(new AsyncRequestElement(accumulator, asyncAdd));
}
}
else
{
cache.Add(new ESBulkEntry(accumulator));
if (cache.Count >= cacheSize)
{
FlushCache();
}
}
Clear();
}
public override int CallNextPostProcessor(PipelineContext ctx)
{
ctx.PostProcessor = this;
ReportStart(ctx);
if (mapper != null)
{
ctx.ImportLog.Log(_LogType.ltTimerStart, "Reduce phase maxparallel={0}, fanout={1}.", readMaxParallel, fanOut);
AsyncRequestQueue q = (readMaxParallel == 0 || fanOut <= 1) ? null : AsyncRequestQueue.Create(readMaxParallel);
MappedObjectEnumerator e;
int i;
if (q == null)
{
for (i = 0; true; i++)
{
e = mapper.GetObjectEnumerator(i);
if (e == null)
{
break;
}
enumeratePartialAndClose(ctx, e, i);
}
}
else
{
//Push enum requests into the Q and process the results
for (i = 0; true; i++)
{
var x = q.PushAndOptionalPop(new AsyncRequestElement(i, getEnum));
if (x == null)
{
continue;
}
e = (MappedObjectEnumerator)x.Result;
if (e == null)
{
break;
}
enumeratePartialAndClose(ctx, e, i);
}
//Pop all existing from the Q and process them
while (true)
{
var x = q.Pop();
if (x == null)
{
break;
}
e = (MappedObjectEnumerator)x.Result;
if (e == null)
{
continue;
}
;
enumeratePartialAndClose(ctx, e, i++);
}
}
}
ReportEnd(ctx);
ctx.ImportLog.Log(_LogType.ltTimerStop, "Reduce phase ended.");
Utils.FreeAndNil(ref mapper);
return(base.CallNextPostProcessor(ctx));
}
