internal bool UpdateDocument(IEnumerable <IIndexableField> doc, Analyzer analyzer, Term delTerm)
{
bool hasEvents = PreUpdate();
ThreadState perThread = flushControl.ObtainAndLock();
DocumentsWriterPerThread flushingDWPT;
try
{
if (!perThread.IsActive)
{
EnsureOpen();
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, () => "perThread is not active but we are still open");
}
}
EnsureInitialized(perThread);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsInitialized);
}
DocumentsWriterPerThread dwpt = perThread.dwpt;
int dwptNumDocs = dwpt.NumDocsInRAM;
try
{
dwpt.UpdateDocument(doc, analyzer, delTerm);
numDocsInRAM.IncrementAndGet();
}
finally
{
if (dwpt.CheckAndResetHasAborted())
{
if (dwpt.PendingFilesToDelete.Count > 0)
{
PutEvent(new DeleteNewFilesEvent(dwpt.PendingFilesToDelete));
}
SubtractFlushedNumDocs(dwptNumDocs);
flushControl.DoOnAbort(perThread);
}
}
bool isUpdate = delTerm != null;
flushingDWPT = flushControl.DoAfterDocument(perThread, isUpdate);
}
finally
{
perThread.Unlock();
}
return(PostUpdate(flushingDWPT, hasEvents));
}
private bool DoFlush(DocumentsWriterPerThread flushingDWPT)
{
bool hasEvents = false;
while (flushingDWPT != null)
{
hasEvents = true;
bool success = false;
SegmentFlushTicket ticket = null;
try
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(currentFullFlushDelQueue == null || flushingDWPT.deleteQueue == currentFullFlushDelQueue, () => "expected: " + currentFullFlushDelQueue + "but was: " + flushingDWPT.deleteQueue + " " + flushControl.IsFullFlush);
}
/*
* Since with DWPT the flush process is concurrent and several DWPT
* could flush at the same time we must maintain the order of the
* flushes before we can apply the flushed segment and the frozen global
* deletes it is buffering. The reason for this is that the global
* deletes mark a certain point in time where we took a DWPT out of
* rotation and freeze the global deletes.
*
* Example: A flush 'A' starts and freezes the global deletes, then
* flush 'B' starts and freezes all deletes occurred since 'A' has
* started. if 'B' finishes before 'A' we need to wait until 'A' is done
* otherwise the deletes frozen by 'B' are not applied to 'A' and we
* might miss to deletes documents in 'A'.
*/
try
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue lock
ticket = ticketQueue.AddFlushTicket(flushingDWPT);
int flushingDocsInRam = flushingDWPT.NumDocsInRAM;
bool dwptSuccess = false;
try
{
// flush concurrently without locking
FlushedSegment newSegment = flushingDWPT.Flush();
ticketQueue.AddSegment(ticket, newSegment);
dwptSuccess = true;
}
finally
{
SubtractFlushedNumDocs(flushingDocsInRam);
if (flushingDWPT.PendingFilesToDelete.Count > 0)
{
PutEvent(new DeleteNewFilesEvent(flushingDWPT.PendingFilesToDelete));
hasEvents = true;
}
if (!dwptSuccess)
{
PutEvent(new FlushFailedEvent(flushingDWPT.SegmentInfo));
hasEvents = true;
}
}
// flush was successful once we reached this point - new seg. has been assigned to the ticket!
success = true;
}
finally
{
if (!success && ticket != null)
{
// In the case of a failure make sure we are making progress and
// apply all the deletes since the segment flush failed since the flush
// ticket could hold global deletes see FlushTicket#canPublish()
ticketQueue.MarkTicketFailed(ticket);
}
}
/*
* Now we are done and try to flush the ticket queue if the head of the
* queue has already finished the flush.
*/
if (ticketQueue.TicketCount >= perThreadPool.NumThreadStatesActive)
{
// this means there is a backlog: the one
// thread in innerPurge can't keep up with all
// other threads flushing segments. In this case
// we forcefully stall the producers.
PutEvent(ForcedPurgeEvent.INSTANCE);
break;
}
}
finally
{
flushControl.DoAfterFlush(flushingDWPT);
flushingDWPT.CheckAndResetHasAborted();
}
flushingDWPT = flushControl.NextPendingFlush();
}
if (hasEvents)
{
PutEvent(MergePendingEvent.INSTANCE);
}
// If deletes alone are consuming > 1/2 our RAM
// buffer, force them all to apply now. this is to
// prevent too-frequent flushing of a long tail of
// tiny segments:
double ramBufferSizeMB = config.RAMBufferSizeMB;
if (ramBufferSizeMB != Index.IndexWriterConfig.DISABLE_AUTO_FLUSH && flushControl.DeleteBytesUsed > (1024 * 1024 * ramBufferSizeMB / 2))
{
if (infoStream.IsEnabled("DW"))
{
infoStream.Message("DW", "force apply deletes bytesUsed=" + flushControl.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * ramBufferSizeMB));
}
hasEvents = true;
if (!this.ApplyAllDeletes(deleteQueue))
{
PutEvent(ApplyDeletesEvent.INSTANCE);
}
}
return(hasEvents);
}
private bool DoFlush(DocumentsWriterPerThread flushingDWPT)
{
bool hasEvents = false;
while (flushingDWPT != null)
{
hasEvents = true;
bool success = false;
SegmentFlushTicket ticket = null;
try
{
Debug.Assert(CurrentFullFlushDelQueue == null || flushingDWPT.DeleteQueue == CurrentFullFlushDelQueue, "expected: " + CurrentFullFlushDelQueue + "but was: " + flushingDWPT.DeleteQueue + " " + FlushControl.FullFlush);
/*
* Since with DWPT the flush process is concurrent and several DWPT
* could flush at the same time we must maintain the order of the
* flushes before we can apply the flushed segment and the frozen global
* deletes it is buffering. The reason for this is that the global
* deletes mark a certain point in time where we took a DWPT out of
* rotation and freeze the global deletes.
*
* Example: A flush 'A' starts and freezes the global deletes, then
* flush 'B' starts and freezes all deletes occurred since 'A' has
* started. if 'B' finishes before 'A' we need to wait until 'A' is done
* otherwise the deletes frozen by 'B' are not applied to 'A' and we
* might miss to deletes documents in 'A'.
*/
try
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue lock
ticket = TicketQueue.AddFlushTicket(flushingDWPT);
int flushingDocsInRam = flushingDWPT.NumDocsInRAM;
bool dwptSuccess = false;
try
{
// flush concurrently without locking
FlushedSegment newSegment = flushingDWPT.Flush();
TicketQueue.AddSegment(ticket, newSegment);
dwptSuccess = true;
}
finally
{
SubtractFlushedNumDocs(flushingDocsInRam);
if (flushingDWPT.PendingFilesToDelete().Count > 0)
{
PutEvent(new DeleteNewFilesEvent(flushingDWPT.PendingFilesToDelete()));
hasEvents = true;
}
if (!dwptSuccess)
{
PutEvent(new FlushFailedEvent(flushingDWPT.SegmentInfo));
hasEvents = true;
}
}
// flush was successful once we reached this point - new seg. has been assigned to the ticket!
success = true;
}
finally
{
if (!success && ticket != null)
{
// In the case of a failure make sure we are making progress and
// apply all the deletes since the segment flush failed since the flush
// ticket could hold global deletes see FlushTicket#canPublish()
TicketQueue.MarkTicketFailed(ticket);
}
}
/*
* Now we are done and try to flush the ticket queue if the head of the
* queue has already finished the flush.
*/
if (TicketQueue.TicketCount >= PerThreadPool.ActiveThreadState)
{
// this means there is a backlog: the one
// thread in innerPurge can't keep up with all
// other threads flushing segments. In this case
// we forcefully stall the producers.
PutEvent(ForcedPurgeEvent.INSTANCE);
break;
}
}
finally
{
FlushControl.DoAfterFlush(flushingDWPT);
flushingDWPT.CheckAndResetHasAborted();
}
flushingDWPT = FlushControl.NextPendingFlush();
}
if (hasEvents)
{
PutEvent(MergePendingEvent.INSTANCE);
}
// If deletes alone are consuming > 1/2 our RAM
// buffer, force them all to apply now. this is to
// prevent too-frequent flushing of a long tail of
// tiny segments:
double ramBufferSizeMB = LIWConfig.RAMBufferSizeMB;
if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH && FlushControl.DeleteBytesUsed > (1024 * 1024 * ramBufferSizeMB / 2))
{
if (InfoStream.IsEnabled("DW"))
{
InfoStream.Message("DW", "force apply deletes bytesUsed=" + FlushControl.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * ramBufferSizeMB));
}
hasEvents = true;
if (!this.ApplyAllDeletes(DeleteQueue))
{
PutEvent(ApplyDeletesEvent.INSTANCE);
}
}
return hasEvents;
}