internal virtual SegmentFlushTicket AddFlushTicket(DocumentsWriterPerThread dwpt)
{
lock (this)
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue
// lock
IncTickets();
bool success = false;
try
{
// prepare flush freezes the global deletes - do in synced block!
SegmentFlushTicket ticket = new SegmentFlushTicket(dwpt.PrepareFlush());
queue.AddLast(ticket);
success = true;
return(ticket);
}
finally
{
if (!success)
{
DecTickets();
}
}
}
}
internal virtual void MarkTicketFailed(SegmentFlushTicket ticket)
{
lock (this)
{
// to free the queue we mark tickets as failed just to clean up the queue.
ticket.SetFailed();
}
}
internal virtual void AddSegment(SegmentFlushTicket ticket, FlushedSegment segment)
{
lock (this)
{
// the actual flush is done asynchronously and once done the FlushedSegment
// is passed to the flush ticket
ticket.SetSegment(segment);
}
}
internal virtual void MarkTicketFailed(SegmentFlushTicket ticket)
{
UninterruptableMonitor.Enter(this);
try
{
// to free the queue we mark tickets as failed just to clean up the queue.
ticket.SetFailed();
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
internal virtual void AddSegment(SegmentFlushTicket ticket, FlushedSegment segment)
{
UninterruptableMonitor.Enter(this);
try
{
// the actual flush is done asynchronously and once done the FlushedSegment
// is passed to the flush ticket
ticket.SetSegment(segment);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
private bool DoFlush(DocumentsWriterPerThread flushingDWPT)
{
bool hasEvents = false;
while (flushingDWPT != null)
{
hasEvents = true;
bool success = false;
SegmentFlushTicket ticket = null;
try
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(currentFullFlushDelQueue == null || flushingDWPT.deleteQueue == currentFullFlushDelQueue, () => "expected: " + currentFullFlushDelQueue + "but was: " + flushingDWPT.deleteQueue + " " + flushControl.IsFullFlush);
}
/*
* Since with DWPT the flush process is concurrent and several DWPT
* could flush at the same time we must maintain the order of the
* flushes before we can apply the flushed segment and the frozen global
* deletes it is buffering. The reason for this is that the global
* deletes mark a certain point in time where we took a DWPT out of
* rotation and freeze the global deletes.
*
* Example: A flush 'A' starts and freezes the global deletes, then
* flush 'B' starts and freezes all deletes occurred since 'A' has
* started. if 'B' finishes before 'A' we need to wait until 'A' is done
* otherwise the deletes frozen by 'B' are not applied to 'A' and we
* might miss to deletes documents in 'A'.
*/
try
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue lock
ticket = ticketQueue.AddFlushTicket(flushingDWPT);
int flushingDocsInRam = flushingDWPT.NumDocsInRAM;
bool dwptSuccess = false;
try
{
// flush concurrently without locking
FlushedSegment newSegment = flushingDWPT.Flush();
ticketQueue.AddSegment(ticket, newSegment);
dwptSuccess = true;
}
finally
{
SubtractFlushedNumDocs(flushingDocsInRam);
if (flushingDWPT.PendingFilesToDelete.Count > 0)
{
PutEvent(new DeleteNewFilesEvent(flushingDWPT.PendingFilesToDelete));
hasEvents = true;
}
if (!dwptSuccess)
{
PutEvent(new FlushFailedEvent(flushingDWPT.SegmentInfo));
hasEvents = true;
}
}
// flush was successful once we reached this point - new seg. has been assigned to the ticket!
success = true;
}
finally
{
if (!success && ticket != null)
{
// In the case of a failure make sure we are making progress and
// apply all the deletes since the segment flush failed since the flush
// ticket could hold global deletes see FlushTicket#canPublish()
ticketQueue.MarkTicketFailed(ticket);
}
}
/*
* Now we are done and try to flush the ticket queue if the head of the
* queue has already finished the flush.
*/
if (ticketQueue.TicketCount >= perThreadPool.NumThreadStatesActive)
{
// this means there is a backlog: the one
// thread in innerPurge can't keep up with all
// other threads flushing segments. In this case
// we forcefully stall the producers.
PutEvent(ForcedPurgeEvent.INSTANCE);
break;
}
}
finally
{
flushControl.DoAfterFlush(flushingDWPT);
flushingDWPT.CheckAndResetHasAborted();
}
flushingDWPT = flushControl.NextPendingFlush();
}
if (hasEvents)
{
PutEvent(MergePendingEvent.INSTANCE);
}
// If deletes alone are consuming > 1/2 our RAM
// buffer, force them all to apply now. this is to
// prevent too-frequent flushing of a long tail of
// tiny segments:
double ramBufferSizeMB = config.RAMBufferSizeMB;
if (ramBufferSizeMB != Index.IndexWriterConfig.DISABLE_AUTO_FLUSH && flushControl.DeleteBytesUsed > (1024 * 1024 * ramBufferSizeMB / 2))
{
if (infoStream.IsEnabled("DW"))
{
infoStream.Message("DW", "force apply deletes bytesUsed=" + flushControl.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * ramBufferSizeMB));
}
hasEvents = true;
if (!this.ApplyAllDeletes(deleteQueue))
{
PutEvent(ApplyDeletesEvent.INSTANCE);
}
}
return(hasEvents);
}
internal virtual SegmentFlushTicket AddFlushTicket(DocumentsWriterPerThread dwpt)
{
lock (this)
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue
// lock
IncTickets();
bool success = false;
try
{
// prepare flush freezes the global deletes - do in synced block!
SegmentFlushTicket ticket = new SegmentFlushTicket(dwpt.PrepareFlush());
Queue.AddLast(ticket);
success = true;
return ticket;
}
finally
{
if (!success)
{
DecTickets();
}
}
}
}
internal virtual void MarkTicketFailed(SegmentFlushTicket ticket)
{
lock (this)
{
// to free the queue we mark tickets as failed just to clean up the queue.
ticket.SetFailed();
}
}
internal virtual void AddSegment(SegmentFlushTicket ticket, FlushedSegment segment)
{
lock (this)
{
// the actual flush is done asynchronously and once done the FlushedSegment
// is passed to the flush ticket
ticket.Segment = segment;
}
}
