internal virtual SegmentFlushTicket AddFlushTicket(DocumentsWriterPerThread dwpt)
{
lock (this)
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue
// lock
IncTickets();
bool success = false;
try
{
// prepare flush freezes the global deletes - do in synced block!
SegmentFlushTicket ticket = new SegmentFlushTicket(dwpt.PrepareFlush());
Queue.AddLast(ticket);
success = true;
return ticket;
}
finally
{
if (!success)
{
DecTickets();
}
}
}
}
public TermsHash(DocumentsWriterPerThread docWriter, TermsHashConsumer consumer, bool trackAllocations, TermsHash nextTermsHash)
{
this.DocState = docWriter.docState;
this.Consumer = consumer;
this.TrackAllocations = trackAllocations;
this.NextTermsHash = nextTermsHash;
this.BytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.NewCounter();
IntPool = new IntBlockPool(docWriter.intBlockAllocator);
BytePool = new ByteBlockPool(docWriter.ByteBlockAllocator);
if (nextTermsHash != null)
{
// We are primary
Primary = true;
TermBytePool = BytePool;
nextTermsHash.TermBytePool = BytePool;
}
else
{
Primary = false;
}
}
private bool PostUpdate(DocumentsWriterPerThread flushingDWPT, bool hasEvents)
{
hasEvents |= ApplyAllDeletes(DeleteQueue);
if (flushingDWPT != null)
{
hasEvents |= DoFlush(flushingDWPT);
}
else
{
DocumentsWriterPerThread nextPendingFlush = FlushControl.NextPendingFlush();
if (nextPendingFlush != null)
{
hasEvents |= DoFlush(nextPendingFlush);
}
}
return hasEvents;
}
public StoredFieldsProcessor(DocumentsWriterPerThread docWriter)
{
this.DocWriter = docWriter;
this.DocState = docWriter.docState;
this.Codec = docWriter.Codec;
}
internal void Reset()
{
//Debug.Assert(this.HeldByCurrentThread);
this.Dwpt = null;
this.BytesUsed = 0;
this.FlushPending_Renamed = false;
}
private bool DoFlush(DocumentsWriterPerThread flushingDWPT)
{
bool hasEvents = false;
while (flushingDWPT != null)
{
hasEvents = true;
bool success = false;
SegmentFlushTicket ticket = null;
try
{
Debug.Assert(CurrentFullFlushDelQueue == null || flushingDWPT.DeleteQueue == CurrentFullFlushDelQueue, "expected: " + CurrentFullFlushDelQueue + "but was: " + flushingDWPT.DeleteQueue + " " + FlushControl.FullFlush);
/*
* Since with DWPT the flush process is concurrent and several DWPT
* could flush at the same time we must maintain the order of the
* flushes before we can apply the flushed segment and the frozen global
* deletes it is buffering. The reason for this is that the global
* deletes mark a certain point in time where we took a DWPT out of
* rotation and freeze the global deletes.
*
* Example: A flush 'A' starts and freezes the global deletes, then
* flush 'B' starts and freezes all deletes occurred since 'A' has
* started. if 'B' finishes before 'A' we need to wait until 'A' is done
* otherwise the deletes frozen by 'B' are not applied to 'A' and we
* might miss to deletes documents in 'A'.
*/
try
{
// Each flush is assigned a ticket in the order they acquire the ticketQueue lock
ticket = TicketQueue.AddFlushTicket(flushingDWPT);
int flushingDocsInRam = flushingDWPT.NumDocsInRAM;
bool dwptSuccess = false;
try
{
// flush concurrently without locking
FlushedSegment newSegment = flushingDWPT.Flush();
TicketQueue.AddSegment(ticket, newSegment);
dwptSuccess = true;
}
finally
{
SubtractFlushedNumDocs(flushingDocsInRam);
if (flushingDWPT.PendingFilesToDelete().Count > 0)
{
PutEvent(new DeleteNewFilesEvent(flushingDWPT.PendingFilesToDelete()));
hasEvents = true;
}
if (!dwptSuccess)
{
PutEvent(new FlushFailedEvent(flushingDWPT.SegmentInfo));
hasEvents = true;
}
}
// flush was successful once we reached this point - new seg. has been assigned to the ticket!
success = true;
}
finally
{
if (!success && ticket != null)
{
// In the case of a failure make sure we are making progress and
// apply all the deletes since the segment flush failed since the flush
// ticket could hold global deletes see FlushTicket#canPublish()
TicketQueue.MarkTicketFailed(ticket);
}
}
/*
* Now we are done and try to flush the ticket queue if the head of the
* queue has already finished the flush.
*/
if (TicketQueue.TicketCount >= PerThreadPool.ActiveThreadState)
{
// this means there is a backlog: the one
// thread in innerPurge can't keep up with all
// other threads flushing segments. In this case
// we forcefully stall the producers.
PutEvent(ForcedPurgeEvent.INSTANCE);
break;
}
}
finally
{
FlushControl.DoAfterFlush(flushingDWPT);
flushingDWPT.CheckAndResetHasAborted();
}
flushingDWPT = FlushControl.NextPendingFlush();
}
if (hasEvents)
{
PutEvent(MergePendingEvent.INSTANCE);
}
// If deletes alone are consuming > 1/2 our RAM
// buffer, force them all to apply now. this is to
// prevent too-frequent flushing of a long tail of
// tiny segments:
double ramBufferSizeMB = LIWConfig.RAMBufferSizeMB;
if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH && FlushControl.DeleteBytesUsed > (1024 * 1024 * ramBufferSizeMB / 2))
{
if (InfoStream.IsEnabled("DW"))
{
InfoStream.Message("DW", "force apply deletes bytesUsed=" + FlushControl.DeleteBytesUsed + " vs ramBuffer=" + (1024 * 1024 * ramBufferSizeMB));
}
hasEvents = true;
if (!this.ApplyAllDeletes(DeleteQueue))
{
PutEvent(ApplyDeletesEvent.INSTANCE);
}
}
return hasEvents;
}
internal virtual void Recycle(DocumentsWriterPerThread dwpt)
{
// don't recycle DWPT by default
}
public ThreadState(DocumentsWriterPerThread dpwt)
{
this.Dwpt = dpwt;
}
public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread)
{
/*
this is the current indexing chain:
DocConsumer / DocConsumerPerThread
--> code: DocFieldProcessor
--> DocFieldConsumer / DocFieldConsumerPerField
--> code: DocFieldConsumers / DocFieldConsumersPerField
--> code: DocInverter / DocInverterPerField
--> InvertedDocConsumer / InvertedDocConsumerPerField
--> code: TermsHash / TermsHashPerField
--> TermsHashConsumer / TermsHashConsumerPerField
--> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
--> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
--> InvertedDocEndConsumer / InvertedDocConsumerPerField
--> code: NormsConsumer / NormsConsumerPerField
--> StoredFieldsConsumer
--> TwoStoredFieldConsumers
-> code: StoredFieldsProcessor
-> code: DocValuesProcessor
*/
// Build up indexing chain:
TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true, new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
NormsConsumer normsWriter = new NormsConsumer();
DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(new StoredFieldsProcessor(documentsWriterPerThread), new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
}
internal abstract DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread);
public TermVectorsConsumer(DocumentsWriterPerThread docWriter)
{
this.DocWriter = docWriter;
DocState = docWriter.docState;
}
internal DocState(DocumentsWriterPerThread docWriter, InfoStream infoStream)
{
this.DocWriter = docWriter;
this.InfoStream = infoStream;
}
internal virtual void Recycle(DocumentsWriterPerThread dwpt)
{
// don't recycle DWPT by default
}
internal BlockedFlush(DocumentsWriterPerThread dwpt, long bytes)
: base()
{
this.Dwpt = dwpt;
this.Bytes = bytes;
}
// Does not implement anything - used only for type checking on IndexWriterConfig.
internal override DocConsumer GetChain(DocumentsWriterPerThread documentsWriter)
{
return null;
}
internal ThreadState(DocumentsWriterPerThread dpwt)
{
this.dwpt = dpwt;
}
public StoredFieldsProcessor(DocumentsWriterPerThread docWriter)
{
this.docWriter = docWriter;
this.docState = docWriter.docState;
this.codec = docWriter.codec;
}
public abstract DocConsumer GetChain(DocumentsWriterPerThread documentsWriterPerThread);
public DocInverter(DocumentsWriterPerThread.DocState docState, InvertedDocConsumer consumer, InvertedDocEndConsumer endConsumer)
{
this.DocState = docState;
this.Consumer = consumer;
this.EndConsumer = endConsumer;
}
// Does not implement anything - used only for type checking on IndexWriterConfig.
public override DocConsumer GetChain(DocumentsWriterPerThread documentsWriter)
{
return null;
}
internal DocState(DocumentsWriterPerThread docWriter, InfoStream infoStream)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
}
