public DocumentsWriterPerThread(string segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue, FieldInfos.Builder fieldInfos) { this.directoryOrig = directory; this.directory = new TrackingDirectoryWrapper(directory); this.fieldInfos = fieldInfos; this.indexWriterConfig = indexWriterConfig; this.infoStream = infoStream; this.codec = indexWriterConfig.Codec; this.docState = new DocState(this, infoStream); this.docState.similarity = indexWriterConfig.Similarity; bytesUsed = Counter.NewCounter(); byteBlockAllocator = new DirectTrackingAllocator(bytesUsed); pendingUpdates = new BufferedUpdates(); intBlockAllocator = new Int32BlockAllocator(bytesUsed); this.deleteQueue = deleteQueue; Debug.Assert(numDocsInRAM == 0, "num docs " + numDocsInRAM); pendingUpdates.Clear(); deleteSlice = deleteQueue.NewSlice(); segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segmentName, -1, false, codec, null); Debug.Assert(numDocsInRAM == 0); if (INFO_VERBOSE && infoStream.IsEnabled("DWPT")) { infoStream.Message("DWPT", Thread.CurrentThread.Name + " init seg=" + segmentName + " delQueue=" + deleteQueue); } // this should be the last call in the ctor // it really sucks that we need to pull this within the ctor and pass this ref to the chain! consumer = indexWriterConfig.IndexingChain.GetChain(this); }
internal virtual void AddDeletes(DocumentsWriterDeleteQueue deleteQueue) { UninterruptableMonitor.Enter(this); try { IncTickets(); // first inc the ticket count - freeze opens // a window for #anyChanges to fail bool success = false; try { queue.Enqueue(new GlobalDeletesTicket(deleteQueue.FreezeGlobalBuffer(null))); success = true; } finally { if (!success) { DecTickets(); } } } finally { UninterruptableMonitor.Exit(this); } }
// for asserts private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session) { lock (this) { currentFullFlushDelQueue = session; return(true); } }
internal bool AssertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue) { foreach (BlockedFlush blockedFlush in blockedFlushes) { Debug.Assert(blockedFlush.Dwpt.deleteQueue == flushingQueue); } return(true); }
internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value) { lock (this) { DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value)); flushControl.DoOnDelete(); return(ApplyAllDeletes(deleteQueue)); } }
internal bool UpdateNumericDocValue(Term term, string field, long?value) { lock (this) { DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; deleteQueue.AddNumericUpdate(new NumericDocValuesUpdate(term, field, value)); flushControl.DoOnDelete(); return(ApplyAllDeletes(deleteQueue)); } }
// TODO: we could check w/ FreqProxTermsWriter: if the // term doesn't exist, don't bother buffering into the // per-DWPT map (but still must go into the global map) internal bool DeleteTerms(params Term[] terms) { lock (this) { // TODO why is this synchronized? DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; deleteQueue.AddDelete(terms); flushControl.DoOnDelete(); return(ApplyAllDeletes(deleteQueue)); } }
// for asserts private bool SetFlushingDeleteQueue(DocumentsWriterDeleteQueue session) { UninterruptableMonitor.Enter(this); try { currentFullFlushDelQueue = session; return(true); } finally { UninterruptableMonitor.Exit(this); } }
private bool ApplyAllDeletes(DocumentsWriterDeleteQueue deleteQueue) { if (flushControl.GetAndResetApplyAllDeletes()) { if (deleteQueue != null && !flushControl.IsFullFlush) { ticketQueue.AddDeletes(deleteQueue); } PutEvent(ApplyDeletesEvent.INSTANCE); // apply deletes event forces a purge return(true); } return(false); }
internal bool UpdateBinaryDocValue(Term term, string field, BytesRef value) { UninterruptableMonitor.Enter(this); try { DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; deleteQueue.AddBinaryUpdate(new BinaryDocValuesUpdate(term, field, value)); flushControl.DoOnDelete(); return(ApplyAllDeletes(deleteQueue)); } finally { UninterruptableMonitor.Exit(this); } }
// TODO: we could check w/ FreqProxTermsWriter: if the // term doesn't exist, don't bother buffering into the // per-DWPT map (but still must go into the global map) internal bool DeleteTerms(params Term[] terms) { UninterruptableMonitor.Enter(this); try { // TODO why is this synchronized? DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; deleteQueue.AddDelete(terms); flushControl.DoOnDelete(); return(ApplyAllDeletes(deleteQueue)); } finally { UninterruptableMonitor.Exit(this); } }
private bool AssertActiveDeleteQueue(DocumentsWriterDeleteQueue queue) { int limit = perThreadPool.NumThreadStatesActive; for (int i = 0; i < limit; i++) { ThreadState next = perThreadPool.GetThreadState(i); next.@Lock(); try { Debug.Assert(!next.IsInitialized || next.dwpt.deleteQueue == queue, "isInitialized: " + next.IsInitialized + " numDocs: " + (next.IsInitialized ? next.dwpt.NumDocsInRAM : 0)); } finally { next.Unlock(); } } return(true); }
/// <summary> /// Prunes the blockedQueue by removing all DWPT that are associated with the given flush queue. /// </summary> private void PruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue) { var node = blockedFlushes.First; while (node != null) { var nextNode = node.Next; BlockedFlush blockedFlush = node.Value; if (blockedFlush.Dwpt.deleteQueue == flushingQueue) { blockedFlushes.Remove(node); Debug.Assert(!flushingWriters.ContainsKey(blockedFlush.Dwpt), "DWPT is already flushing"); // Record the flushing DWPT to reduce flushBytes in doAfterFlush flushingWriters[blockedFlush.Dwpt] = blockedFlush.Bytes; // don't decr pending here - its already done when DWPT is blocked flushQueue.Enqueue(blockedFlush.Dwpt); } node = nextNode; } }
internal void MarkForFullFlush() { DocumentsWriterDeleteQueue flushingQueue; lock (this) { if (Debugging.AssertsEnabled) { Debugging.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running"); Debugging.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: {0}", fullFlushBuffer); } fullFlush = true; flushingQueue = documentsWriter.deleteQueue; // Set a new delete queue - all subsequent DWPT will use this queue until // we do another full flush DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1); documentsWriter.deleteQueue = newQueue; } int limit = perThreadPool.NumThreadStatesActive; for (int i = 0; i < limit; i++) { ThreadState next = perThreadPool.GetThreadState(i); next.@Lock(); try { if (!next.IsInitialized) { if (closed && next.IsActive) { DocumentsWriterPerThreadPool.DeactivateThreadState(next); // LUCENENET specific - made method static per CA1822 } continue; } if (Debugging.AssertsEnabled) { Debugging.Assert(next.dwpt.deleteQueue == flushingQueue || next.dwpt.deleteQueue == documentsWriter.deleteQueue, " flushingQueue: {0} currentqueue: {1} perThread queue: {2} numDocsInRam: {3}", flushingQueue, documentsWriter.deleteQueue, next.dwpt.deleteQueue, next.dwpt.NumDocsInRAM); } if (next.dwpt.deleteQueue != flushingQueue) { // this one is already a new DWPT continue; } AddFlushableState(next); } finally { next.Unlock(); } } lock (this) { /* make sure we move all DWPT that are where concurrently marked as * pending and moved to blocked are moved over to the flushQueue. There is * a chance that this happens since we marking DWPT for full flush without * blocking indexing.*/ PruneBlockedQueue(flushingQueue); if (Debugging.AssertsEnabled) { Debugging.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue)); } //FlushQueue.AddAll(FullFlushBuffer); foreach (var dwpt in fullFlushBuffer) { flushQueue.Enqueue(dwpt); } fullFlushBuffer.Clear(); UpdateStallState(); } if (Debugging.AssertsEnabled) { Debugging.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue)); } }