internal void DoOnAbort(ThreadState state) { lock (this) { try { if (state.flushPending) { flushBytes -= state.bytesUsed; } else { activeBytes -= state.bytesUsed; } if (Debugging.AssertsEnabled) { Debugging.Assert(AssertMemory()); } // Take it out of the loop this DWPT is stale DocumentsWriterPerThreadPool.Reset(state, closed); // LUCENENET specific - made static per CA1822 } finally { UpdateStallState(); } } }
/// <summary> /// Creates a new config that that handles the live <see cref="IndexWriter"/> /// settings. /// </summary> internal LiveIndexWriterConfig(IndexWriterConfig config) { maxBufferedDeleteTerms = config.MaxBufferedDeleteTerms; maxBufferedDocs = config.MaxBufferedDocs; mergedSegmentWarmer = config.MergedSegmentWarmer; ramBufferSizeMB = config.RAMBufferSizeMB; readerTermsIndexDivisor = config.ReaderTermsIndexDivisor; termIndexInterval = config.TermIndexInterval; matchVersion = config.matchVersion; analyzer = config.Analyzer; delPolicy = config.IndexDeletionPolicy; commit = config.IndexCommit; openMode = config.OpenMode; similarity = config.Similarity; mergeScheduler = config.MergeScheduler; writeLockTimeout = config.WriteLockTimeout; indexingChain = config.IndexingChain; codec = config.Codec; infoStream = config.InfoStream; mergePolicy = config.MergePolicy; indexerThreadPool = config.IndexerThreadPool; readerPooling = config.UseReaderPooling; flushPolicy = config.FlushPolicy; perThreadHardLimitMB = config.RAMPerThreadHardLimitMB; useCompoundFile = config.UseCompoundFile; checkIntegrityAtMerge = config.CheckIntegrityAtMerge; }
// used by IndexWriterConfig internal LiveIndexWriterConfig(Analyzer analyzer, LuceneVersion matchVersion) { this.analyzer = analyzer; this.matchVersion = matchVersion; ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB; maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS; readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR; mergedSegmentWarmer = null; termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here delPolicy = new KeepOnlyLastCommitDeletionPolicy(); commit = null; useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM; openMode = Index.OpenMode.CREATE_OR_APPEND; similarity = IndexSearcher.DefaultSimilarity; #if !FEATURE_CONCURRENTMERGESCHEDULER mergeScheduler = new TaskMergeScheduler(); #else mergeScheduler = new ConcurrentMergeScheduler(); #endif writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT; indexingChain = DocumentsWriterPerThread.DefaultIndexingChain; codec = Codec.Default; if (codec == null) { throw new System.NullReferenceException(); } infoStream = Util.InfoStream.Default; mergePolicy = new TieredMergePolicy(); flushPolicy = new FlushByRamOrCountsPolicy(); readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING; indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES); perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; }
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory) { this.directory = directory; this.config = config; this.infoStream = config.InfoStream; this.perThreadPool = config.IndexerThreadPool; flushPolicy = config.FlushPolicy; this.writer = writer; this.events = new ConcurrentQueue <IEvent>(); flushControl = new DocumentsWriterFlushControl(this, config, writer.bufferedUpdatesStream); }
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream) { this.infoStream = config.InfoStream; this.stallControl = new DocumentsWriterStallControl(); this.perThreadPool = documentsWriter.perThreadPool; this.flushPolicy = documentsWriter.flushPolicy; this.config = config; this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024; this.documentsWriter = documentsWriter; this.bufferedUpdatesStream = bufferedUpdatesStream; }
internal void AddFlushableState(ThreadState perThread) { if (infoStream.IsEnabled("DWFC")) { infoStream.Message("DWFC", "addFlushableState " + perThread.dwpt); } DocumentsWriterPerThread dwpt = perThread.dwpt; if (Debugging.AssertsEnabled) { Debugging.Assert(perThread.IsHeldByCurrentThread); Debugging.Assert(perThread.IsInitialized); Debugging.Assert(fullFlush); Debugging.Assert(dwpt.deleteQueue != documentsWriter.deleteQueue); } if (dwpt.NumDocsInRAM > 0) { UninterruptableMonitor.Enter(this); try { if (!perThread.flushPending) { SetFlushPending(perThread); } DocumentsWriterPerThread flushingDWPT = InternalTryCheckOutForFlush(perThread); if (Debugging.AssertsEnabled) { Debugging.Assert(flushingDWPT != null, "DWPT must never be null here since we hold the lock and it holds documents"); Debugging.Assert(dwpt == flushingDWPT, "flushControl returned different DWPT"); } fullFlushBuffer.Add(flushingDWPT); } finally { UninterruptableMonitor.Exit(this); } } else { DocumentsWriterPerThreadPool.Reset(perThread, closed); // make this state inactive // LUCENENET specific - made method static per CA1822 } }
private void CheckoutAndBlock(ThreadState perThread) { perThread.@Lock(); try { if (Debugging.AssertsEnabled) { Debugging.Assert(perThread.flushPending, "can not block non-pending threadstate"); Debugging.Assert(fullFlush, "can not block if fullFlush == false"); } DocumentsWriterPerThread dwpt; long bytes = perThread.bytesUsed; dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822 numPending--; blockedFlushes.AddLast(new BlockedFlush(dwpt, bytes)); } finally { perThread.Unlock(); } }
private DocumentsWriterPerThread InternalTryCheckOutForFlush(ThreadState perThread) { if (Debugging.AssertsEnabled) { // LUCENENET specific - Since we need to mimic the unfair behavior of ReentrantLock, we need to ensure that all threads that enter here hold the lock. Debugging.Assert(perThread.IsHeldByCurrentThread); Debugging.Assert(Monitor.IsEntered(this)); Debugging.Assert(perThread.flushPending); } try { // LUCENENET specific - We removed the call to perThread.TryLock() and the try-finally below as they are no longer needed. // We are pending so all memory is already moved to flushBytes if (perThread.IsInitialized) { if (Debugging.AssertsEnabled) { Debugging.Assert(perThread.IsHeldByCurrentThread); } DocumentsWriterPerThread dwpt; long bytes = perThread.bytesUsed; // do that before // replace! dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822 if (Debugging.AssertsEnabled) { Debugging.Assert(!flushingWriters.ContainsKey(dwpt), "DWPT is already flushing"); } // Record the flushing DWPT to reduce flushBytes in doAfterFlush flushingWriters[dwpt] = bytes; numPending--; // write access synced return(dwpt); } return(null); } finally { UpdateStallState(); } }
internal void MarkForFullFlush() { DocumentsWriterDeleteQueue flushingQueue; lock (this) { if (Debugging.AssertsEnabled) { Debugging.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running"); Debugging.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: {0}", fullFlushBuffer); } fullFlush = true; flushingQueue = documentsWriter.deleteQueue; // Set a new delete queue - all subsequent DWPT will use this queue until // we do another full flush DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1); documentsWriter.deleteQueue = newQueue; } int limit = perThreadPool.NumThreadStatesActive; for (int i = 0; i < limit; i++) { ThreadState next = perThreadPool.GetThreadState(i); next.@Lock(); try { if (!next.IsInitialized) { if (closed && next.IsActive) { DocumentsWriterPerThreadPool.DeactivateThreadState(next); // LUCENENET specific - made method static per CA1822 } continue; } if (Debugging.AssertsEnabled) { Debugging.Assert(next.dwpt.deleteQueue == flushingQueue || next.dwpt.deleteQueue == documentsWriter.deleteQueue, " flushingQueue: {0} currentqueue: {1} perThread queue: {2} numDocsInRam: {3}", flushingQueue, documentsWriter.deleteQueue, next.dwpt.deleteQueue, next.dwpt.NumDocsInRAM); } if (next.dwpt.deleteQueue != flushingQueue) { // this one is already a new DWPT continue; } AddFlushableState(next); } finally { next.Unlock(); } } lock (this) { /* make sure we move all DWPT that are where concurrently marked as * pending and moved to blocked are moved over to the flushQueue. There is * a chance that this happens since we marking DWPT for full flush without * blocking indexing.*/ PruneBlockedQueue(flushingQueue); if (Debugging.AssertsEnabled) { Debugging.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue)); } //FlushQueue.AddAll(FullFlushBuffer); foreach (var dwpt in fullFlushBuffer) { flushQueue.Enqueue(dwpt); } fullFlushBuffer.Clear(); UpdateStallState(); } if (Debugging.AssertsEnabled) { Debugging.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue)); } }