internal void DoOnAbort(ThreadState state)
{
lock (this)
{
try
{
if (state.flushPending)
{
flushBytes -= state.bytesUsed;
}
else
{
activeBytes -= state.bytesUsed;
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertMemory());
}
// Take it out of the loop this DWPT is stale
DocumentsWriterPerThreadPool.Reset(state, closed); // LUCENENET specific - made static per CA1822
}
finally
{
UpdateStallState();
}
}
}
/// <summary>
/// Creates a new config that that handles the live <see cref="IndexWriter"/>
/// settings.
/// </summary>
internal LiveIndexWriterConfig(IndexWriterConfig config)
{
maxBufferedDeleteTerms = config.MaxBufferedDeleteTerms;
maxBufferedDocs = config.MaxBufferedDocs;
mergedSegmentWarmer = config.MergedSegmentWarmer;
ramBufferSizeMB = config.RAMBufferSizeMB;
readerTermsIndexDivisor = config.ReaderTermsIndexDivisor;
termIndexInterval = config.TermIndexInterval;
matchVersion = config.matchVersion;
analyzer = config.Analyzer;
delPolicy = config.IndexDeletionPolicy;
commit = config.IndexCommit;
openMode = config.OpenMode;
similarity = config.Similarity;
mergeScheduler = config.MergeScheduler;
writeLockTimeout = config.WriteLockTimeout;
indexingChain = config.IndexingChain;
codec = config.Codec;
infoStream = config.InfoStream;
mergePolicy = config.MergePolicy;
indexerThreadPool = config.IndexerThreadPool;
readerPooling = config.UseReaderPooling;
flushPolicy = config.FlushPolicy;
perThreadHardLimitMB = config.RAMPerThreadHardLimitMB;
useCompoundFile = config.UseCompoundFile;
checkIntegrityAtMerge = config.CheckIntegrityAtMerge;
}
// used by IndexWriterConfig
internal LiveIndexWriterConfig(Analyzer analyzer, LuceneVersion matchVersion)
{
this.analyzer = analyzer;
this.matchVersion = matchVersion;
ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS;
readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR;
mergedSegmentWarmer = null;
termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here
delPolicy = new KeepOnlyLastCommitDeletionPolicy();
commit = null;
useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
openMode = Index.OpenMode.CREATE_OR_APPEND;
similarity = IndexSearcher.DefaultSimilarity;
#if !FEATURE_CONCURRENTMERGESCHEDULER
mergeScheduler = new TaskMergeScheduler();
#else
mergeScheduler = new ConcurrentMergeScheduler();
#endif
writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT;
indexingChain = DocumentsWriterPerThread.DefaultIndexingChain;
codec = Codec.Default;
if (codec == null)
{
throw new System.NullReferenceException();
}
infoStream = Util.InfoStream.Default;
mergePolicy = new TieredMergePolicy();
flushPolicy = new FlushByRamOrCountsPolicy();
readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
}
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory)
{
this.directory = directory;
this.config = config;
this.infoStream = config.InfoStream;
this.perThreadPool = config.IndexerThreadPool;
flushPolicy = config.FlushPolicy;
this.writer = writer;
this.events = new ConcurrentQueue <IEvent>();
flushControl = new DocumentsWriterFlushControl(this, config, writer.bufferedUpdatesStream);
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.infoStream = config.InfoStream;
this.stallControl = new DocumentsWriterStallControl();
this.perThreadPool = documentsWriter.perThreadPool;
this.flushPolicy = documentsWriter.flushPolicy;
this.config = config;
this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.documentsWriter = documentsWriter;
this.bufferedUpdatesStream = bufferedUpdatesStream;
}
internal void AddFlushableState(ThreadState perThread)
{
if (infoStream.IsEnabled("DWFC"))
{
infoStream.Message("DWFC", "addFlushableState " + perThread.dwpt);
}
DocumentsWriterPerThread dwpt = perThread.dwpt;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsHeldByCurrentThread);
Debugging.Assert(perThread.IsInitialized);
Debugging.Assert(fullFlush);
Debugging.Assert(dwpt.deleteQueue != documentsWriter.deleteQueue);
}
if (dwpt.NumDocsInRAM > 0)
{
UninterruptableMonitor.Enter(this);
try
{
if (!perThread.flushPending)
{
SetFlushPending(perThread);
}
DocumentsWriterPerThread flushingDWPT = InternalTryCheckOutForFlush(perThread);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(flushingDWPT != null, "DWPT must never be null here since we hold the lock and it holds documents");
Debugging.Assert(dwpt == flushingDWPT, "flushControl returned different DWPT");
}
fullFlushBuffer.Add(flushingDWPT);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
else
{
DocumentsWriterPerThreadPool.Reset(perThread, closed); // make this state inactive // LUCENENET specific - made method static per CA1822
}
}
private void CheckoutAndBlock(ThreadState perThread)
{
perThread.@Lock();
try
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.flushPending, "can not block non-pending threadstate");
Debugging.Assert(fullFlush, "can not block if fullFlush == false");
}
DocumentsWriterPerThread dwpt;
long bytes = perThread.bytesUsed;
dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822
numPending--;
blockedFlushes.AddLast(new BlockedFlush(dwpt, bytes));
}
finally
{
perThread.Unlock();
}
}
private DocumentsWriterPerThread InternalTryCheckOutForFlush(ThreadState perThread)
{
if (Debugging.AssertsEnabled)
{
// LUCENENET specific - Since we need to mimic the unfair behavior of ReentrantLock, we need to ensure that all threads that enter here hold the lock.
Debugging.Assert(perThread.IsHeldByCurrentThread);
Debugging.Assert(Monitor.IsEntered(this));
Debugging.Assert(perThread.flushPending);
}
try
{
// LUCENENET specific - We removed the call to perThread.TryLock() and the try-finally below as they are no longer needed.
// We are pending so all memory is already moved to flushBytes
if (perThread.IsInitialized)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsHeldByCurrentThread);
}
DocumentsWriterPerThread dwpt;
long bytes = perThread.bytesUsed; // do that before
// replace!
dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!flushingWriters.ContainsKey(dwpt), "DWPT is already flushing");
}
// Record the flushing DWPT to reduce flushBytes in doAfterFlush
flushingWriters[dwpt] = bytes;
numPending--; // write access synced
return(dwpt);
}
return(null);
}
finally
{
UpdateStallState();
}
}
internal void MarkForFullFlush()
{
DocumentsWriterDeleteQueue flushingQueue;
lock (this)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running");
Debugging.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: {0}", fullFlushBuffer);
}
fullFlush = true;
flushingQueue = documentsWriter.deleteQueue;
// Set a new delete queue - all subsequent DWPT will use this queue until
// we do another full flush
DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1);
documentsWriter.deleteQueue = newQueue;
}
int limit = perThreadPool.NumThreadStatesActive;
for (int i = 0; i < limit; i++)
{
ThreadState next = perThreadPool.GetThreadState(i);
next.@Lock();
try
{
if (!next.IsInitialized)
{
if (closed && next.IsActive)
{
DocumentsWriterPerThreadPool.DeactivateThreadState(next); // LUCENENET specific - made method static per CA1822
}
continue;
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(next.dwpt.deleteQueue == flushingQueue ||
next.dwpt.deleteQueue == documentsWriter.deleteQueue,
" flushingQueue: {0} currentqueue: {1} perThread queue: {2} numDocsInRam: {3}",
flushingQueue, documentsWriter.deleteQueue, next.dwpt.deleteQueue, next.dwpt.NumDocsInRAM);
}
if (next.dwpt.deleteQueue != flushingQueue)
{
// this one is already a new DWPT
continue;
}
AddFlushableState(next);
}
finally
{
next.Unlock();
}
}
lock (this)
{
/* make sure we move all DWPT that are where concurrently marked as
* pending and moved to blocked are moved over to the flushQueue. There is
* a chance that this happens since we marking DWPT for full flush without
* blocking indexing.*/
PruneBlockedQueue(flushingQueue);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue));
}
//FlushQueue.AddAll(FullFlushBuffer);
foreach (var dwpt in fullFlushBuffer)
{
flushQueue.Enqueue(dwpt);
}
fullFlushBuffer.Clear();
UpdateStallState();
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue));
}
}