internal void DoOnAbort(ThreadState state)
{
lock (this)
{
try
{
if (state.flushPending)
{
flushBytes -= state.bytesUsed;
}
else
{
activeBytes -= state.bytesUsed;
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertMemory());
}
// Take it out of the loop this DWPT is stale
DocumentsWriterPerThreadPool.Reset(state, closed); // LUCENENET specific - made static per CA1822
}
finally
{
UpdateStallState();
}
}
}
public virtual void TestClone()
{
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriterConfig clone = (IndexWriterConfig)conf.Clone();
// Make sure parameters that can't be reused are cloned
IndexDeletionPolicy delPolicy = conf.DelPolicy;
IndexDeletionPolicy delPolicyClone = clone.DelPolicy;
Assert.IsTrue(delPolicy.GetType() == delPolicyClone.GetType() && (delPolicy != delPolicyClone || delPolicy.Clone() == delPolicyClone.Clone()));
FlushPolicy flushPolicy = conf.FlushPolicy;
FlushPolicy flushPolicyClone = clone.FlushPolicy;
Assert.IsTrue(flushPolicy.GetType() == flushPolicyClone.GetType() && (flushPolicy != flushPolicyClone || flushPolicy.Clone() == flushPolicyClone.Clone()));
DocumentsWriterPerThreadPool pool = conf.IndexerThreadPool;
DocumentsWriterPerThreadPool poolClone = clone.IndexerThreadPool;
Assert.IsTrue(pool.GetType() == poolClone.GetType() && (pool != poolClone || pool.Clone() == poolClone.Clone()));
MergePolicy mergePolicy = conf.MergePolicy;
MergePolicy mergePolicyClone = clone.MergePolicy;
Assert.IsTrue(mergePolicy.GetType() == mergePolicyClone.GetType() && (mergePolicy != mergePolicyClone || mergePolicy.Clone() == mergePolicyClone.Clone()));
IMergeScheduler mergeSched = conf.MergeScheduler;
IMergeScheduler mergeSchedClone = clone.MergeScheduler;
Assert.IsTrue(mergeSched.GetType() == mergeSchedClone.GetType() && (mergeSched != mergeSchedClone || mergeSched.Clone() == mergeSchedClone.Clone()));
conf.SetMergeScheduler(new SerialMergeScheduler());
Assert.AreEqual(typeof(ConcurrentMergeScheduler), clone.MergeScheduler.GetType());
}
// used by IndexWriterConfig
internal LiveIndexWriterConfig(Analyzer analyzer, LuceneVersion matchVersion)
{
this.analyzer = analyzer;
this.matchVersion = matchVersion;
ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS;
readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR;
mergedSegmentWarmer = null;
termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here
delPolicy = new KeepOnlyLastCommitDeletionPolicy();
commit = null;
useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
openMode = Index.OpenMode.CREATE_OR_APPEND;
similarity = IndexSearcher.DefaultSimilarity;
#if !FEATURE_CONCURRENTMERGESCHEDULER
mergeScheduler = new TaskMergeScheduler();
#else
mergeScheduler = new ConcurrentMergeScheduler();
#endif
writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT;
indexingChain = DocumentsWriterPerThread.DefaultIndexingChain;
codec = Codec.Default;
if (codec == null)
{
throw new NullReferenceException();
}
infoStream = Util.InfoStream.Default;
mergePolicy = new TieredMergePolicy();
flushPolicy = new FlushByRamOrCountsPolicy();
readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
indexerThreadPool = new DocumentsWriterPerThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
}
/// <summary>
/// Creates a new config that that handles the live <see cref="IndexWriter"/>
/// settings.
/// </summary>
internal LiveIndexWriterConfig(IndexWriterConfig config)
{
maxBufferedDeleteTerms = config.MaxBufferedDeleteTerms;
maxBufferedDocs = config.MaxBufferedDocs;
mergedSegmentWarmer = config.MergedSegmentWarmer;
ramBufferSizeMB = config.RAMBufferSizeMB;
readerTermsIndexDivisor = config.ReaderTermsIndexDivisor;
termIndexInterval = config.TermIndexInterval;
matchVersion = config.matchVersion;
analyzer = config.Analyzer;
delPolicy = config.IndexDeletionPolicy;
commit = config.IndexCommit;
openMode = config.OpenMode;
similarity = config.Similarity;
mergeScheduler = config.MergeScheduler;
writeLockTimeout = config.WriteLockTimeout;
indexingChain = config.IndexingChain;
codec = config.Codec;
infoStream = config.InfoStream;
mergePolicy = config.MergePolicy;
indexerThreadPool = config.IndexerThreadPool;
readerPooling = config.UseReaderPooling;
flushPolicy = config.FlushPolicy;
perThreadHardLimitMB = config.RAMPerThreadHardLimitMB;
useCompoundFile = config.UseCompoundFile;
checkIntegrityAtMerge = config.CheckIntegrityAtMerge;
}
/// <summary>
/// Expert: Sets the <seealso cref="DocumentsWriterPerThreadPool"/> instance used by the
/// IndexWriter to assign thread-states to incoming indexing threads. If no
/// <seealso cref="DocumentsWriterPerThreadPool"/> is set <seealso cref="IndexWriter"/> will use
/// <seealso cref="ThreadAffinityDocumentsWriterThreadPool"/> with max number of
/// thread-states set to <seealso cref="#DEFAULT_MAX_THREAD_STATES"/> (see
/// <seealso cref="#DEFAULT_MAX_THREAD_STATES"/>).
/// </p>
/// <p>
/// NOTE: The given <seealso cref="DocumentsWriterPerThreadPool"/> instance must not be used with
/// other <seealso cref="IndexWriter"/> instances once it has been initialized / associated with an
/// <seealso cref="IndexWriter"/>.
/// </p>
/// <p>
/// NOTE: this only takes effect when IndexWriter is first created.</p>
/// </summary>
public IndexWriterConfig SetIndexerThreadPool(DocumentsWriterPerThreadPool threadPool)
{
if (threadPool == null)
{
throw new System.ArgumentException("threadPool must not be null");
}
this.indexerThreadPool = threadPool;
return(this);
}
[Slow] // LUCENENET: occasionally
public virtual void TestStallControl()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 4 + Random.Next(8), 1 };
int numDocumentsToIndex = 50 + Random.Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = Throttling.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
}
if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.InfoStream_Renamed = config.InfoStream;
this.StallControl = new DocumentsWriterStallControl();
this.PerThreadPool = documentsWriter.PerThreadPool;
this.FlushPolicy = documentsWriter.FlushPolicy;
this.Config = config;
this.HardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.DocumentsWriter = documentsWriter;
this.BufferedUpdatesStream = bufferedUpdatesStream;
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.infoStream = config.InfoStream;
this.stallControl = new DocumentsWriterStallControl();
this.perThreadPool = documentsWriter.perThreadPool;
this.flushPolicy = documentsWriter.flushPolicy;
this.config = config;
this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.documentsWriter = documentsWriter;
this.bufferedUpdatesStream = bufferedUpdatesStream;
}
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory)
{
this.directory = directory;
this.config = config;
this.infoStream = config.InfoStream;
this.perThreadPool = config.IndexerThreadPool;
flushPolicy = config.FlushPolicy;
this.writer = writer;
this.events = new ConcurrentQueue <IEvent>();
flushControl = new DocumentsWriterFlushControl(this, config, writer.bufferedUpdatesStream);
}
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory)
{
this.Directory = directory;
this.LIWConfig = config;
this.InfoStream = config.InfoStream;
this.PerThreadPool = config.IndexerThreadPool;
FlushPolicy = config.FlushPolicy;
this.Writer = writer;
this.Events = new ConcurrentQueue <Event>();
FlushControl = new DocumentsWriterFlushControl(this, config, writer.BufferedUpdatesStream);
}
public virtual void TestFlushDocCount()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsFalse(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
}
internal void AddFlushableState(ThreadState perThread)
{
if (infoStream.IsEnabled("DWFC"))
{
infoStream.Message("DWFC", "addFlushableState " + perThread.dwpt);
}
DocumentsWriterPerThread dwpt = perThread.dwpt;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsHeldByCurrentThread);
Debugging.Assert(perThread.IsInitialized);
Debugging.Assert(fullFlush);
Debugging.Assert(dwpt.deleteQueue != documentsWriter.deleteQueue);
}
if (dwpt.NumDocsInRAM > 0)
{
UninterruptableMonitor.Enter(this);
try
{
if (!perThread.flushPending)
{
SetFlushPending(perThread);
}
DocumentsWriterPerThread flushingDWPT = InternalTryCheckOutForFlush(perThread);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(flushingDWPT != null, "DWPT must never be null here since we hold the lock and it holds documents");
Debugging.Assert(dwpt == flushingDWPT, "flushControl returned different DWPT");
}
fullFlushBuffer.Add(flushingDWPT);
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
else
{
DocumentsWriterPerThreadPool.Reset(perThread, closed); // make this state inactive // LUCENENET specific - made method static per CA1822
}
}
private void CheckoutAndBlock(ThreadState perThread)
{
perThread.@Lock();
try
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.flushPending, "can not block non-pending threadstate");
Debugging.Assert(fullFlush, "can not block if fullFlush == false");
}
DocumentsWriterPerThread dwpt;
long bytes = perThread.bytesUsed;
dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822
numPending--;
blockedFlushes.AddLast(new BlockedFlush(dwpt, bytes));
}
finally
{
perThread.Unlock();
}
}
private DocumentsWriterPerThread InternalTryCheckOutForFlush(ThreadState perThread)
{
if (Debugging.AssertsEnabled)
{
// LUCENENET specific - Since we need to mimic the unfair behavior of ReentrantLock, we need to ensure that all threads that enter here hold the lock.
Debugging.Assert(perThread.IsHeldByCurrentThread);
Debugging.Assert(Monitor.IsEntered(this));
Debugging.Assert(perThread.flushPending);
}
try
{
// LUCENENET specific - We removed the call to perThread.TryLock() and the try-finally below as they are no longer needed.
// We are pending so all memory is already moved to flushBytes
if (perThread.IsInitialized)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(perThread.IsHeldByCurrentThread);
}
DocumentsWriterPerThread dwpt;
long bytes = perThread.bytesUsed; // do that before
// replace!
dwpt = DocumentsWriterPerThreadPool.Reset(perThread, closed); // LUCENENET specific - made method static per CA1822
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!flushingWriters.ContainsKey(dwpt), "DWPT is already flushing");
}
// Record the flushing DWPT to reduce flushBytes in doAfterFlush
flushingWriters[dwpt] = bytes;
numPending--; // write access synced
return(dwpt);
}
return(null);
}
finally
{
UpdateStallState();
}
}
/// <summary>
/// Expert: Sets the <seealso cref="DocumentsWriterPerThreadPool"/> instance used by the
/// IndexWriter to assign thread-states to incoming indexing threads. If no
/// <seealso cref="DocumentsWriterPerThreadPool"/> is set <seealso cref="IndexWriter"/> will use
/// <seealso cref="ThreadAffinityDocumentsWriterThreadPool"/> with max number of
/// thread-states set to <seealso cref="#DEFAULT_MAX_THREAD_STATES"/> (see
/// <seealso cref="#DEFAULT_MAX_THREAD_STATES"/>).
/// </p>
/// <p>
/// NOTE: The given <seealso cref="DocumentsWriterPerThreadPool"/> instance must not be used with
/// other <seealso cref="IndexWriter"/> instances once it has been initialized / associated with an
/// <seealso cref="IndexWriter"/>.
/// </p>
/// <p>
/// NOTE: this only takes effect when IndexWriter is first created.</p>
/// </summary>
public IndexWriterConfig SetIndexerThreadPool(DocumentsWriterPerThreadPool threadPool)
{
if (threadPool == null)
{
throw new System.ArgumentException("threadPool must not be null");
}
this.indexerThreadPool = threadPool;
return this;
}
// used by IndexWriterConfig
internal LiveIndexWriterConfig(Analyzer analyzer, Version matchVersion)
{
this.analyzer = analyzer;
this.MatchVersion = matchVersion;
RamBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
maxBufferedDeleteTerms = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS;
readerTermsIndexDivisor = IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR;
mergedSegmentWarmer = null;
termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; // TODO: this should be private to the codec, not settable here
delPolicy = new KeepOnlyLastCommitDeletionPolicy();
Commit = null;
useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
openMode = OpenMode_e.CREATE_OR_APPEND;
similarity = IndexSearcher.DefaultSimilarity;
mergeScheduler = new ConcurrentMergeScheduler();
writeLockTimeout = IndexWriterConfig.WRITE_LOCK_TIMEOUT;
indexingChain = DocumentsWriterPerThread.defaultIndexingChain;
codec = Codec.Default;
if (codec == null)
{
throw new System.NullReferenceException();
}
infoStream = Util.InfoStream.Default;
mergePolicy = new TieredMergePolicy();
flushPolicy = new FlushByRamOrCountsPolicy();
readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
PerThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
}
/// <summary>
/// Creates a new config that that handles the live <seealso cref="IndexWriter"/>
/// settings.
/// </summary>
internal LiveIndexWriterConfig(IndexWriterConfig config)
{
maxBufferedDeleteTerms = config.MaxBufferedDeleteTerms;
maxBufferedDocs = config.MaxBufferedDocs;
mergedSegmentWarmer = config.MergedSegmentWarmer;
RamBufferSizeMB = config.RAMBufferSizeMB;
readerTermsIndexDivisor = config.ReaderTermsIndexDivisor;
termIndexInterval = config.TermIndexInterval;
MatchVersion = config.MatchVersion;
analyzer = config.Analyzer;
delPolicy = config.DelPolicy;
Commit = config.IndexCommit;
openMode = config.OpenMode;
similarity = config.Similarity;
mergeScheduler = config.MergeScheduler;
writeLockTimeout = config.WriteLockTimeout;
indexingChain = config.IndexingChain;
codec = config.Codec;
infoStream = config.InfoStream;
mergePolicy = config.MergePolicy;
indexerThreadPool = config.IndexerThreadPool;
readerPooling = config.ReaderPooling;
flushPolicy = config.FlushPolicy;
PerThreadHardLimitMB = config.RAMPerThreadHardLimitMB;
useCompoundFile = config.UseCompoundFile;
checkIntegrityAtMerge = config.CheckIntegrityAtMerge;
}
internal DocumentsWriter(IndexWriter writer, LiveIndexWriterConfig config, Directory directory)
{
this.Directory = directory;
this.LIWConfig = config;
this.InfoStream = config.InfoStream;
this.PerThreadPool = config.IndexerThreadPool;
FlushPolicy = config.FlushPolicy;
this.Writer = writer;
this.Events = new ConcurrentQueue<Event>();
FlushControl = new DocumentsWriterFlushControl(this, config, writer.BufferedUpdatesStream);
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsTrue(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.hasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
public virtual void TestRandom()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int numThreads = 1 + Random.Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random.Next(8);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms)
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.peakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.hasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes);
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM)
{
assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled);
assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked);
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
internal void MarkForFullFlush()
{
DocumentsWriterDeleteQueue flushingQueue;
lock (this)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!fullFlush, "called DWFC#markForFullFlush() while full flush is still running");
Debugging.Assert(fullFlushBuffer.Count == 0, "full flush buffer should be empty: {0}", fullFlushBuffer);
}
fullFlush = true;
flushingQueue = documentsWriter.deleteQueue;
// Set a new delete queue - all subsequent DWPT will use this queue until
// we do another full flush
DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation + 1);
documentsWriter.deleteQueue = newQueue;
}
int limit = perThreadPool.NumThreadStatesActive;
for (int i = 0; i < limit; i++)
{
ThreadState next = perThreadPool.GetThreadState(i);
next.@Lock();
try
{
if (!next.IsInitialized)
{
if (closed && next.IsActive)
{
DocumentsWriterPerThreadPool.DeactivateThreadState(next); // LUCENENET specific - made method static per CA1822
}
continue;
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(next.dwpt.deleteQueue == flushingQueue ||
next.dwpt.deleteQueue == documentsWriter.deleteQueue,
" flushingQueue: {0} currentqueue: {1} perThread queue: {2} numDocsInRam: {3}",
flushingQueue, documentsWriter.deleteQueue, next.dwpt.deleteQueue, next.dwpt.NumDocsInRAM);
}
if (next.dwpt.deleteQueue != flushingQueue)
{
// this one is already a new DWPT
continue;
}
AddFlushableState(next);
}
finally
{
next.Unlock();
}
}
lock (this)
{
/* make sure we move all DWPT that are where concurrently marked as
* pending and moved to blocked are moved over to the flushQueue. There is
* a chance that this happens since we marking DWPT for full flush without
* blocking indexing.*/
PruneBlockedQueue(flushingQueue);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertBlockedFlushes(documentsWriter.deleteQueue));
}
//FlushQueue.AddAll(FullFlushBuffer);
foreach (var dwpt in fullFlushBuffer)
{
flushQueue.Enqueue(dwpt);
}
fullFlushBuffer.Clear();
UpdateStallState();
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(AssertActiveDeleteQueue(documentsWriter.deleteQueue));
}
}
