public SearcherContextInternal(Directory dir, Analyzer defaultAnalyzer, TimeSpan targetMinStale, TimeSpan targetMaxStale)
{
Analyzer = new PerFieldAnalyzerWrapper(defaultAnalyzer);
_writer = new IndexWriter(dir, new IndexWriterConfig(LeoLuceneVersion.Version, Analyzer)
{
OpenMode = OpenMode.CREATE_OR_APPEND
});
_trackingWriter = new TrackingIndexWriter(_writer);
_searcherManager = new SearcherManager(_writer, true, null);
_nrtReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(_trackingWriter, _searcherManager, targetMaxStale.TotalSeconds, targetMinStale.TotalSeconds);
_nrtReopenThread.SetDaemon(true);
_nrtReopenThread.Start();
}
static LuceneIndex()
{
System.IO.Directory.CreateDirectory(IndexDirectory);
directory = new NRTCachingDirectory(new MMapDirectory(IndexDirectory), 256, 256);
analyzer = new WordAnalyzer(LuceneVersion.LUCENE_48);
writer = new IndexWriter(directory, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer)
{
IndexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy(),
MergedSegmentWarmer = new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT),
//UseCompoundFile = false,
OpenMode = OpenMode.CREATE_OR_APPEND,
UseReaderPooling = true,
MergeScheduler = new TaskMergeScheduler(),
MergePolicy = new TieredMergePolicy(),
RAMBufferSizeMB = 1024,
MaxBufferedDocs = 1024
}.SetInfoStream(InfoStream.NO_OUTPUT));
trackingWriter = new TrackingIndexWriter(writer);
searcherKeeper = new SearcherManager(writer, false, null);
reopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherKeeper, 3.0, 0.0)
{
Priority = ThreadPriority.BelowNormal,
IsBackground = true
};
reopenThread.Start();
new Thread(() => {
while (true)
{
Thread.Sleep(30000);
try { writer.Commit(); } catch {}
}
})
{
IsBackground = true, Priority = ThreadPriority.BelowNormal
}.Start();
}
public ThreadAnonymousInnerClassHelper2(TestControlledRealTimeReopenThread outerInstance, long lastGen, ControlledRealTimeReopenThread<IndexSearcher> thread, AtomicBoolean finished)
{
this.OuterInstance = outerInstance;
this.LastGen = lastGen;
this.thread = thread;
this.Finished = finished;
}
protected internal override void DoAfterWriter(TaskScheduler es)
{
double minReopenSec = 0.01 + 0.05 * Random().NextDouble();
double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble());
if (VERBOSE)
{
Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
}
GenWriter = new TrackingIndexWriter(Writer);
SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es);
NrtNoDeletes = new SearcherManager(Writer, false, sf);
NrtDeletes = new SearcherManager(Writer, true, sf);
NrtDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtDeletes, maxReopenSec, minReopenSec);
NrtDeletesThread.Name = "NRTDeletes Reopen Thread";
NrtDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
NrtDeletesThread.SetDaemon(true);
NrtDeletesThread.Start();
NrtNoDeletesThread = new ControlledRealTimeReopenThread<>(GenWriter, NrtNoDeletes, maxReopenSec, minReopenSec);
NrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread";
NrtNoDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
NrtNoDeletesThread.SetDaemon(true);
NrtNoDeletesThread.Start();
}
/*
* LUCENE-3528 - NRTManager hangs in certain situations
*/
public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountDownLatch latch = new CountDownLatch(1);
CountDownLatch signal = new CountDownLatch(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);
t.Start();
_writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
Assert.AreEqual(2, searcher.IndexReader.NumDocs());
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread<IndexSearcher> thread = new ControlledRealTimeReopenThread<IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (VERBOSE)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished.Get())
{
waiter.Interrupt();
Assert.Fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Close(manager, _writer, d);
}
// LUCENE-5461
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random().Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.SetDaemon(true);
controlledRealTimeReopenThread.Start();
IList<Thread> commitThreads = new List<Thread>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = DateTime.Now.Millisecond;
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = DateTime.Now.Millisecond - start;
Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
Assert.AreEqual(1, td.TotalHits);
}
foreach (Thread commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
