public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
conf.SetMergePolicy(Random.NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountdownEvent latch = new CountdownEvent(1);
CountdownEvent signal = new CountdownEvent(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
var t = new ThreadAnonymousClass(this, latch, signal, writer, manager);
t.Start();
_writer.waitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
assertFalse(manager.IsSearcherCurrent()); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
assertEquals(2, searcher.IndexReader.NumDocs);
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (Verbose)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
var waiter = new ThreadAnonymousClass2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished)
{
waiter.Interrupt();
fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Dispose(manager, _writer, d);
}
// LUCENE-5461
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random().Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.SetDaemon(true);
controlledRealTimeReopenThread.Start();
IList <Thread> commitThreads = new List <Thread>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = DateTime.Now.Millisecond;
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = DateTime.Now.Millisecond - start;
Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
Assert.AreEqual(1, td.TotalHits);
}
foreach (Thread commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random.Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(
#pragma warning disable 612, 618
Version.LUCENE_46,
#pragma warning restore 612, 618
new MockAnalyzer(Random));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(OpenMode.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread =
new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.IsBackground = (true);
controlledRealTimeReopenThread.Start();
IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw);
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000));
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
assertEquals(1, td.TotalHits);
}
foreach (ThreadJob commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public void TestMultithreadedWaitForGeneration()
{
Thread CreateWorker(int threadNum, ControlledRealTimeReopenThread <IndexSearcher> controlledReopen, long generation,
SearcherManager searcherManager, List <ThreadOutput> outputList)
{
ThreadStart threadStart = delegate
{
Stopwatch stopwatch = Stopwatch.StartNew();
controlledReopen.WaitForGeneration(generation);
stopwatch.Stop();
double milliSecsWaited = stopwatch.Elapsed.TotalMilliseconds;
int numRecs = 0;
IndexSearcher indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
numRecs = topDocs.TotalHits;
}
finally
{
searcherManager.Release(indexSearcher);
}
lock (outputList)
{
outputList.Add(new ThreadOutput {
ThreadNum = threadNum, NumRecs = numRecs, MilliSecsWaited = milliSecsWaited
});
}
};
return(new Thread(threadStart));
}
int threadCount = 3;
List <ThreadOutput> outputList = new List <ThreadOutput>();
RAMDirectory indexDir = new RAMDirectory();
Analyzer standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
IndexWriterConfig indexConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer);
IndexWriter indexWriter = new IndexWriter(indexDir, indexConfig);
TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter);
//Add two documents
Document doc = new Document();
doc.Add(new Int32Field("id", 1, Field.Store.YES));
doc.Add(new StringField("name", "Doc1", Field.Store.YES));
long generation = trackingWriter.AddDocument(doc);
doc.Add(new Int32Field("id", 2, Field.Store.YES));
doc.Add(new StringField("name", "Doc3", Field.Store.YES));
generation = trackingWriter.AddDocument(doc);
SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null);
//Reopen SearcherManager every 2 secs via background thread if no thread waiting for newer generation.
//Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation.
double maxRefreshSecs = 2.0;
double minRefreshSecs = .2;
var controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, maxRefreshSecs, minRefreshSecs);
//Start() will start a seperate thread that will invoke the object's Run(). However,
//calling Run() directly would execute that code on the current thread rather then a new thread
//which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API
//is not as intuitive as it could be. ie. Call Start() not Run().
controlledRealTimeReopenThread.IsBackground = true; //Set as a background thread
controlledRealTimeReopenThread.Name = "Controlled Real Time Reopen Thread";
controlledRealTimeReopenThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
controlledRealTimeReopenThread.Start();
//Create the threads for doing searchers
List <Thread> threadList = new List <Thread>();
for (int i = 1; i <= threadCount; i++)
{
threadList.Add(CreateWorker(i, controlledRealTimeReopenThread, generation, searcherManager, outputList));
}
//Start all the threads
foreach (Thread thread in threadList)
{
thread.Start();
}
//wait for the threads to finish.
foreach (Thread thread in threadList)
{
thread.Join(); //will wait here until the thread terminates.
}
//Now make sure that no thread waited longer then our min refresh time
//plus a small fudge factor. Also verify that all threads resported back and
//each saw 2 records.
//Verify all threads reported back a result.
assertEquals(threadCount, outputList.Count);
int millisecsPerSec = 1000;
foreach (ThreadOutput output in outputList)
{
//Verify the thread saw exactly 2 docs
assertEquals(2, output.NumRecs);
//Verify the thread wait time was around what was expected.
Assert.True(output.MilliSecsWaited <= (minRefreshSecs * millisecsPerSec) + 30); //30ms is fudged factor to account for call overhead
}
controlledRealTimeReopenThread.Dispose(); //will kill and join to the thread
Assert.False(controlledRealTimeReopenThread.IsAlive); //to prove that Dispose really does kill the thread.
searcherManager.Dispose();
indexWriter.Dispose();
indexDir.Dispose();
}
public void TestStraightForwardDemonstration()
{
RAMDirectory indexDir = new RAMDirectory();
Analyzer standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
IndexWriterConfig indexConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer);
IndexWriter indexWriter = new IndexWriter(indexDir, indexConfig);
TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter);
Document doc = new Document();
doc.Add(new Int32Field("id", 1, Field.Store.YES));
doc.Add(new StringField("name", "Doc1", Field.Store.YES));
trackingWriter.AddDocument(doc);
SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null);
//Reopen SearcherManager every 1 secs via background thread if no thread waiting for newer generation.
//Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation.
var controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, 1, 0.2);
//Start() will start a seperate thread that will invoke the object's Run(). However,
//calling Run() directly would execute that code on the current thread rather then a new thread
//which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API
//is not as intuitive as it could be. ie. Call Start() not Run().
controlledRealTimeReopenThread.IsBackground = true; //Set as a background thread
controlledRealTimeReopenThread.Name = "Controlled Real Time Reopen Thread";
controlledRealTimeReopenThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
controlledRealTimeReopenThread.Start();
//An indexSearcher only sees Doc1
IndexSearcher indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
assertEquals(1, topDocs.TotalHits); //There is only one doc
}
finally
{
searcherManager.Release(indexSearcher);
}
//Add a 2nd document
doc = new Document();
doc.Add(new Int32Field("id", 2, Field.Store.YES));
doc.Add(new StringField("name", "Doc2", Field.Store.YES));
trackingWriter.AddDocument(doc);
//Demonstrate that we can only see the first doc because we haven't
//waited 1 sec or called WaitForGeneration
indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
assertEquals(1, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs
}
finally
{
searcherManager.Release(indexSearcher);
}
//Demonstrate that we can see both docs after we wait a little more
//then 1 sec so that controlledRealTimeReopenThread max interval is exceeded
//and it calls MaybeRefresh
Thread.Sleep(1100); //wait 1.1 secs as ms
indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
assertEquals(2, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs
}
finally
{
searcherManager.Release(indexSearcher);
}
//Add a 3rd document
doc = new Document();
doc.Add(new Int32Field("id", 3, Field.Store.YES));
doc.Add(new StringField("name", "Doc3", Field.Store.YES));
long generation = trackingWriter.AddDocument(doc);
//Demonstrate that if we call WaitForGeneration our wait will be
// .2 secs or less (the min interval we set earlier) and then we will
//see all 3 documents.
Stopwatch stopwatch = Stopwatch.StartNew();
controlledRealTimeReopenThread.WaitForGeneration(generation);
stopwatch.Stop();
assertTrue(stopwatch.Elapsed.TotalMilliseconds <= 200 + 30); //30ms is fudged factor to account for call overhead.
indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
assertEquals(3, topDocs.TotalHits); //Can see both docs due to auto refresh after 1.1 secs
}
finally
{
searcherManager.Release(indexSearcher);
}
controlledRealTimeReopenThread.Dispose();
searcherManager.Dispose();
indexWriter.Dispose();
indexDir.Dispose();
}