/*
* LUCENE-3528 - NRTManager hangs in certain situations
*/
public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountDownLatch latch = new CountDownLatch(1);
CountDownLatch signal = new CountDownLatch(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);
t.Start();
_writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
Assert.AreEqual(2, searcher.IndexReader.NumDocs());
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (VERBOSE)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished.Get())
{
waiter.Interrupt();
Assert.Fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Close(manager, _writer, d);
}
public virtual void TestMaybeRefreshBlockingLock()
{
// make sure that maybeRefreshBlocking releases the lock, otherwise other
// threads cannot obtain it.
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
w.Dispose();
SearcherManager sm = new SearcherManager(dir, null);
ThreadJob t = new ThreadAnonymousInnerClassHelper2(this, sm);
t.Start();
t.Join();
// if maybeRefreshBlocking didn't release the lock, this will fail.
assertTrue("failde to obtain the refreshLock!", sm.MaybeRefresh());
sm.Dispose();
dir.Dispose();
}
public void Run()
{
try
{
triedReopen.Set(true);
if (VERBOSE)
{
Console.WriteLine("NOW call maybeReopen");
}
searcherManager.MaybeRefresh();
success.Set(true);
}
catch (ObjectDisposedException)
{
// expected
}
catch (Exception e)
{
if (VERBOSE)
{
Console.WriteLine("FAIL: unexpected exc");
Console.Out.Write(e.StackTrace);
}
exc[0] = e;
// use success as the barrier here to make sure we see the write
success.Set(false);
}
}
public void Run()
{
try
{
triedReopen.Value = (true);
if (Verbose)
{
Console.WriteLine("NOW call maybeReopen");
}
searcherManager.MaybeRefresh();
success.Value = (true);
}
catch (Exception e) when(e.IsAlreadyClosedException())
{
// expected
}
catch (Exception e) when(e.IsThrowable())
{
if (Verbose)
{
Console.WriteLine("FAIL: unexpected exc");
Console.Out.Write(e.StackTrace);
}
exc[0] = e;
// use success as the barrier here to make sure we see the write
success.Value = (false);
}
}
public virtual void TestEnsureOpen()
{
Directory dir = NewDirectory();
(new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
SearcherManager sm = new SearcherManager(dir, null);
IndexSearcher s = sm.Acquire();
sm.Dispose();
// this should succeed;
sm.Release(s);
try
{
// this should fail
sm.Acquire();
}
catch (Exception e) when(e.IsAlreadyClosedException())
{
// ok
}
try
{
// this should fail
sm.MaybeRefresh();
}
catch (Exception e) when(e.IsAlreadyClosedException())
{
// ok
}
dir.Dispose();
}
public virtual void TestDeleteDocuments()
{
Directory directory = CreateIndex();
IndexWriter writer = GetWriter(directory);
ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
IndexSearcher searcher = mgr.Acquire();
TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(1, topDocs.TotalHits);
TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0")));
Assert.AreEqual(1, result);
// writer.Commit();
Assert.IsTrue(writer.HasDeletions());
mgr.MaybeRefresh();
searcher = mgr.Acquire();
topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(0, topDocs.TotalHits);
}
public override void Run()
{
try
{
signal.Wait();
manager.MaybeRefresh();
writer.DeleteDocuments(new TermQuery(new Term("foo", "barista")));
manager.MaybeRefresh(); // kick off another reopen so we inc. the internal gen
}
catch (Exception e) when(e.IsException())
{
e.printStackTrace();
}
finally
{
latch.Reset(latch.CurrentCount == 0 ? 0 : latch.CurrentCount - 1); // let the add below finish
}
}
public override void Run()
{
try
{
Signal.@await();
Manager.MaybeRefresh();
Writer.DeleteDocuments(new TermQuery(new Term("foo", "barista")));
Manager.MaybeRefresh(); // kick off another reopen so we inc. the internal gen
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
}
finally
{
Latch.countDown(); // let the add below finish
}
}
public virtual void Test()
{
Directory dir = NewFSDirectory(CreateTempDir("livefieldupdates"));
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter w = new IndexWriter(dir, iwc);
SearcherManager mgr = new SearcherManager(w, true, new SearcherFactoryAnonymousInnerClassHelper());
const int missing = -1;
LiveFieldValues <IndexSearcher, int?> rt = new LiveFieldValuesAnonymousInnerClassHelper(mgr, missing);
int numThreads = TestUtil.NextInt(Random(), 2, 5);
if (VERBOSE)
{
Console.WriteLine(numThreads + " threads");
}
CountdownEvent startingGun = new CountdownEvent(1);
IList <ThreadClass> threads = new List <ThreadClass>();
int iters = AtLeast(1000);
int idCount = TestUtil.NextInt(Random(), 100, 10000);
double reopenChance = Random().NextDouble() * 0.01;
double deleteChance = Random().NextDouble() * 0.25;
double addChance = Random().NextDouble() * 0.5;
for (int t = 0; t < numThreads; t++)
{
int threadID = t;
Random threadRandom = new Random(Random().Next());
ThreadClass thread = new ThreadAnonymousInnerClassHelper(w, mgr, missing, rt, startingGun, iters, idCount, reopenChance, deleteChance, addChance, t, threadID, threadRandom);
threads.Add(thread);
thread.Start();
}
startingGun.Signal();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
mgr.MaybeRefresh();
Assert.AreEqual(0, rt.Count);
rt.Dispose();
mgr.Dispose();
w.Dispose();
dir.Dispose();
}
public virtual void TestMaybeRefreshBlockingLock()
{
// make sure that maybeRefreshBlocking releases the lock, otherwise other
// threads cannot obtain it.
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
w.Dispose();
SearcherManager sm = new SearcherManager(dir, null);
ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, sm);
t.Start();
t.Join();
// if maybeRefreshBlocking didn't release the lock, this will fail.
Assert.IsTrue(sm.MaybeRefresh(), "failde to obtain the refreshLock!");
sm.Dispose();
dir.Dispose();
}
public virtual void TestEnsureOpen()
{
Directory dir = NewDirectory();
(new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
SearcherManager sm = new SearcherManager(dir, null);
IndexSearcher s = sm.Acquire();
sm.Dispose();
// this should succeed;
sm.Release(s);
try
{
// this should fail
sm.Acquire();
}
#pragma warning disable 168
catch (ObjectDisposedException e)
#pragma warning restore 168
{
// ok
}
try
{
// this should fail
sm.MaybeRefresh();
}
#pragma warning disable 168
catch (ObjectDisposedException e)
#pragma warning restore 168
{
// ok
}
dir.Dispose();
}
// Reopen local reader
public void Reopen()
{
IndexSearcher before = Mgr.Acquire();
Mgr.Release(before);
Mgr.MaybeRefresh();
IndexSearcher after = Mgr.Acquire();
try
{
if (after != before)
{
// New searcher was opened
long version = Searchers.Record(after);
Searchers.Prune(new SearcherLifetimeManager.PruneByAge(OuterInstance.MaxSearcherAgeSeconds));
OuterInstance.BroadcastNodeReopen(MyNodeID, version, after);
}
}
finally
{
Mgr.Release(after);
}
}
public override void Run()
{
try
{
IDictionary <string, int?> values = new Dictionary <string, int?>();
IList <string> allIDs = new SynchronizedList <string>();
StartingGun.Wait();
for (int iter = 0; iter < Iters; iter++)
{
// Add/update a document
Document doc = new Document();
// Threads must not update the same id at the
// same time:
if (ThreadRandom.NextDouble() <= AddChance)
{
string id = string.Format(CultureInfo.InvariantCulture, "{0}_{1:X4}", ThreadID, ThreadRandom.Next(IdCount));
int field = ThreadRandom.Next(int.MaxValue);
doc.Add(new StringField("id", id, Field.Store.YES));
doc.Add(new Int32Field("field", (int)field, Field.Store.YES));
w.UpdateDocument(new Term("id", id), doc);
Rt.Add(id, field);
if (!values.ContainsKey(id))//Key didn't exist before
{
allIDs.Add(id);
}
values[id] = field;
}
if (allIDs.Count > 0 && ThreadRandom.NextDouble() <= DeleteChance)
{
string randomID = allIDs[ThreadRandom.Next(allIDs.Count)];
w.DeleteDocuments(new Term("id", randomID));
Rt.Delete(randomID);
values[randomID] = Missing;
}
if (ThreadRandom.NextDouble() <= ReopenChance || Rt.Count > 10000)
{
//System.out.println("refresh @ " + rt.Size());
Mgr.MaybeRefresh();
if (VERBOSE)
{
IndexSearcher s = Mgr.Acquire();
try
{
Console.WriteLine("TEST: reopen " + s);
}
finally
{
Mgr.Release(s);
}
Console.WriteLine("TEST: " + values.Count + " values");
}
}
if (ThreadRandom.Next(10) == 7)
{
Assert.AreEqual(null, Rt.Get("foo"));
}
if (allIDs.Count > 0)
{
string randomID = allIDs[ThreadRandom.Next(allIDs.Count)];
int? expected = values[randomID];
if (expected == Missing)
{
expected = null;
}
Assert.AreEqual(expected, Rt.Get(randomID), "id=" + randomID);
}
}
}
catch (Exception t)
{
throw new Exception(t.Message, t);
}
}
public virtual void TestTryDeleteDocument()
{
Directory directory = CreateIndex();
IndexWriter writer = GetWriter(directory);
ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
IndexSearcher searcher = mgr.Acquire();
TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(1, topDocs.TotalHits);
long result;
if (Random().NextBoolean())
{
IndexReader r = DirectoryReader.Open(writer, true);
result = mgrWriter.TryDeleteDocument(r, 0);
r.Dispose();
}
else
{
result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0);
}
// The tryDeleteDocument should have succeeded:
Assert.IsTrue(result != -1);
Assert.IsTrue(writer.HasDeletions());
if (Random().NextBoolean())
{
writer.Commit();
}
Assert.IsTrue(writer.HasDeletions());
mgr.MaybeRefresh();
searcher = mgr.Acquire();
topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(0, topDocs.TotalHits);
}