public virtual void TestEnsureOpen()
{
Directory dir = NewDirectory();
(new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
SearcherManager sm = new SearcherManager(dir, null);
IndexSearcher s = sm.Acquire();
sm.Dispose();
// this should succeed;
sm.Release(s);
try
{
// this should fail
sm.Acquire();
}
catch (Exception e) when(e.IsAlreadyClosedException())
{
// ok
}
try
{
// this should fail
sm.MaybeRefresh();
}
catch (Exception e) when(e.IsAlreadyClosedException())
{
// ok
}
dir.Dispose();
}
public IEnumerable <LuceneSearchEntry> ExecuteQuery(string indexFolder, LSearch.Query query)
{
var searchIndex = LStore.FSDirectory.Open(indexFolder);
var searchManager = new LSearch.SearcherManager(searchIndex, null);
// try to release locs
searchManager.MaybeRefreshBlocking();
//execute search
var searcher = searchManager.Acquire();
var rawResults = searcher.Search(query, 20);
//convert raw results to model
var outResults = new List <LuceneSearchEntry>();
for (var i = 0; i < rawResults.ScoreDocs.Length; i++)
{
var result = rawResults.ScoreDocs[i];
var doc = searcher.Doc(result.Doc);
outResults.Add(new LuceneSearchEntry
{
Rank = i,
Id = doc.Get("id"),
Uri = doc.Get("uri"),
Title = doc.Get("title"),
Snippet = doc.Get("snippet"),
Score = result.Score
});
}
return(outResults);
}
public virtual void TestReferenceDecrementIllegally([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
Directory dir = NewDirectory();
var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
.SetMergeScheduler(newScheduler());
IndexWriter writer = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory());
writer.AddDocument(new Document());
writer.Commit();
sm.MaybeRefreshBlocking();
IndexSearcher acquire = sm.Acquire();
IndexSearcher acquire2 = sm.Acquire();
sm.Release(acquire);
sm.Release(acquire2);
acquire = sm.Acquire();
acquire.IndexReader.DecRef();
sm.Release(acquire);
Assert.Throws <InvalidOperationException>(() => sm.Acquire(), "acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager");
// sm.Dispose(); -- already closed
writer.Dispose();
dir.Dispose();
}
protected override void DoAfterWriter(TaskScheduler es)
{
double minReopenSec = 0.01 + 0.05 * Random.NextDouble();
double maxReopenSec = minReopenSec * (1.0 + 10 * Random.NextDouble());
if (Verbose)
{
Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
}
genWriter = new TrackingIndexWriter(m_writer);
SearcherFactory sf = new SearcherFactoryAnonymousClass(this, es);
nrtNoDeletes = new SearcherManager(m_writer, false, sf);
nrtDeletes = new SearcherManager(m_writer, true, sf);
nrtDeletesThread = new ControlledRealTimeReopenThread <IndexSearcher>(genWriter, nrtDeletes, maxReopenSec, minReopenSec);
nrtDeletesThread.Name = "NRTDeletes Reopen Thread";
#if FEATURE_THREAD_PRIORITY
nrtDeletesThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
#endif
nrtDeletesThread.IsBackground = (true);
nrtDeletesThread.Start();
nrtNoDeletesThread = new ControlledRealTimeReopenThread <IndexSearcher>(genWriter, nrtNoDeletes, maxReopenSec, minReopenSec);
nrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread";
#if FEATURE_THREAD_PRIORITY
nrtNoDeletesThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
#endif
nrtNoDeletesThread.IsBackground = (true);
nrtNoDeletesThread.Start();
}
protected internal override void DoAfterWriter(TaskScheduler es)
{
double minReopenSec = 0.01 + 0.05 * Random().NextDouble();
double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble());
if (VERBOSE)
{
Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
}
genWriter = new TrackingIndexWriter(writer);
SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es);
nrtNoDeletes = new SearcherManager(writer, false, sf);
nrtDeletes = new SearcherManager(writer, true, sf);
nrtDeletesThread = new ControlledRealTimeReopenThread <IndexSearcher>(genWriter, nrtDeletes, maxReopenSec, minReopenSec);
nrtDeletesThread.Name = "NRTDeletes Reopen Thread";
#if !NETSTANDARD
nrtDeletesThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
#endif
nrtDeletesThread.SetDaemon(true);
nrtDeletesThread.Start();
nrtNoDeletesThread = new ControlledRealTimeReopenThread <IndexSearcher>(genWriter, nrtNoDeletes, maxReopenSec, minReopenSec);
nrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread";
#if !NETSTANDARD
nrtNoDeletesThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
#endif
nrtNoDeletesThread.SetDaemon(true);
nrtNoDeletesThread.Start();
}
public virtual void TestReferenceDecrementIllegally()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new ConcurrentMergeScheduler()));
SearcherManager sm = new SearcherManager(writer, false, new SearcherFactory());
writer.AddDocument(new Document());
writer.Commit();
sm.MaybeRefreshBlocking();
IndexSearcher acquire = sm.Acquire();
IndexSearcher acquire2 = sm.Acquire();
sm.Release(acquire);
sm.Release(acquire2);
acquire = sm.Acquire();
acquire.IndexReader.DecRef();
sm.Release(acquire);
try
{
sm.Acquire();
Assert.Fail("acquire should have thrown an InvalidOperationException since we modified the refCount outside of the manager");
}
catch (InvalidOperationException ex)
{
//
}
// sm.Dispose(); -- already closed
writer.Dispose();
dir.Dispose();
}
public virtual void TestDeleteDocuments()
{
Directory directory = CreateIndex();
IndexWriter writer = GetWriter(directory);
ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
IndexSearcher searcher = mgr.Acquire();
TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(1, topDocs.TotalHits);
TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0")));
Assert.AreEqual(1, result);
// writer.Commit();
Assert.IsTrue(writer.HasDeletions());
mgr.MaybeRefresh();
searcher = mgr.Acquire();
topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(0, topDocs.TotalHits);
}
protected internal override void DoAfterWriter(TaskScheduler es)
{
double minReopenSec = 0.01 + 0.05 * Random().NextDouble();
double maxReopenSec = minReopenSec * (1.0 + 10 * Random().NextDouble());
if (VERBOSE)
{
Console.WriteLine("TEST: make SearcherManager maxReopenSec=" + maxReopenSec + " minReopenSec=" + minReopenSec);
}
GenWriter = new TrackingIndexWriter(Writer);
SearcherFactory sf = new SearcherFactoryAnonymousInnerClassHelper(this, es);
NrtNoDeletes = new SearcherManager(Writer, false, sf);
NrtDeletes = new SearcherManager(Writer, true, sf);
NrtDeletesThread = new ControlledRealTimeReopenThread <>(GenWriter, NrtDeletes, maxReopenSec, minReopenSec);
NrtDeletesThread.Name = "NRTDeletes Reopen Thread";
NrtDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
NrtDeletesThread.SetDaemon(true);
NrtDeletesThread.Start();
NrtNoDeletesThread = new ControlledRealTimeReopenThread <>(GenWriter, NrtNoDeletes, maxReopenSec, minReopenSec);
NrtNoDeletesThread.Name = "NRTNoDeletes Reopen Thread";
NrtNoDeletesThread.Priority = Math.Min(Thread.CurrentThread.Priority + 2, Thread.MAX_PRIORITY);
NrtNoDeletesThread.SetDaemon(true);
NrtNoDeletesThread.Start();
}
#pragma warning disable IDE0060 // Remove unused parameter
public NodeState(ShardSearchingTestBase shardSearchingTestBase, Random random, int nodeID, int numNodes)
#pragma warning restore IDE0060 // Remove unused parameter
{
this.outerInstance = shardSearchingTestBase;
MyNodeID = nodeID;
Dir = NewFSDirectory(CreateTempDir("ShardSearchingTestBase"));
// TODO: set warmer
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
iwc.SetOpenMode(OpenMode.CREATE);
if (Verbose)
{
iwc.SetInfoStream(new TextWriterInfoStream(Console.Out));
}
Writer = new IndexWriter(Dir, iwc);
Mgr = new SearcherManager(Writer, true, null);
Searchers = new SearcherLifetimeManager();
// Init w/ 0s... caller above will do initial
// "broadcast" by calling initSearcher:
currentNodeVersions = new long[numNodes];
}
public virtual void TestMaybeRefreshBlockingLock()
{
// make sure that maybeRefreshBlocking releases the lock, otherwise other
// threads cannot obtain it.
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
w.Dispose();
SearcherManager sm = new SearcherManager(dir, null);
ThreadJob t = new ThreadAnonymousInnerClassHelper2(this, sm);
t.Start();
t.Join();
// if maybeRefreshBlocking didn't release the lock, this will fail.
assertTrue("failde to obtain the refreshLock!", sm.MaybeRefresh());
sm.Dispose();
dir.Dispose();
}
public RunnableAnonymousInnerClassHelper(TestSearcherManager outerInstance, AtomicBoolean triedReopen, SearcherManager searcherManager, AtomicBoolean success, Exception[] exc)
{
this.outerInstance = outerInstance;
this.triedReopen = triedReopen;
this.searcherManager = searcherManager;
this.success = success;
this.exc = exc;
}
public ThreadAnonymousClass(TestControlledRealTimeReopenThread outerInstance, CountdownEvent latch, CountdownEvent signal, TrackingIndexWriter writer, SearcherManager manager)
{
this.outerInstance = outerInstance;
this.latch = latch;
this.signal = signal;
this.writer = writer;
this.manager = manager;
}
public ThreadAnonymousInnerClassHelper(TestControlledRealTimeReopenThread outerInstance, CountDownLatch latch, CountDownLatch signal, TrackingIndexWriter writer, SearcherManager manager)
{
this.OuterInstance = outerInstance;
this.Latch = latch;
this.Signal = signal;
this.Writer = writer;
this.Manager = manager;
}
/*
* LUCENE-3528 - NRTManager hangs in certain situations
*/
public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountDownLatch latch = new CountDownLatch(1);
CountDownLatch signal = new CountDownLatch(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);
t.Start();
_writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
Assert.AreEqual(2, searcher.IndexReader.NumDocs());
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread <IndexSearcher> thread = new ControlledRealTimeReopenThread <IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (VERBOSE)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished.Get())
{
waiter.Interrupt();
Assert.Fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Close(manager, _writer, d);
}
public virtual void TestCloseTwice()
{
// test that we can close SM twice (per IDisposable's contract).
Directory dir = NewDirectory();
(new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
SearcherManager sm = new SearcherManager(dir, null);
sm.Dispose();
sm.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Directory dir = NewFSDirectory(CreateTempDir("livefieldupdates"));
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter w = new IndexWriter(dir, iwc);
SearcherManager mgr = new SearcherManager(w, true, new SearcherFactoryAnonymousInnerClassHelper());
const int missing = -1;
LiveFieldValues <IndexSearcher, int?> rt = new LiveFieldValuesAnonymousInnerClassHelper(mgr, missing);
int numThreads = TestUtil.NextInt(Random(), 2, 5);
if (VERBOSE)
{
Console.WriteLine(numThreads + " threads");
}
CountdownEvent startingGun = new CountdownEvent(1);
IList <ThreadClass> threads = new List <ThreadClass>();
int iters = AtLeast(1000);
int idCount = TestUtil.NextInt(Random(), 100, 10000);
double reopenChance = Random().NextDouble() * 0.01;
double deleteChance = Random().NextDouble() * 0.25;
double addChance = Random().NextDouble() * 0.5;
for (int t = 0; t < numThreads; t++)
{
int threadID = t;
Random threadRandom = new Random(Random().Next());
ThreadClass thread = new ThreadAnonymousInnerClassHelper(w, mgr, missing, rt, startingGun, iters, idCount, reopenChance, deleteChance, addChance, t, threadID, threadRandom);
threads.Add(thread);
thread.Start();
}
startingGun.Signal();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
mgr.MaybeRefresh();
Assert.AreEqual(0, rt.Count);
rt.Dispose();
mgr.Dispose();
w.Dispose();
dir.Dispose();
}
public ThreadAnonymousClass(IndexWriter w, SearcherManager mgr, int?missing, LiveFieldValues <IndexSearcher, int?> rt, CountdownEvent startingGun, int iters, int idCount, double reopenChance, double deleteChance, double addChance, int t, int threadID, Random threadRandom)
{
this.w = w;
this.mgr = mgr;
this.missing = missing;
this.rt = rt;
this.startingGun = startingGun;
this.iters = iters;
this.idCount = idCount;
this.reopenChance = reopenChance;
this.deleteChance = deleteChance;
this.addChance = addChance;
this.t = t;
this.threadID = threadID;
this.threadRandom = threadRandom;
}
public ThreadAnonymousInnerClassHelper(IndexWriter w, SearcherManager mgr, int?missing, LiveFieldValues <IndexSearcher, int?> rt, CountdownEvent startingGun, int iters, int idCount, double reopenChance, double deleteChance, double addChance, int t, int threadID, Random threadRandom)
{
this.w = w;
this.Mgr = mgr;
this.Missing = missing;
this.Rt = rt;
this.StartingGun = startingGun;
this.Iters = iters;
this.IdCount = idCount;
this.ReopenChance = reopenChance;
this.DeleteChance = deleteChance;
this.AddChance = addChance;
this.t = t;
this.ThreadID = threadID;
this.ThreadRandom = threadRandom;
}
public virtual void TestListenerCalled()
{
Directory dir = NewDirectory();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
AtomicBoolean afterRefreshCalled = new AtomicBoolean(false);
SearcherManager sm = new SearcherManager(iw, false, new SearcherFactory());
sm.AddListener(new RefreshListenerAnonymousInnerClassHelper(this, afterRefreshCalled));
iw.AddDocument(new Document());
iw.Commit();
Assert.IsFalse(afterRefreshCalled.Get());
sm.MaybeRefreshBlocking();
Assert.IsTrue(afterRefreshCalled.Get());
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestMaybeRefreshBlockingLock()
{
// make sure that maybeRefreshBlocking releases the lock, otherwise other
// threads cannot obtain it.
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
w.Dispose();
SearcherManager sm = new SearcherManager(dir, null);
ThreadClass t = new ThreadAnonymousInnerClassHelper2(this, sm);
t.Start();
t.Join();
// if maybeRefreshBlocking didn't release the lock, this will fail.
Assert.IsTrue(sm.MaybeRefresh(), "failde to obtain the refreshLock!");
sm.Dispose();
dir.Dispose();
}
protected override void DoAfterWriter(TaskScheduler es)
{
SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper(this, es);
if (Random().NextBoolean())
{
// TODO: can we randomize the applyAllDeletes? But
// somehow for final searcher we must apply
// deletes...
Mgr = new SearcherManager(Writer, true, factory);
IsNRT = true;
}
else
{
// SearcherManager needs to see empty commit:
Writer.Commit();
Mgr = new SearcherManager(Dir, factory);
IsNRT = false;
AssertMergedSegmentsWarmed = false;
}
LifetimeMGR = new SearcherLifetimeManager();
}
public virtual void TestEnsureOpen()
{
Directory dir = NewDirectory();
(new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null))).Dispose();
SearcherManager sm = new SearcherManager(dir, null);
IndexSearcher s = sm.Acquire();
sm.Dispose();
// this should succeed;
sm.Release(s);
try
{
// this should fail
sm.Acquire();
}
#pragma warning disable 168
catch (ObjectDisposedException e)
#pragma warning restore 168
{
// ok
}
try
{
// this should fail
sm.MaybeRefresh();
}
#pragma warning disable 168
catch (ObjectDisposedException e)
#pragma warning restore 168
{
// ok
}
dir.Dispose();
}
public NodeState(ShardSearchingTestBase outerInstance, Random random, int nodeID, int numNodes)
{
this.OuterInstance = outerInstance;
MyNodeID = nodeID;
Dir = NewFSDirectory(CreateTempDir("ShardSearchingTestBase"));
// TODO: set warmer
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
iwc.SetOpenMode(OpenMode.CREATE);
if (VERBOSE)
{
iwc.SetInfoStream(new PrintStreamInfoStream(Console.Out));
}
Writer = new IndexWriter(Dir, iwc);
Mgr = new SearcherManager(Writer, true, null);
Searchers = new SearcherLifetimeManager();
// Init w/ 0s... caller above will do initial
// "broadcast" by calling initSearcher:
CurrentNodeVersions = new long[numNodes];
}
public virtual void TestIntermediateClose()
{
Directory dir = NewDirectory();
// Test can deadlock if we use SMS:
IConcurrentMergeScheduler scheduler;
#if !FEATURE_CONCURRENTMERGESCHEDULER
scheduler = new TaskMergeScheduler();
#else
scheduler = new ConcurrentMergeScheduler();
#endif
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(scheduler));
writer.AddDocument(new Document());
writer.Commit();
CountdownEvent awaitEnterWarm = new CountdownEvent(1);
CountdownEvent awaitClose = new CountdownEvent(1);
AtomicBoolean triedReopen = new AtomicBoolean(false);
//TaskScheduler es = Random().NextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
TaskScheduler es = Random().NextBoolean() ? null : TaskScheduler.Default;
SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper2(this, awaitEnterWarm, awaitClose, triedReopen, es);
SearcherManager searcherManager = Random().NextBoolean() ? new SearcherManager(dir, factory) : new SearcherManager(writer, Random().NextBoolean(), factory);
if (VERBOSE)
{
Console.WriteLine("sm created");
}
IndexSearcher searcher = searcherManager.Acquire();
try
{
assertEquals(1, searcher.IndexReader.NumDocs);
}
finally
{
searcherManager.Release(searcher);
}
writer.AddDocument(new Document());
writer.Commit();
AtomicBoolean success = new AtomicBoolean(false);
Exception[] exc = new Exception[1];
ThreadClass thread = new ThreadClass(() => new RunnableAnonymousInnerClassHelper(this, triedReopen, searcherManager, success, exc).Run());
thread.Start();
if (VERBOSE)
{
Console.WriteLine("THREAD started");
}
awaitEnterWarm.Wait();
if (VERBOSE)
{
Console.WriteLine("NOW call close");
}
searcherManager.Dispose();
awaitClose.Signal();
thread.Join();
try
{
searcherManager.Acquire();
fail("already closed");
}
#pragma warning disable 168
catch (ObjectDisposedException ex)
#pragma warning restore 168
{
// expected
}
assertFalse(success.Get());
assertTrue(triedReopen.Get());
assertNull("" + exc[0], exc[0]);
writer.Dispose();
dir.Dispose();
//if (es != null)
//{
// es.shutdown();
// es.awaitTermination(1, TimeUnit.SECONDS);
//}
}
public LiveFieldValuesAnonymousInnerClassHelper(SearcherManager mgr, int missing)
: base(mgr, missing)
{
}
// LUCENE-5461
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random().Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_46, new MockAnalyzer(Random()));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.SetDaemon(true);
controlledRealTimeReopenThread.Start();
IList <Thread> commitThreads = new List <Thread>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
Thread commitThread = new Thread(new RunnableAnonymousInnerClassHelper(this, sdp, dir, iw));
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = DateTime.Now.Millisecond;
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = DateTime.Now.Millisecond - start;
Assert.IsTrue(wait < (maxStaleSecs * 1000), "waited too long for generation " + wait);
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
Assert.AreEqual(1, td.TotalHits);
}
foreach (Thread commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public void TestMultithreadedWaitForGeneration()
{
Thread CreateWorker(int threadNum, ControlledRealTimeReopenThread <IndexSearcher> controlledReopen, long generation,
SearcherManager searcherManager, List <ThreadOutput> outputList)
{
ThreadStart threadStart = delegate
{
Stopwatch stopwatch = Stopwatch.StartNew();
controlledReopen.WaitForGeneration(generation);
stopwatch.Stop();
double milliSecsWaited = stopwatch.Elapsed.TotalMilliseconds;
int numRecs = 0;
IndexSearcher indexSearcher = searcherManager.Acquire();
try
{
TopDocs topDocs = indexSearcher.Search(new MatchAllDocsQuery(), 1);
numRecs = topDocs.TotalHits;
}
finally
{
searcherManager.Release(indexSearcher);
}
lock (outputList)
{
outputList.Add(new ThreadOutput {
ThreadNum = threadNum, NumRecs = numRecs, MilliSecsWaited = milliSecsWaited
});
}
};
return(new Thread(threadStart));
}
int threadCount = 3;
List <ThreadOutput> outputList = new List <ThreadOutput>();
RAMDirectory indexDir = new RAMDirectory();
Analyzer standardAnalyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
IndexWriterConfig indexConfig = new IndexWriterConfig(TEST_VERSION_CURRENT, standardAnalyzer);
IndexWriter indexWriter = new IndexWriter(indexDir, indexConfig);
TrackingIndexWriter trackingWriter = new TrackingIndexWriter(indexWriter);
//Add two documents
Document doc = new Document();
doc.Add(new Int32Field("id", 1, Field.Store.YES));
doc.Add(new StringField("name", "Doc1", Field.Store.YES));
long generation = trackingWriter.AddDocument(doc);
doc.Add(new Int32Field("id", 2, Field.Store.YES));
doc.Add(new StringField("name", "Doc3", Field.Store.YES));
generation = trackingWriter.AddDocument(doc);
SearcherManager searcherManager = new SearcherManager(indexWriter, applyAllDeletes: true, null);
//Reopen SearcherManager every 2 secs via background thread if no thread waiting for newer generation.
//Reopen SearcherManager after .2 secs if another thread IS waiting on a newer generation.
double maxRefreshSecs = 2.0;
double minRefreshSecs = .2;
var controlledRealTimeReopenThread = new ControlledRealTimeReopenThread <IndexSearcher>(trackingWriter, searcherManager, maxRefreshSecs, minRefreshSecs);
//Start() will start a seperate thread that will invoke the object's Run(). However,
//calling Run() directly would execute that code on the current thread rather then a new thread
//which would defeat the purpose of using controlledRealTimeReopenThread. This aspect of the API
//is not as intuitive as it could be. ie. Call Start() not Run().
controlledRealTimeReopenThread.IsBackground = true; //Set as a background thread
controlledRealTimeReopenThread.Name = "Controlled Real Time Reopen Thread";
controlledRealTimeReopenThread.Priority = (ThreadPriority)Math.Min((int)Thread.CurrentThread.Priority + 2, (int)ThreadPriority.Highest);
controlledRealTimeReopenThread.Start();
//Create the threads for doing searchers
List <Thread> threadList = new List <Thread>();
for (int i = 1; i <= threadCount; i++)
{
threadList.Add(CreateWorker(i, controlledRealTimeReopenThread, generation, searcherManager, outputList));
}
//Start all the threads
foreach (Thread thread in threadList)
{
thread.Start();
}
//wait for the threads to finish.
foreach (Thread thread in threadList)
{
thread.Join(); //will wait here until the thread terminates.
}
//Now make sure that no thread waited longer then our min refresh time
//plus a small fudge factor. Also verify that all threads resported back and
//each saw 2 records.
//Verify all threads reported back a result.
assertEquals(threadCount, outputList.Count);
int millisecsPerSec = 1000;
foreach (ThreadOutput output in outputList)
{
//Verify the thread saw exactly 2 docs
assertEquals(2, output.NumRecs);
//Verify the thread wait time was around what was expected.
Assert.True(output.MilliSecsWaited <= (minRefreshSecs * millisecsPerSec) + 30); //30ms is fudged factor to account for call overhead
}
controlledRealTimeReopenThread.Dispose(); //will kill and join to the thread
Assert.False(controlledRealTimeReopenThread.IsAlive); //to prove that Dispose really does kill the thread.
searcherManager.Dispose();
indexWriter.Dispose();
indexDir.Dispose();
}
public virtual void TestTryDeleteDocument()
{
Directory directory = CreateIndex();
IndexWriter writer = GetWriter(directory);
ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
IndexSearcher searcher = mgr.Acquire();
TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(1, topDocs.TotalHits);
long result;
if (Random().NextBoolean())
{
IndexReader r = DirectoryReader.Open(writer, true);
result = mgrWriter.TryDeleteDocument(r, 0);
r.Dispose();
}
else
{
result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0);
}
// The tryDeleteDocument should have succeeded:
Assert.IsTrue(result != -1);
Assert.IsTrue(writer.HasDeletions());
if (Random().NextBoolean())
{
writer.Commit();
}
Assert.IsTrue(writer.HasDeletions());
mgr.MaybeRefresh();
searcher = mgr.Acquire();
topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
Assert.AreEqual(0, topDocs.TotalHits);
}
public LiveFieldValuesAnonymousInnerClassHelper(TestLiveFieldValues outerInstance, SearcherManager mgr, int missing)
: base(mgr, missing)
{
this.OuterInstance = outerInstance;
}
public virtual void TestCRTReopen()
{
//test behaving badly
//should be high enough
int maxStaleSecs = 20;
//build crap data just to store it.
string s = " abcdefghijklmnopqrstuvwxyz ";
char[] chars = s.ToCharArray();
StringBuilder builder = new StringBuilder(2048);
for (int i = 0; i < 2048; i++)
{
builder.Append(chars[Random.Next(chars.Length)]);
}
string content = builder.ToString();
SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
Directory dir = new NRTCachingDirectory(NewFSDirectory(CreateTempDir("nrt")), 5, 128);
IndexWriterConfig config = new IndexWriterConfig(
#pragma warning disable 612, 618
Version.LUCENE_46,
#pragma warning restore 612, 618
new MockAnalyzer(Random));
config.SetIndexDeletionPolicy(sdp);
config.SetOpenMode(OpenMode.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dir, config);
SearcherManager sm = new SearcherManager(iw, true, new SearcherFactory());
TrackingIndexWriter tiw = new TrackingIndexWriter(iw);
ControlledRealTimeReopenThread <IndexSearcher> controlledRealTimeReopenThread =
new ControlledRealTimeReopenThread <IndexSearcher>(tiw, sm, maxStaleSecs, 0);
controlledRealTimeReopenThread.IsBackground = (true);
controlledRealTimeReopenThread.Start();
IList <ThreadJob> commitThreads = new JCG.List <ThreadJob>();
for (int i = 0; i < 500; i++)
{
if (i > 0 && i % 50 == 0)
{
ThreadJob commitThread = new RunnableAnonymousClass(this, sdp, dir, iw);
commitThread.Start();
commitThreads.Add(commitThread);
}
Document d = new Document();
d.Add(new TextField("count", i + "", Field.Store.NO));
d.Add(new TextField("content", content, Field.Store.YES));
long start = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
long l = tiw.AddDocument(d);
controlledRealTimeReopenThread.WaitForGeneration(l);
long wait = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - start; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
assertTrue("waited too long for generation " + wait, wait < (maxStaleSecs * 1000));
IndexSearcher searcher = sm.Acquire();
TopDocs td = searcher.Search(new TermQuery(new Term("count", i + "")), 10);
sm.Release(searcher);
assertEquals(1, td.TotalHits);
}
foreach (ThreadJob commitThread in commitThreads)
{
commitThread.Join();
}
controlledRealTimeReopenThread.Dispose();
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public ThreadAnonymousInnerClassHelper2(TestSearcherManager outerInstance, SearcherManager sm)
{
this.outerInstance = outerInstance;
this.sm = sm;
}
public LiveFieldValuesAnonymousClass(SearcherManager mgr, int missing)
: base(mgr, missing)
{
}