public virtual void TestMixedTypesDifferentThreads()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
CountDownLatch startingGun = new CountDownLatch(1);
AtomicBoolean hitExc = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < 3; i++)
{
Field field;
if (i == 0)
{
field = new SortedDocValuesField("foo", new BytesRef("hello"));
}
else if (i == 1)
{
field = new NumericDocValuesField("foo", 0);
}
else
{
field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
}
Document doc = new Document();
doc.Add(field);
threads[i] = new ThreadAnonymousInnerClassHelper(this, w, startingGun, hitExc, doc);
threads[i].Start();
}
startingGun.countDown();
foreach (ThreadClass t in threads)
{
t.Join();
}
Assert.IsTrue(hitExc.Get());
w.Dispose();
dir.Dispose();
}
public virtual void TestApplyDeletesOnFlush()
{
Directory dir = NewDirectory();
// Cannot use RandomIndexWriter because we don't want to
// ever call commit() for this test:
AtomicInteger docsInSegment = new AtomicInteger();
AtomicBoolean closing = new AtomicBoolean();
AtomicBoolean sawAfterFlush = new AtomicBoolean();
IndexWriter w = new IndexWriterAnonymousInnerClassHelper(this, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.5).SetMaxBufferedDocs(-1).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false), docsInSegment, closing, sawAfterFlush);
int id = 0;
while (true)
{
StringBuilder sb = new StringBuilder();
for (int termIDX = 0; termIDX < 100; termIDX++)
{
sb.Append(' ').Append(TestUtil.RandomRealisticUnicodeString(Random()));
}
if (id == 500)
{
w.DeleteDocuments(new Term("id", "0"));
}
Document doc = new Document();
doc.Add(NewStringField("id", "" + id, Field.Store.NO));
doc.Add(NewTextField("body", sb.ToString(), Field.Store.NO));
w.UpdateDocument(new Term("id", "" + id), doc);
docsInSegment.IncrementAndGet();
// TODO: fix this test
if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv"))
{
if (VERBOSE)
{
Console.WriteLine("TEST: deletes created @ id=" + id);
}
break;
}
id++;
}
closing.Set(true);
Assert.IsTrue(sawAfterFlush.Get());
w.Dispose();
dir.Dispose();
}
public virtual void TestMaxMergeCount()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
int maxMergeCount = TestUtil.NextInt(Random(), 1, 5);
int maxMergeThreads = TestUtil.NextInt(Random(), 1, maxMergeCount);
CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount);
AtomicInteger runningMergeCount = new AtomicInteger(0);
AtomicBoolean failed = new AtomicBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads);
}
ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed);
cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
iwc.SetMergeScheduler(cms);
iwc.SetMaxBufferedDocs(2);
TieredMergePolicy tmp = new TieredMergePolicy();
iwc.SetMergePolicy(tmp);
tmp.MaxMergeAtOnce = 2;
tmp.SegmentsPerTier = 2;
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED));
while (enoughMergesWaiting.CurrentCount != 0 && !failed.Get())
{
for (int i = 0; i < 10; i++)
{
w.AddDocument(doc);
}
}
w.Dispose(false);
dir.Dispose();
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadClass[numReleasers + numStallers + numWaiters];
IList<Exception> exceptions = new SynchronizedCollection<Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint.Get())
{
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked() && ctrl.Healthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Set(false);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
}
Assert.IsFalse(checkPoint.Get());
Assert.AreEqual(0, sync.Waiter.Remaining);
if (checkPointProbability >= (float)Random().NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
}
if (!checkPoint.Get())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Set(false);
stop.Set(true);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public virtual void TestCommitThreadSafety()
{
const int NUM_THREADS = 5;
const double RUN_SEC = 0.5;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
TestUtil.ReduceOpenFiles(w.w);
w.Commit();
AtomicBoolean failed = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
long endTime = DateTime.Now.Millisecond + ((long)(RUN_SEC * 1000));
for (int i = 0; i < NUM_THREADS; i++)
{
int finalI = i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, dir, w, failed, endTime, finalI);
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
Assert.IsFalse(failed.Get());
w.Dispose();
dir.Dispose();
}
public virtual void TestListenerCalled()
{
Directory dir = NewDirectory();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
AtomicBoolean afterRefreshCalled = new AtomicBoolean(false);
SearcherManager sm = new SearcherManager(iw, false, new SearcherFactory());
sm.AddListener(new RefreshListenerAnonymousInnerClassHelper(this, afterRefreshCalled));
iw.AddDocument(new Document());
iw.Commit();
Assert.IsFalse(afterRefreshCalled.Get());
sm.MaybeRefreshBlocking();
Assert.IsTrue(afterRefreshCalled.Get());
sm.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestCustomDoublesValueSource()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir);
Document doc = new Document();
writer.AddDocument(doc);
writer.AddDocument(doc);
writer.AddDocument(doc);
// Test wants 3 docs in one segment:
writer.ForceMerge(1);
var vs = new ValueSourceAnonymousInnerClassHelper(this, doc);
FacetsConfig config = new FacetsConfig();
FacetsCollector fc = new FacetsCollector();
IndexReader r = writer.Reader;
IndexSearcher s = NewSearcher(r);
s.Search(new MatchAllDocsQuery(), fc);
DoubleRange[] ranges = new DoubleRange[] { new DoubleRange("< 1", 0.0, true, 1.0, false), new DoubleRange("< 2", 0.0, true, 2.0, false), new DoubleRange("< 5", 0.0, true, 5.0, false), new DoubleRange("< 10", 0.0, true, 10.0, false), new DoubleRange("< 20", 0.0, true, 20.0, false), new DoubleRange("< 50", 0.0, true, 50.0, false) };
Filter fastMatchFilter;
AtomicBoolean filterWasUsed = new AtomicBoolean();
if (Random().NextBoolean())
{
// Sort of silly:
fastMatchFilter = new CachingWrapperFilterAnonymousInnerClassHelper(this, new QueryWrapperFilter(new MatchAllDocsQuery()), filterWasUsed);
}
else
{
fastMatchFilter = null;
}
if (VERBOSE)
{
Console.WriteLine("TEST: fastMatchFilter=" + fastMatchFilter);
}
Facets facets = new DoubleRangeFacetCounts("field", vs, fc, fastMatchFilter, ranges);
Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", facets.GetTopChildren(10, "field").ToString());
Assert.True(fastMatchFilter == null || filterWasUsed.Get());
DrillDownQuery ddq = new DrillDownQuery(config);
ddq.Add("field", ranges[1].GetFilter(fastMatchFilter, vs));
// Test simple drill-down:
Assert.AreEqual(1, s.Search(ddq, 10).TotalHits);
// Test drill-sideways after drill-down
DrillSideways ds = new DrillSidewaysAnonymousInnerClassHelper2(this, s, config, (TaxonomyReader)null, vs, ranges, fastMatchFilter);
DrillSidewaysResult dsr = ds.Search(ddq, 10);
Assert.AreEqual(1, dsr.Hits.TotalHits);
Assert.AreEqual("dim=field path=[] value=3 childCount=6\n < 1 (0)\n < 2 (1)\n < 5 (3)\n < 10 (3)\n < 20 (3)\n < 50 (3)\n", dsr.Facets.GetTopChildren(10, "field").ToString());
IOUtils.Close(r, writer, dir);
}
/*
* LUCENE-3528 - NRTManager hangs in certain situations
*/
public virtual void TestThreadStarvationNoDeleteNRTReader()
{
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
conf.SetMergePolicy(Random().NextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
Directory d = NewDirectory();
CountDownLatch latch = new CountDownLatch(1);
CountDownLatch signal = new CountDownLatch(1);
LatchedIndexWriter _writer = new LatchedIndexWriter(d, conf, latch, signal);
TrackingIndexWriter writer = new TrackingIndexWriter(_writer);
SearcherManager manager = new SearcherManager(_writer, false, null);
Document doc = new Document();
doc.Add(NewTextField("test", "test", Field.Store.YES));
writer.AddDocument(doc);
manager.MaybeRefresh();
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, latch, signal, writer, manager);
t.Start();
_writer.WaitAfterUpdate = true; // wait in addDocument to let some reopens go through
long lastGen = writer.UpdateDocument(new Term("foo", "bar"), doc); // once this returns the doc is already reflected in the last reopen
Assert.IsFalse(manager.SearcherCurrent); // false since there is a delete in the queue
IndexSearcher searcher = manager.Acquire();
try
{
Assert.AreEqual(2, searcher.IndexReader.NumDocs());
}
finally
{
manager.Release(searcher);
}
ControlledRealTimeReopenThread<IndexSearcher> thread = new ControlledRealTimeReopenThread<IndexSearcher>(writer, manager, 0.01, 0.01);
thread.Start(); // start reopening
if (VERBOSE)
{
Console.WriteLine("waiting now for generation " + lastGen);
}
AtomicBoolean finished = new AtomicBoolean(false);
ThreadClass waiter = new ThreadAnonymousInnerClassHelper2(this, lastGen, thread, finished);
waiter.Start();
manager.MaybeRefresh();
waiter.Join(1000);
if (!finished.Get())
{
waiter.Interrupt();
Assert.Fail("thread deadlocked on waitForGeneration");
}
thread.Dispose();
thread.Join();
IOUtils.Close(manager, _writer, d);
}
public virtual void TestRollbackAndCommitWithThreads()
{
BaseDirectoryWrapper d = NewDirectory();
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).PreventDoubleWrite = false;
}
int threadCount = TestUtil.NextInt(Random(), 2, 6);
AtomicReference<IndexWriter> writerRef = new AtomicReference<IndexWriter>();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
writerRef.Value = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
LineFileDocs docs = new LineFileDocs(Random());
ThreadClass[] threads = new ThreadClass[threadCount];
int iters = AtLeast(100);
AtomicBoolean failed = new AtomicBoolean();
ReentrantLock rollbackLock = new ReentrantLock();
ReentrantLock commitLock = new ReentrantLock();
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, d, writerRef, docs, iters, failed, rollbackLock, commitLock);
threads[threadID].Start();
}
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID].Join();
}
Assert.IsTrue(!failed.Get());
writerRef.Value.Dispose();
d.Dispose();
}
public virtual void TestDirectory()
{
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriter w = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
var tw = new DirectoryTaxonomyWriter(taxoDir);
// first empty commit
w.Commit();
tw.Commit();
var mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, indexDir);
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, dir);
}
public virtual void TestSimpleMergedSegmentWramer()
{
Directory dir = NewDirectory();
AtomicBoolean didWarm = new AtomicBoolean();
InfoStream infoStream = new InfoStreamAnonymousInnerClassHelper(this, didWarm);
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetReaderPooling(true).SetInfoStream(infoStream).SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream)).SetMergePolicy(NewLogMergePolicy(10)));
Document doc = new Document();
doc.Add(NewStringField("foo", "bar", Field.Store.NO));
for (int i = 0; i < 20; i++)
{
w.AddDocument(doc);
}
w.WaitForMerges();
w.Dispose();
dir.Dispose();
Assert.IsTrue(didWarm.Get());
}
public virtual void TestGetDocsWithFieldThreadSafety()
{
IFieldCache cache = FieldCache.DEFAULT;
cache.PurgeAllCaches();
int NUM_THREADS = 3;
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
AtomicBoolean failed = new AtomicBoolean();
AtomicInteger iters = new AtomicInteger();
int NUM_ITER = 200 * RANDOM_MULTIPLIER;
Barrier restart = new Barrier(NUM_THREADS, (barrier) => new RunnableAnonymousInnerClassHelper(this, cache, iters).Run());
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX] = new ThreadAnonymousInnerClassHelper(this, cache, failed, iters, NUM_ITER, restart);
threads[threadIDX].Start();
}
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX].Join();
}
Assert.IsFalse(failed.Get());
}
public virtual void TestIntermediateClose()
{
Directory dir = NewDirectory();
// Test can deadlock if we use SMS:
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new ConcurrentMergeScheduler()));
writer.AddDocument(new Document());
writer.Commit();
CountdownEvent awaitEnterWarm = new CountdownEvent(1);
CountdownEvent awaitClose = new CountdownEvent(1);
AtomicBoolean triedReopen = new AtomicBoolean(false);
//TaskScheduler es = Random().NextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
TaskScheduler es = Random().NextBoolean() ? null : TaskScheduler.Default;
SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper2(this, awaitEnterWarm, awaitClose, triedReopen, es);
SearcherManager searcherManager = Random().NextBoolean() ? new SearcherManager(dir, factory) : new SearcherManager(writer, Random().NextBoolean(), factory);
if (VERBOSE)
{
Console.WriteLine("sm created");
}
IndexSearcher searcher = searcherManager.Acquire();
try
{
Assert.AreEqual(1, searcher.IndexReader.NumDocs);
}
finally
{
searcherManager.Release(searcher);
}
writer.AddDocument(new Document());
writer.Commit();
AtomicBoolean success = new AtomicBoolean(false);
Exception[] exc = new Exception[1];
ThreadClass thread = new ThreadClass(() => new RunnableAnonymousInnerClassHelper(this, triedReopen, searcherManager, success, exc).Run());
thread.Start();
if (VERBOSE)
{
Console.WriteLine("THREAD started");
}
awaitEnterWarm.Wait();
if (VERBOSE)
{
Console.WriteLine("NOW call close");
}
searcherManager.Dispose();
awaitClose.Signal();
thread.Join();
try
{
searcherManager.Acquire();
Assert.Fail("already closed");
}
catch (AlreadyClosedException ex)
{
// expected
}
Assert.IsFalse(success.Get());
Assert.IsTrue(triedReopen.Get());
Assert.IsNull(exc[0], "" + exc[0]);
writer.Dispose();
dir.Dispose();
//if (es != null)
//{
// es.shutdown();
// es.awaitTermination(1, TimeUnit.SECONDS);
//}
}