public virtual void TestIndexing()
{
DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete");
BaseDirectoryWrapper d = NewFSDirectory(tmpDir);
// We want to "see" files removed if Lucene removed
// them. this is still worth running on Windows since
// some files the IR opens and closes.
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).NoDeleteOpenFile = false;
}
RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30));
w.Commit();
ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)];
long stopTime = Environment.TickCount + AtLeast(1000);
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
HashSet<string> allFiles = new HashSet<string>();
DirectoryReader r = DirectoryReader.Open(d);
while (Environment.TickCount < stopTime)
{
IndexCommit ic = r.IndexCommit;
if (VERBOSE)
{
Console.WriteLine("TEST: check files: " + ic.FileNames);
}
allFiles.AddAll(ic.FileNames);
// Make sure no old files were removed
foreach (string fileName in allFiles)
{
Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist");
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Thread.Sleep(1);
}
r.Dispose();
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
w.Dispose();
d.Dispose();
System.IO.Directory.Delete(tmpDir.FullName, true);
}
public virtual void TestRandom()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
ThreadClass[] stallThreads = new ThreadClass[AtLeast(3)];
for (int i = 0; i < stallThreads.Length; i++)
{
int stallProbability = 1 + Random().Next(10);
stallThreads[i] = new ThreadAnonymousInnerClassHelper(this, ctrl, stallProbability);
}
Start(stallThreads);
long time = DateTime.Now.Millisecond;
/*
* use a 100 sec timeout to make sure we not hang forever. join will fail in
* that case
*/
while ((DateTime.Now.Millisecond - time) < 100 * 1000 && !Terminated(stallThreads))
{
ctrl.UpdateStalled(false);
if (Random().NextBoolean())
{
Thread.@Yield();
}
else
{
Thread.Sleep(1);
}
}
Join(stallThreads);
}
private void Dotest(int ncats, int range)
{
AtomicInteger numCats = new AtomicInteger(ncats);
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.Length; i++)
{
dirs[i] = NewDirectory();
var tw = new DirectoryTaxonomyWriter(dirs[i]);
ThreadClass[] addThreads = new ThreadClass[4];
for (int j = 0; j < addThreads.Length; j++)
{
addThreads[j] = new ThreadAnonymousInnerClassHelper(this, range, numCats, tw);
}
foreach (ThreadClass t in addThreads)
{
t.Start();
}
foreach (ThreadClass t in addThreads)
{
t.Join();
}
tw.Dispose();
}
var tw1 = new DirectoryTaxonomyWriter(dirs[0]);
IOrdinalMap map = randomOrdinalMap();
tw1.AddTaxonomy(dirs[1], map);
tw1.Dispose();
validate(dirs[0], dirs[1], map);
IOUtils.Close(dirs);
}
public static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("Usage: java Lucene.Net.Store.LockVerifyServer bindToIp clients\n");
Environment.Exit(1);
}
int arg = 0;
IPHostEntry ipHostInfo = Dns.GetHostEntry(args[arg++]);
IPAddress ipAddress = ipHostInfo.AddressList[0];
IPEndPoint localEndPoint = new IPEndPoint(ipAddress, 0);
int maxClients = Convert.ToInt32(args[arg++]);
using (Socket s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp))
{
s.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);
s.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, 30000);// SoTimeout = 30000; // initially 30 secs to give clients enough time to startup
s.Bind(localEndPoint);
Console.WriteLine("Listening on " + ((IPEndPoint)s.LocalEndPoint).Port.ToString() + "...");
// we set the port as a sysprop, so the ANT task can read it. For that to work, this server must run in-process:
System.Environment.SetEnvironmentVariable("lockverifyserver.port", ((IPEndPoint)s.LocalEndPoint).Port.ToString());
object localLock = new object();
int[] lockedID = new int[1];
lockedID[0] = -1;
CountdownEvent startingGun = new CountdownEvent(1);
ThreadClass[] threads = new ThreadClass[maxClients];
for (int count = 0; count < maxClients; count++)
{
Socket cs = s.Accept();
threads[count] = new ThreadAnonymousInnerClassHelper(localLock, lockedID, startingGun, cs);
threads[count].Start();
}
// start
Console.WriteLine("All clients started, fire gun...");
startingGun.Signal();
// wait for all threads to finish
foreach (ThreadClass t in threads)
{
t.Join();
}
//LUCENE TO-DO Not sure if equivalent?
// cleanup sysprop
//System.clearProperty("lockverifyserver.port");
Console.WriteLine("Server terminated.");
}
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Ignore public void testThreadLeak()
public virtual void TestThreadLeak()
{
Thread t = new ThreadAnonymousInnerClassHelper(this);
t.Start();
while (!t.IsAlive)
{
Thread.@Yield();
}
// once alive, leave it to run outside of the test scope.
}
private void RunTest(Random random, Directory dir)
{
// Run for ~1 seconds
long stopTime = Environment.TickCount + 1000;
SnapshotDeletionPolicy dp = DeletionPolicy;
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetIndexDeletionPolicy(dp).SetMaxBufferedDocs(2));
// Verify we catch misuse:
try
{
dp.Snapshot();
Assert.Fail("did not hit exception");
}
#pragma warning disable 168
catch (InvalidOperationException ise)
#pragma warning restore 168
{
// expected
}
dp = (SnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
writer.Commit();
ThreadClass t = new ThreadAnonymousInnerClassHelper(stopTime, writer, NewField);
t.Start();
// While the above indexing thread is running, take many
// backups:
do
{
BackupIndex(dir, dp);
Thread.Sleep(20);
} while (t.IsAlive);
t.Join();
// Add one more document to force writer to commit a
// final segment, so deletion policy has a chance to
// delete again:
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("content", "aaa", customType));
writer.AddDocument(doc);
// Make sure we don't have any leftover files in the
// directory:
writer.Dispose();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
}
private ThreadClass[] LaunchIndexingThreads(LineFileDocs docs, int numThreads, DateTime stopTime, ISet <string> delIDs, ISet <string> delPackIDs, IList <SubDocs> allSubDocs)
{
ThreadClass[] threads = new ThreadClass[numThreads];
for (int thread = 0; thread < numThreads; thread++)
{
threads[thread] = new ThreadAnonymousInnerClassHelper(this, docs, stopTime, delIDs, delPackIDs, allSubDocs);
threads[thread].SetDaemon(true);
threads[thread].Start();
}
return(threads);
}
public virtual void Test()
{
Directory dir = NewFSDirectory(CreateTempDir("livefieldupdates"));
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
IndexWriter w = new IndexWriter(dir, iwc);
SearcherManager mgr = new SearcherManager(w, true, new SearcherFactoryAnonymousInnerClassHelper());
const int missing = -1;
LiveFieldValues <IndexSearcher, int?> rt = new LiveFieldValuesAnonymousInnerClassHelper(mgr, missing);
int numThreads = TestUtil.NextInt32(Random, 2, 5);
if (VERBOSE)
{
Console.WriteLine(numThreads + " threads");
}
CountdownEvent startingGun = new CountdownEvent(1);
IList <ThreadJob> threads = new List <ThreadJob>();
int iters = AtLeast(1000);
int idCount = TestUtil.NextInt32(Random, 100, 10000);
double reopenChance = Random.NextDouble() * 0.01;
double deleteChance = Random.NextDouble() * 0.25;
double addChance = Random.NextDouble() * 0.5;
for (int t = 0; t < numThreads; t++)
{
int threadID = t;
Random threadRandom = new Random(Random.Next());
ThreadJob thread = new ThreadAnonymousInnerClassHelper(w, mgr, missing, rt, startingGun, iters, idCount, reopenChance, deleteChance, addChance, t, threadID, threadRandom);
threads.Add(thread);
thread.Start();
}
startingGun.Signal();
foreach (ThreadJob thread in threads)
{
thread.Join();
}
mgr.MaybeRefresh();
Assert.AreEqual(0, rt.Count);
rt.Dispose();
mgr.Dispose();
w.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
IList<long?> numbers = new List<long?>();
IList<BytesRef> binary = new List<BytesRef>();
IList<BytesRef> sorted = new List<BytesRef>();
int numDocs = AtLeast(100);
for (int i = 0; i < numDocs; i++)
{
Document d = new Document();
long number = Random().NextLong();
d.Add(new NumericDocValuesField("number", number));
BytesRef bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()));
d.Add(new BinaryDocValuesField("bytes", bytes));
binary.Add(bytes);
bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random()));
d.Add(new SortedDocValuesField("sorted", bytes));
sorted.Add(bytes);
w.AddDocument(d);
numbers.Add(number);
}
w.ForceMerge(1);
IndexReader r = w.Reader;
w.Dispose();
Assert.AreEqual(1, r.Leaves.Count);
AtomicReader ar = (AtomicReader)r.Leaves[0].Reader;
int numThreads = TestUtil.NextInt(Random(), 2, 5);
IList<ThreadClass> threads = new List<ThreadClass>();
CountDownLatch startingGun = new CountDownLatch(1);
for (int t = 0; t < numThreads; t++)
{
Random threadRandom = new Random(Random().Next());
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, numbers, binary, sorted, numDocs, ar, startingGun, threadRandom);
thread.Start();
threads.Add(thread);
}
startingGun.countDown();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
r.Dispose();
dir.Dispose();
}
public virtual void TestThreadLeak()
{
ThreadClass t = new ThreadAnonymousInnerClassHelper(this);
t.Start();
while (!t.IsAlive)
{
Thread.Yield();
}
// once alive, leave it to run outside of the test scope.
}
public virtual void AssertThreadSafe(Analyzer analyzer)
{
int numTestPoints = 100;
int numThreads = TestUtil.NextInt(Random(), 3, 5);
Dictionary <string, BytesRef> map = new Dictionary <string, BytesRef>();
// create a map<String,SortKey> up front.
// then with multiple threads, generate sort keys for all the keys in the map
// and ensure they are the same as the ones we produced in serial fashion.
for (int i = 0; i < numTestPoints; i++)
{
string term = TestUtil.RandomSimpleString(Random());
IOException priorException = null;
TokenStream ts = analyzer.TokenStream("fake", new StreamReader(term));
try
{
ITermToBytesRefAttribute termAtt = ts.AddAttribute <ITermToBytesRefAttribute>();
BytesRef bytes = termAtt.BytesRef;
ts.Reset();
Assert.IsTrue(ts.IncrementToken());
termAtt.FillBytesRef();
// ensure we make a copy of the actual bytes too
map[term] = BytesRef.DeepCopyOf(bytes);
Assert.IsFalse(ts.IncrementToken());
ts.End();
}
catch (IOException e)
{
priorException = e;
}
finally
{
IOUtils.CloseWhileHandlingException(priorException, ts);
}
}
ThreadClass[] threads = new ThreadClass[numThreads];
for (int i = 0; i < numThreads; i++)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, analyzer, map);
}
for (int i = 0; i < numThreads; i++)
{
threads[i].Start();
}
for (int i = 0; i < numThreads; i++)
{
threads[i].Join();
}
}
public virtual void Test()
{
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
MyIndexWriter w = new MyIndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
// Try to make an index that requires merging:
w.Config.SetMaxBufferedDocs(TestUtil.NextInt32(Random, 2, 11));
int numStartDocs = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
for (int docIDX = 0; docIDX < numStartDocs; docIDX++)
{
w.AddDocument(docs.NextDoc());
}
MergePolicy mp = w.Config.MergePolicy;
int mergeAtOnce = 1 + w.segmentInfos.Count;
if (mp is TieredMergePolicy)
{
((TieredMergePolicy)mp).MaxMergeAtOnce = mergeAtOnce;
}
else if (mp is LogMergePolicy)
{
((LogMergePolicy)mp).MergeFactor = mergeAtOnce;
}
else
{
// skip test
w.Dispose();
d.Dispose();
return;
}
AtomicBoolean doStop = new AtomicBoolean();
w.Config.SetMaxBufferedDocs(2);
ThreadJob t = new ThreadAnonymousInnerClassHelper(this, w, numStartDocs, docs, doStop);
t.Start();
w.ForceMerge(1);
doStop.Value = true;
t.Join();
Assert.IsTrue(w.mergeCount <= 1, "merge count is " + w.mergeCount);
w.Dispose();
d.Dispose();
docs.Dispose();
}
public virtual void TestRollbackAndCommitWithThreads()
{
BaseDirectoryWrapper d = NewDirectory();
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).PreventDoubleWrite = false;
}
int threadCount = TestUtil.NextInt32(Random, 2, 6);
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
AtomicObject <IndexWriter> writerRef =
new AtomicObject <IndexWriter>(new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)));
LineFileDocs docs = new LineFileDocs(Random);
ThreadClass[] threads = new ThreadClass[threadCount];
int iters = AtLeast(100);
AtomicBoolean failed = new AtomicBoolean();
ReentrantLock rollbackLock = new ReentrantLock();
ReentrantLock commitLock = new ReentrantLock();
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, d, writerRef, docs, iters, failed, rollbackLock, commitLock);
threads[threadID].Start();
}
for (int threadID = 0; threadID < threadCount; threadID++)
{
try
{
threads[threadID].Join();
}
catch (Exception e)
{
Console.WriteLine("EXCEPTION in ThreadAnonymousInnerClassHelper: " + Environment.NewLine + e);
}
}
Assert.IsTrue(!failed.Get());
writerRef.Value.Dispose();
d.Dispose();
}
// run random tests from different threads to make sure the per-thread clones
// don't share mutable data
public virtual void TestClone()
{
RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
int numDocs = AtLeast(100);
foreach (Options options in ValidOptions())
{
RandomDocument[] docs = new RandomDocument[numDocs];
for (int i = 0; i < numDocs; ++i)
{
docs[i] = docFactory.NewDocument(TestUtil.NextInt32(Random, 1, 3), AtLeast(10), options);
}
AtomicObject <Exception> exception = new AtomicObject <Exception>();
using (Directory dir = NewDirectory())
using (RandomIndexWriter writer = new RandomIndexWriter(Random, dir, ClassEnvRule.similarity, ClassEnvRule.timeZone))
{
for (int i = 0; i < numDocs; ++i)
{
writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
}
using (IndexReader reader = writer.GetReader())
{
for (int i = 0; i < numDocs; ++i)
{
int docID = DocID(reader, "" + i);
AssertEquals(docs[i], reader.GetTermVectors(docID));
}
ThreadClass[] threads = new ThreadClass[2];
for (int i = 0; i < threads.Length; ++i)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, docs, reader, exception, i);
}
foreach (ThreadClass thread in threads)
{
thread.Start();
}
foreach (ThreadClass thread in threads)
{
thread.Join();
}
} // reader.Dispose();
}// writer.Dispose();, dir.Dispose();
Assert.IsNull(exception.Value, "One thread threw an exception");
}
}
public virtual void TestThreadSafety()
{
const int numThreads = 5;
int numDocs = AtLeast(50);
ByteArrayPool pool = new ByteArrayPool(numThreads, 5);
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
const string field = "test";
ThreadJob[] ingesters = new ThreadJob[numThreads];
for (int i = 0; i < numThreads; i++)
{
ingesters[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, pool, writer, field);
ingesters[i].Start();
}
for (int i = 0; i < numThreads; i++)
{
ingesters[i].Join();
}
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
TermsEnum terms = MultiFields.GetFields(reader).GetTerms(field).GetEnumerator();
IBits liveDocs = MultiFields.GetLiveDocs(reader);
DocsAndPositionsEnum tp = null;
while (terms.MoveNext())
{
string termText = terms.Term.Utf8ToString();
tp = terms.DocsAndPositions(liveDocs, tp);
while (tp.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
int freq = tp.Freq;
for (int i = 0; i < freq; i++)
{
tp.NextPosition();
BytesRef payload = tp.GetPayload();
Assert.AreEqual(termText, payload.Utf8ToString());
}
}
}
reader.Dispose();
dir.Dispose();
Assert.AreEqual(pool.Count, numThreads);
}
public virtual void Test()
{
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
MyIndexWriter w = new MyIndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
// Try to make an index that requires merging:
w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 2, 11));
int numStartDocs = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random(), DefaultCodecSupportsDocValues());
for (int docIDX = 0; docIDX < numStartDocs; docIDX++)
{
w.AddDocument(docs.NextDoc());
}
MergePolicy mp = w.Config.MergePolicy;
int mergeAtOnce = 1 + w.GetSegmentInfosSize_Nunit();
if (mp is TieredMergePolicy)
{
((TieredMergePolicy)mp).MaxMergeAtOnce = mergeAtOnce;
}
else if (mp is LogMergePolicy)
{
((LogMergePolicy)mp).MergeFactor = mergeAtOnce;
}
else
{
// skip test
w.Dispose();
d.Dispose();
return;
}
AtomicBoolean doStop = new AtomicBoolean();
w.Config.SetMaxBufferedDocs(2);
ThreadClass t = new ThreadAnonymousInnerClassHelper(this, w, numStartDocs, docs, doStop);
t.Start();
w.ForceMerge(1);
doStop.Set(true);
t.Join();
Assert.IsTrue(w.MergeCount.Get() <= 1, "merge count is " + w.MergeCount.Get());
w.Dispose();
d.Dispose();
docs.Dispose();
}
public virtual void TestWithThreads()
{
// LUCENE-5303: OrdinalsCache used the ThreadLocal BinaryDV instead of reader.getCoreCacheKey().
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(indexDir, conf);
var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
Document doc = new Document();
doc.Add(new FacetField("A", "1"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("A", "2"));
writer.AddDocument(config.Build(taxoWriter, doc));
var reader = DirectoryReader.Open(writer, true);
CachedOrdinalsReader ordsReader = new CachedOrdinalsReader(new DocValuesOrdinalsReader(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, "CachedOrdsThread-" + i, reader, ordsReader);
}
long ramBytesUsed = 0;
foreach (ThreadClass t in threads)
{
t.Start();
t.Join();
if (ramBytesUsed == 0)
{
ramBytesUsed = ordsReader.RamBytesUsed();
}
else
{
Assert.AreEqual(ramBytesUsed, ordsReader.RamBytesUsed());
}
}
IOUtils.Dispose(writer, taxoWriter, reader, indexDir, taxoDir);
}
public virtual void TestWithThreads()
{
// LUCENE-5303: OrdinalsCache used the ThreadLocal BinaryDV instead of reader.getCoreCacheKey().
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(indexDir, conf);
var taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
Document doc = new Document();
doc.Add(new FacetField("A", "1"));
writer.AddDocument(config.Build(taxoWriter, doc));
doc = new Document();
doc.Add(new FacetField("A", "2"));
writer.AddDocument(config.Build(taxoWriter, doc));
var reader = DirectoryReader.Open(writer, true);
CachedOrdinalsReader ordsReader = new CachedOrdinalsReader(new DocValuesOrdinalsReader(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, "CachedOrdsThread-" + i, reader, ordsReader);
}
long ramBytesUsed = 0;
foreach (ThreadClass t in threads)
{
t.Start();
t.Join();
if (ramBytesUsed == 0)
{
ramBytesUsed = ordsReader.RamBytesUsed();
}
else
{
Assert.AreEqual(ramBytesUsed, ordsReader.RamBytesUsed());
}
}
IOUtils.Close(writer, taxoWriter, reader, indexDir, taxoDir);
}
public virtual void TestMixedTypesDifferentThreads()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
CountDownLatch startingGun = new CountDownLatch(1);
AtomicBoolean hitExc = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < 3; i++)
{
Field field;
if (i == 0)
{
field = new SortedDocValuesField("foo", new BytesRef("hello"));
}
else if (i == 1)
{
field = new NumericDocValuesField("foo", 0);
}
else
{
field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
}
Document doc = new Document();
doc.Add(field);
threads[i] = new ThreadAnonymousInnerClassHelper(this, w, startingGun, hitExc, doc);
threads[i].Start();
}
startingGun.countDown();
foreach (ThreadClass t in threads)
{
t.Join();
}
Assert.IsTrue(hitExc.Get());
w.Dispose();
dir.Dispose();
}
public virtual void TestPartiallyAppliedGlobalSlice()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock",
BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
@lock.Lock();
var t = new ThreadAnonymousInnerClassHelper(this, queue);
t.Start();
t.Join();
@lock.Unlock();
Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
queue.TryApplyGlobalSlice();
Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
Assert.IsTrue(freezeGlobalBuffer.Any());
Assert.AreEqual(1, freezeGlobalBuffer.termCount);
Assert.IsFalse(queue.AnyChanges(), "all changes applied");
}
public virtual void TestHashCodeWithThreads()
{
AutomatonQuery[] queries = new AutomatonQuery[1000];
for (int i = 0; i < queries.Length; i++)
{
queries[i] = new AutomatonQuery(new Term("bogus", "bogus"), AutomatonTestUtil.RandomAutomaton(Random()));
}
CountdownEvent startingGun = new CountdownEvent(1);
int numThreads = TestUtil.NextInt(Random(), 2, 5);
ThreadClass[] threads = new ThreadClass[numThreads];
for (int threadID = 0; threadID < numThreads; threadID++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, queries, startingGun);
threads[threadID] = thread;
thread.Start();
}
startingGun.Signal();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
}
private void Dotest(int ncats, int range)
{
AtomicInt32 numCats = new AtomicInt32(ncats);
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.Length; i++)
{
dirs[i] = NewDirectory();
var tw = new DirectoryTaxonomyWriter(dirs[i]);
ThreadJob[] addThreads = new ThreadJob[4];
for (int j = 0; j < addThreads.Length; j++)
{
addThreads[j] = new ThreadAnonymousInnerClassHelper(this, range, numCats, tw);
}
foreach (ThreadJob t in addThreads)
{
t.Start();
}
foreach (ThreadJob t in addThreads)
{
t.Join();
}
tw.Dispose();
}
var tw1 = new DirectoryTaxonomyWriter(dirs[0]);
IOrdinalMap map = randomOrdinalMap();
tw1.AddTaxonomy(dirs[1], map);
tw1.Dispose();
validate(dirs[0], dirs[1], map);
IOUtils.Dispose(dirs);
}
internal virtual void LaunchThreads(int numIter)
{
for (int i = 0; i < OuterInstance.NumThreads; i++)
{
Threads[i] = new ThreadAnonymousInnerClassHelper(this, numIter);
}
for (int i = 0; i < OuterInstance.NumThreads; i++)
{
Threads[i].Start();
}
}
public virtual void TestConcurrency()
{
AtomicInteger numDocs = new AtomicInteger(AtLeast(10000));
Directory indexDir = NewDirectory();
Directory taxoDir = NewDirectory();
ConcurrentDictionary<string, string> values = new ConcurrentDictionary<string, string>();
IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
var tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, NewTaxoWriterCache(numDocs.Get()));
ThreadClass[] indexThreads = new ThreadClass[AtLeast(4)];
FacetsConfig config = new FacetsConfig();
for (int i = 0; i < 10; i++)
{
config.SetHierarchical("l1." + i, true);
config.SetMultiValued("l1." + i, true);
}
for (int i = 0; i < indexThreads.Length; i++)
{
indexThreads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, values, iw, tw, config);
}
foreach (ThreadClass t in indexThreads)
{
t.Start();
}
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
var tr = new DirectoryTaxonomyReader(tw);
// +1 for root category
if (values.Count + 1 != tr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (tr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
Fail("mismatch number of categories");
}
int[] parents = tr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.True(tr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = null;
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = tr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Close(tw, iw, tr, taxoDir, indexDir);
}
public virtual void TestNoWaitClose()
{
Directory directory = NewDirectory();
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.Tokenized = false;
Field idField = NewField("id", "", customType);
doc.Add(idField);
for (int pass = 0; pass < 2; pass++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: pass="******"TEST: iter=" + iter);
}
for (int j = 0; j < 199; j++)
{
idField.StringValue = Convert.ToString(iter * 201 + j);
writer.AddDocument(doc);
}
int delID = iter * 199;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(new Term("id", Convert.ToString(delID)));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 2;
IndexWriter finalWriter = writer;
List<Exception> failure = new List<Exception>();
ThreadClass t1 = new ThreadAnonymousInnerClassHelper(this, doc, finalWriter, failure);
if (failure.Count > 0)
{
throw failure[0];
}
t1.Start();
writer.Dispose(false);
t1.Join();
// Make sure reader can read
IndexReader reader = DirectoryReader.Open(directory);
reader.Dispose();
// Reopen
writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode_e.APPEND).SetMergePolicy(NewLogMergePolicy()));
}
writer.Dispose();
}
directory.Dispose();
}
public virtual void Test()
{
// update variables
int commitPercent = Random().Next(20);
int softCommitPercent = Random().Next(100); // what percent of the commits are soft
int deletePercent = Random().Next(50);
int deleteByQueryPercent = Random().Next(25);
int ndocs = AtLeast(50);
int nWriteThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5);
int maxConcurrentCommits = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max
bool tombstones = Random().NextBoolean();
// query variables
AtomicLong operations = new AtomicLong(AtLeast(10000)); // number of query operations to perform in total
int nReadThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5);
InitModel(ndocs);
FieldType storedOnlyType = new FieldType();
storedOnlyType.Stored = true;
if (VERBOSE)
{
Console.WriteLine("\n");
Console.WriteLine("TEST: commitPercent=" + commitPercent);
Console.WriteLine("TEST: softCommitPercent=" + softCommitPercent);
Console.WriteLine("TEST: deletePercent=" + deletePercent);
Console.WriteLine("TEST: deleteByQueryPercent=" + deleteByQueryPercent);
Console.WriteLine("TEST: ndocs=" + ndocs);
Console.WriteLine("TEST: nWriteThreads=" + nWriteThreads);
Console.WriteLine("TEST: nReadThreads=" + nReadThreads);
Console.WriteLine("TEST: maxConcurrentCommits=" + maxConcurrentCommits);
Console.WriteLine("TEST: tombstones=" + tombstones);
Console.WriteLine("TEST: operations=" + operations);
Console.WriteLine("\n");
}
AtomicInteger numCommitting = new AtomicInteger();
IList <ThreadClass> threads = new List <ThreadClass>();
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
writer.DoRandomForceMergeAssert = false;
writer.Commit();
Reader = DirectoryReader.Open(dir);
for (int i = 0; i < nWriteThreads; i++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, "WRITER" + i, commitPercent, softCommitPercent, deletePercent, deleteByQueryPercent, ndocs, maxConcurrentCommits, tombstones, operations, storedOnlyType, numCommitting, writer);
threads.Add(thread);
}
for (int i = 0; i < nReadThreads; i++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper2(this, "READER" + i, ndocs, tombstones, operations);
threads.Add(thread);
}
foreach (ThreadClass thread in threads)
{
thread.Start();
}
foreach (ThreadClass thread in threads)
{
thread.Join();
}
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: close reader=" + Reader);
}
Reader.Dispose();
dir.Dispose();
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testFoo() throws Exception
public virtual void TestFoo()
{
ThreadClass t = new ThreadAnonymousInnerClassHelper(this);
t.Start();
t.Join();
}
public virtual void TestConcurrency()
{
AtomicInt32 numDocs = new AtomicInt32(AtLeast(10000));
Directory indexDir = NewDirectory();
Directory taxoDir = NewDirectory();
ConcurrentDictionary <string, string> values = new ConcurrentDictionary <string, string>();
IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
var tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, NewTaxoWriterCache(numDocs));
ThreadJob[] indexThreads = new ThreadJob[AtLeast(4)];
FacetsConfig config = new FacetsConfig();
for (int i = 0; i < 10; i++)
{
config.SetHierarchical("l1." + i, true);
config.SetMultiValued("l1." + i, true);
}
for (int i = 0; i < indexThreads.Length; i++)
{
indexThreads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, values, iw, tw, config);
}
foreach (ThreadJob t in indexThreads)
{
t.Start();
}
foreach (ThreadJob t in indexThreads)
{
t.Join();
}
var tr = new DirectoryTaxonomyReader(tw);
// +1 for root category
if (values.Count + 1 != tr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (tr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
fail("mismatch number of categories");
}
int[] parents = tr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.IsTrue(tr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = null;
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = tr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Dispose(tw, iw, tr, taxoDir, indexDir);
}
private ThreadClass[] LaunchIndexingThreads(LineFileDocs docs, int numThreads, DateTime stopTime, ISet<string> delIDs, ISet<string> delPackIDs, IList<SubDocs> allSubDocs)
{
ThreadClass[] threads = new ThreadClass[numThreads];
for (int thread = 0; thread < numThreads; thread++)
{
threads[thread] = new ThreadAnonymousInnerClassHelper(this, docs, stopTime, delIDs, delPackIDs, allSubDocs);
threads[thread].SetDaemon(true);
threads[thread].Start();
}
return threads;
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"));
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1))
{
// Otherwise test can take way too long (> 2 hours)
numTerms /= 2;
}
if (VERBOSE)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList = CollectionsHelper.Shuffle(postingsList);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir(GetFullMethodName()));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int threadCount = TestUtil.NextInt(Random(), 1, 5);
if (VERBOSE)
{
Console.WriteLine("config: " + iw.w.Config);
Console.WriteLine("threadCount=" + threadCount);
}
Field prototype = NewTextField("field", "", Field.Store.NO);
FieldType fieldType = new FieldType((FieldType)prototype.FieldType);
if (Random().NextBoolean())
{
fieldType.OmitNorms = true;
}
int options = Random().Next(3);
if (options == 0)
{
fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS; // we dont actually need positions
fieldType.StoreTermVectors = true; // but enforce term vectors when we do this so we check SOMETHING
}
else if (options == 1 && !DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat("field")))
{
fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
// else just positions
ThreadClass[] threads = new ThreadClass[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
Random threadRandom = new Random(Random().Next());
Document document = new Document();
Field field = new Field("field", "", fieldType);
document.Add(field);
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, numTerms, maxTermsPerDoc, postings, iw, startingGun, threadRandom, document, field);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.Reader;
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.Terms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, terms.Size());
TermsEnum termsEnum = terms.Iterator(null);
BytesRef termBR;
while ((termBR = termsEnum.Next()) != null)
{
int value = Convert.ToInt32(termBR.Utf8ToString());
Assert.AreEqual(value, termsEnum.TotalTermFreq());
// don't really need to check more than this, as CheckIndex
// will verify that totalTermFreq == total number of positions seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, dir);
}
public virtual void TestMixedTypesDifferentThreads()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
CountDownLatch startingGun = new CountDownLatch(1);
AtomicBoolean hitExc = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[3];
for (int i = 0; i < 3; i++)
{
Field field;
if (i == 0)
{
field = new SortedDocValuesField("foo", new BytesRef("hello"));
}
else if (i == 1)
{
field = new NumericDocValuesField("foo", 0);
}
else
{
field = new BinaryDocValuesField("foo", new BytesRef("bazz"));
}
Document doc = new Document();
doc.Add(field);
threads[i] = new ThreadAnonymousInnerClassHelper(this, w, startingGun, hitExc, doc);
threads[i].Start();
}
startingGun.countDown();
foreach (ThreadClass t in threads)
{
t.Join();
}
Assert.IsTrue(hitExc.Get());
w.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
long startTime = Environment.TickCount;
// TODO: replace w/ the @nightly test data; make this
// into an optional @nightly stress test
Document doc = new Document();
Field body = NewTextField("body", "", Field.Store.NO);
doc.Add(body);
StringBuilder sb = new StringBuilder();
for (int docCount = 0; docCount < NUM_DOCS; docCount++)
{
int numTerms = Random().Next(10);
for (int termCount = 0; termCount < numTerms; termCount++)
{
sb.Append(Random().NextBoolean() ? "aaa" : "bbb");
sb.Append(' ');
}
body.SetStringValue(sb.ToString());
w.AddDocument(doc);
sb.Remove(0, sb.Length);
}
IndexReader r = w.Reader;
w.Dispose();
long endTime = Environment.TickCount;
if (VERBOSE)
{
Console.WriteLine("BUILD took " + (endTime - startTime));
}
IndexSearcher s = NewSearcher(r);
AtomicBoolean failed = new AtomicBoolean();
AtomicInt64 netSearch = new AtomicInt64();
ThreadClass[] threads = new ThreadClass[NUM_SEARCH_THREADS];
for (int threadID = 0; threadID < NUM_SEARCH_THREADS; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, s, failed, netSearch);
threads[threadID].SetDaemon(true);
}
foreach (ThreadClass t in threads)
{
t.Start();
}
foreach (ThreadClass t in threads)
{
t.Join();
}
if (VERBOSE)
{
Console.WriteLine(NUM_SEARCH_THREADS + " threads did " + netSearch.Get() + " searches");
}
r.Dispose();
dir.Dispose();
}
internal virtual void LaunchThreads(int numIter)
{
for (int i = 0; i < NUM_THREADS; i++)
{
Threads[i] = new ThreadAnonymousInnerClassHelper(this, numIter);
}
for (int i = 0; i < NUM_THREADS; i++)
{
Threads[i].Start();
}
}
public virtual void Test()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, analyzer, Similarity, TimeZone);
LineFileDocs docs = new LineFileDocs(Random(), DefaultCodecSupportsDocValues());
int charsToIndex = AtLeast(100000);
int charsIndexed = 0;
//System.out.println("bytesToIndex=" + charsToIndex);
while (charsIndexed < charsToIndex)
{
Document doc = docs.NextDoc();
charsIndexed += doc.Get("body").Length;
w.AddDocument(doc);
//System.out.println(" bytes=" + charsIndexed + " add: " + doc);
}
IndexReader r = w.Reader;
//System.out.println("numDocs=" + r.NumDocs);
w.Dispose();
IndexSearcher s = NewSearcher(r);
Terms terms = MultiFields.GetFields(r).Terms("body");
int termCount = 0;
TermsEnum termsEnum = terms.Iterator(null);
while (termsEnum.Next() != null)
{
termCount++;
}
Assert.IsTrue(termCount > 0);
// Target ~10 terms to search:
double chance = 10.0 / termCount;
termsEnum = terms.Iterator(termsEnum);
IDictionary<BytesRef, TopDocs> answers = new Dictionary<BytesRef, TopDocs>();
while (termsEnum.Next() != null)
{
if (Random().NextDouble() <= chance)
{
BytesRef term = BytesRef.DeepCopyOf(termsEnum.Term());
answers[term] = s.Search(new TermQuery(new Term("body", term)), 100);
}
}
if (answers.Count > 0)
{
CountdownEvent startingGun = new CountdownEvent(1);
int numThreads = TestUtil.NextInt(Random(), 2, 5);
ThreadClass[] threads = new ThreadClass[numThreads];
for (int threadID = 0; threadID < numThreads; threadID++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, s, answers, startingGun);
threads[threadID] = thread;
thread.Start();
}
startingGun.Signal();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestCommitThreadSafety()
{
const int NUM_THREADS = 5;
const double RUN_SEC = 0.5;
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
TestUtil.ReduceOpenFiles(w.w);
w.Commit();
AtomicBoolean failed = new AtomicBoolean();
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
long endTime = DateTime.Now.Millisecond + ((long)(RUN_SEC * 1000));
for (int i = 0; i < NUM_THREADS; i++)
{
int finalI = i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, dir, w, failed, endTime, finalI);
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
Assert.IsFalse(failed.Get());
w.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir);
long startTime = DateTime.Now.Millisecond;
// TODO: replace w/ the @nightly test data; make this
// into an optional @nightly stress test
Document doc = new Document();
Field body = NewTextField("body", "", Field.Store.NO);
doc.Add(body);
StringBuilder sb = new StringBuilder();
for (int docCount = 0; docCount < NUM_DOCS; docCount++)
{
int numTerms = Random().Next(10);
for (int termCount = 0; termCount < numTerms; termCount++)
{
sb.Append(Random().NextBoolean() ? "aaa" : "bbb");
sb.Append(' ');
}
body.StringValue = sb.ToString();
w.AddDocument(doc);
sb.Remove(0, sb.Length);
}
IndexReader r = w.Reader;
w.Dispose();
long endTime = DateTime.Now.Millisecond;
if (VERBOSE)
{
Console.WriteLine("BUILD took " + (endTime - startTime));
}
IndexSearcher s = NewSearcher(r);
AtomicBoolean failed = new AtomicBoolean();
AtomicLong netSearch = new AtomicLong();
ThreadClass[] threads = new ThreadClass[NUM_SEARCH_THREADS];
for (int threadID = 0; threadID < NUM_SEARCH_THREADS; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, s, failed, netSearch);
threads[threadID].SetDaemon(true);
}
foreach (ThreadClass t in threads)
{
t.Start();
}
foreach (ThreadClass t in threads)
{
t.Join();
}
if (VERBOSE)
{
Console.WriteLine(NUM_SEARCH_THREADS + " threads did " + netSearch.Get() + " searches");
}
r.Dispose();
dir.Dispose();
}
// [Test] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass
// run random tests from different threads to make sure the per-thread clones
// don't share mutable data
public virtual void TestClone()
{
RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
int numDocs = AtLeast(100);
foreach (Options options in ValidOptions())
{
RandomDocument[] docs = new RandomDocument[numDocs];
for (int i = 0; i < numDocs; ++i)
{
docs[i] = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), AtLeast(10), options);
}
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
for (int i = 0; i < numDocs; ++i)
{
writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
}
IndexReader reader = writer.Reader;
for (int i = 0; i < numDocs; ++i)
{
int docID = DocID(reader, "" + i);
AssertEquals(docs[i], reader.GetTermVectors(docID));
}
AtomicObject<Exception> exception = new AtomicObject<Exception>();
ThreadClass[] threads = new ThreadClass[2];
for (int i = 0; i < threads.Length; ++i)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, docs, reader, exception, i);
}
foreach (ThreadClass thread in threads)
{
thread.Start();
}
foreach (ThreadClass thread in threads)
{
thread.Join();
}
reader.Dispose();
writer.Dispose();
dir.Dispose();
Assert.IsNull(exception.Value, "One thread threw an exception");
}
}
public virtual void TestConcurrency()
{
int ncats = AtLeast(100000); // add many categories
int range = ncats * 3; // affects the categories selection
AtomicInteger numCats = new AtomicInteger(ncats);
Directory dir = NewDirectory();
var values = new ConcurrentDictionary <string, string>();
double d = Random().NextDouble();
TaxonomyWriterCache cache;
if (d < 0.7)
{
// this is the fastest, yet most memory consuming
cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
}
else if (TEST_NIGHTLY && d > 0.98)
{
// this is the slowest, but tests the writer concurrency when no caching is done.
// only pick it during NIGHTLY tests, and even then, with very low chances.
cache = NO_OP_CACHE;
}
else
{
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
if (VERBOSE)
{
Console.WriteLine("TEST: use cache=" + cache);
}
var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
ThreadClass[] addThreads = new ThreadClass[AtLeast(4)];
for (int z = 0; z < addThreads.Length; z++)
{
addThreads[z] = new ThreadAnonymousInnerClassHelper(this, range, numCats, values, tw);
}
foreach (var t in addThreads)
{
t.Start();
}
foreach (var t in addThreads)
{
t.Join();
}
tw.Dispose();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
// +1 for root category
if (values.Count + 1 != dtr.Size)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (dtr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
Fail("mismatch number of categories");
}
int[] parents = dtr.ParallelTaxonomyArrays.Parents();
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.True(dtr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = new FacetLabel();
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = dtr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Close(dtr, dir);
}
public virtual void AssertThreadSafe(Analyzer analyzer)
{
int numTestPoints = 100;
int numThreads = TestUtil.NextInt(Random(), 3, 5);
Dictionary<string, BytesRef> map = new Dictionary<string, BytesRef>();
// create a map<String,SortKey> up front.
// then with multiple threads, generate sort keys for all the keys in the map
// and ensure they are the same as the ones we produced in serial fashion.
for (int i = 0; i < numTestPoints; i++)
{
string term = TestUtil.RandomSimpleString(Random());
IOException priorException = null;
TokenStream ts = analyzer.TokenStream("fake", new StreamReader(term));
try
{
ITermToBytesRefAttribute termAtt = ts.AddAttribute<ITermToBytesRefAttribute>();
BytesRef bytes = termAtt.BytesRef;
ts.Reset();
Assert.IsTrue(ts.IncrementToken());
termAtt.FillBytesRef();
// ensure we make a copy of the actual bytes too
map[term] = BytesRef.DeepCopyOf(bytes);
Assert.IsFalse(ts.IncrementToken());
ts.End();
}
catch (IOException e)
{
priorException = e;
}
finally
{
IOUtils.CloseWhileHandlingException(priorException, ts);
}
}
ThreadClass[] threads = new ThreadClass[numThreads];
for (int i = 0; i < numThreads; i++)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, analyzer, map);
}
for (int i = 0; i < numThreads; i++)
{
threads[i].Start();
}
for (int i = 0; i < numThreads; i++)
{
threads[i].Join();
}
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"));
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1))
{
// Otherwise test can take way too long (> 2 hours)
numTerms /= 2;
}
if (VERBOSE)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList = CollectionsHelper.Shuffle(postingsList);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir("bagofpostings"));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int threadCount = TestUtil.NextInt(Random(), 1, 5);
if (VERBOSE)
{
Console.WriteLine("config: " + iw.w.Config);
Console.WriteLine("threadCount=" + threadCount);
}
ThreadClass[] threads = new ThreadClass[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, maxTermsPerDoc, postings, iw, startingGun);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.Reader;
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.Terms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, air.Fields.UniqueTermCount);
if (iwc.Codec is Lucene3xCodec == false)
{
Assert.AreEqual(numTerms - 1, terms.Size());
}
TermsEnum termsEnum = terms.Iterator(null);
BytesRef term_;
while ((term_ = termsEnum.Next()) != null)
{
int value = Convert.ToInt32(term_.Utf8ToString());
Assert.AreEqual(value, termsEnum.DocFreq());
// don't really need to check more than this, as CheckIndex
// will verify that docFreq == actual number of documents seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir, analyzer);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
int charsToIndex = AtLeast(100000);
int charsIndexed = 0;
//System.out.println("bytesToIndex=" + charsToIndex);
while (charsIndexed < charsToIndex)
{
Document doc = docs.NextDoc();
charsIndexed += doc.Get("body").Length;
w.AddDocument(doc);
//System.out.println(" bytes=" + charsIndexed + " add: " + doc);
}
IndexReader r = w.GetReader();
//System.out.println("numDocs=" + r.NumDocs);
w.Dispose();
IndexSearcher s = NewSearcher(r);
Terms terms = MultiFields.GetFields(r).GetTerms("body");
int termCount = 0;
TermsEnum termsEnum = terms.GetIterator(null);
while (termsEnum.Next() != null)
{
termCount++;
}
Assert.IsTrue(termCount > 0);
// Target ~10 terms to search:
double chance = 10.0 / termCount;
termsEnum = terms.GetIterator(termsEnum);
IDictionary <BytesRef, TopDocs> answers = new Dictionary <BytesRef, TopDocs>();
while (termsEnum.Next() != null)
{
if (Random.NextDouble() <= chance)
{
BytesRef term = BytesRef.DeepCopyOf(termsEnum.Term);
answers[term] = s.Search(new TermQuery(new Term("body", term)), 100);
}
}
if (answers.Count > 0)
{
CountdownEvent startingGun = new CountdownEvent(1);
int numThreads = TestUtil.NextInt32(Random, 2, 5);
ThreadJob[] threads = new ThreadJob[numThreads];
for (int threadID = 0; threadID < numThreads; threadID++)
{
ThreadJob thread = new ThreadAnonymousInnerClassHelper(this, s, answers, startingGun);
threads[threadID] = thread;
thread.Start();
}
startingGun.Signal();
foreach (ThreadJob thread in threads)
{
thread.Join();
}
}
r.Dispose();
dir.Dispose();
}
private void AssertConsistentYoungestChild(FacetLabel abPath, int abOrd, int abYoungChildBase1, int abYoungChildBase2, int retry, int numCategories)
{
var indexDir = new SlowRAMDirectory(-1, null); // no slowness for intialization
var tw = new DirectoryTaxonomyWriter(indexDir);
tw.AddCategory(new FacetLabel("a", "0"));
tw.AddCategory(abPath);
tw.Commit();
var tr = new DirectoryTaxonomyReader(indexDir);
for (int i = 0; i < numCategories; i++)
{
var cp = new FacetLabel("a", "b", Convert.ToString(i));
tw.AddCategory(cp);
Assert.AreEqual(TaxonomyReader.INVALID_ORDINAL, tr.GetOrdinal(cp), "Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed");
}
tw.Dispose();
var stop = new AtomicBoolean(false);
Exception[] error = new Exception[] { null };
int[] retrieval = new int[] { 0 };
var thread = new ThreadAnonymousInnerClassHelper(this, abPath, abOrd, abYoungChildBase1, abYoungChildBase2, retry, tr, stop, error, retrieval);
thread.Start();
indexDir.SleepMillis = 1; // some delay for refresh
var newTaxoReader = TaxonomyReader.OpenIfChanged(tr);
if (newTaxoReader != null)
{
newTaxoReader.Dispose();
}
stop.Set(true);
thread.Join();
Assert.Null(error[0], "Unexpcted exception at retry " + retry + " retrieval " + retrieval[0] + ": \n" + stackTraceStr(error[0]));
tr.Dispose();
}
public virtual void TestNoWaitClose()
{
Directory directory = NewDirectory();
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.IsTokenized = false;
Field idField = NewField("id", "", customType);
doc.Add(idField);
for (int pass = 0; pass < 2; pass++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: pass="******"TEST: iter=" + iter);
}
for (int j = 0; j < 199; j++)
{
idField.SetStringValue(Convert.ToString(iter * 201 + j));
writer.AddDocument(doc);
}
int delID = iter * 199;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(new Term("id", Convert.ToString(delID)));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 2;
IndexWriter finalWriter = writer;
List <Exception> failure = new List <Exception>();
ThreadJob t1 = new ThreadAnonymousInnerClassHelper(this, doc, finalWriter, failure);
if (failure.Count > 0)
{
throw failure[0];
}
t1.Start();
writer.Dispose(false);
t1.Join();
// Make sure reader can read
IndexReader reader = DirectoryReader.Open(directory);
reader.Dispose();
// Reopen
writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetMergePolicy(NewLogMergePolicy()));
}
writer.Dispose();
}
directory.Dispose();
}
public virtual void TestRollbackAndCommitWithThreads()
{
BaseDirectoryWrapper d = NewDirectory();
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).PreventDoubleWrite = false;
}
int threadCount = TestUtil.NextInt(Random(), 2, 6);
AtomicReference<IndexWriter> writerRef = new AtomicReference<IndexWriter>();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
writerRef.Value = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
LineFileDocs docs = new LineFileDocs(Random());
ThreadClass[] threads = new ThreadClass[threadCount];
int iters = AtLeast(100);
AtomicBoolean failed = new AtomicBoolean();
ReentrantLock rollbackLock = new ReentrantLock();
ReentrantLock commitLock = new ReentrantLock();
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, d, writerRef, docs, iters, failed, rollbackLock, commitLock);
threads[threadID].Start();
}
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID].Join();
}
Assert.IsTrue(!failed.Get());
writerRef.Value.Dispose();
d.Dispose();
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt32(Random, 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"), StringComparison.Ordinal);
IndexWriterConfig iwc = NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, new MockAnalyzer(Random));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TestNightly || RandomMultiplier > 1))
{
// Otherwise test can take way too long (> 2 hours)
//numTerms /= 2;
// LUCENENET specific - To keep this under the 1 hour free limit
// of Azure DevOps, this was reduced from /2 to /6.
numTerms /= 6;
}
if (Verbose)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i, CultureInfo.InvariantCulture);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList.Shuffle(Random);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir(GetFullMethodName()));
RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwc);
int threadCount = TestUtil.NextInt32(Random, 1, 5);
if (Verbose)
{
Console.WriteLine("config: " + iw.IndexWriter.Config);
Console.WriteLine("threadCount=" + threadCount);
}
Field prototype = NewTextField("field", "", Field.Store.NO);
FieldType fieldType = new FieldType(prototype.FieldType);
if (Random.NextBoolean())
{
fieldType.OmitNorms = true;
}
int options = Random.Next(3);
if (options == 0)
{
fieldType.IndexOptions = IndexOptions.DOCS_AND_FREQS; // we dont actually need positions
fieldType.StoreTermVectors = true; // but enforce term vectors when we do this so we check SOMETHING
}
else if (options == 1 && !DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat("field")))
{
fieldType.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
// else just positions
ThreadJob[] threads = new ThreadJob[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
Random threadRandom = new Random(Random.Next());
Document document = new Document();
Field field = new Field("field", "", fieldType);
document.Add(field);
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, numTerms, maxTermsPerDoc, postings, iw, startingGun, threadRandom, document, field);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadJob t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.GetReader();
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.GetTerms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, terms.Count);
TermsEnum termsEnum = terms.GetEnumerator();
while (termsEnum.MoveNext())
{
int value = Convert.ToInt32(termsEnum.Term.Utf8ToString(), CultureInfo.InvariantCulture);
Assert.AreEqual(value, termsEnum.TotalTermFreq);
// don't really need to check more than this, as CheckIndex
// will verify that totalTermFreq == total number of positions seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
private void RunTest(Random random, Directory dir)
{
// Run for ~1 seconds
long stopTime = Environment.TickCount + 1000;
SnapshotDeletionPolicy dp = DeletionPolicy;
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetIndexDeletionPolicy(dp).SetMaxBufferedDocs(2));
// Verify we catch misuse:
try
{
dp.Snapshot();
Assert.Fail("did not hit exception");
}
catch (InvalidOperationException ise)
{
// expected
}
dp = (SnapshotDeletionPolicy)writer.Config.DelPolicy;
writer.Commit();
ThreadClass t = new ThreadAnonymousInnerClassHelper(stopTime, writer);
t.Start();
// While the above indexing thread is running, take many
// backups:
do
{
BackupIndex(dir, dp);
Thread.Sleep(20);
} while (t.IsAlive);
t.Join();
// Add one more document to force writer to commit a
// final segment, so deletion policy has a chance to
// delete again:
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("content", "aaa", customType));
writer.AddDocument(doc);
// Make sure we don't have any leftover files in the
// directory:
writer.Dispose();
TestIndexWriter.AssertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
}
public virtual void TestHashCodeWithThreads()
{
AutomatonQuery[] queries = new AutomatonQuery[1000];
for (int i = 0; i < queries.Length; i++)
{
queries[i] = new AutomatonQuery(new Term("bogus", "bogus"), AutomatonTestUtil.RandomAutomaton(Random()));
}
CountDownLatch startingGun = new CountDownLatch(1);
int numThreads = TestUtil.NextInt(Random(), 2, 5);
ThreadClass[] threads = new ThreadClass[numThreads];
for (int threadID = 0; threadID < numThreads; threadID++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, queries, startingGun);
threads[threadID] = thread;
thread.Start();
}
startingGun.countDown();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
}
public virtual void TestIndexing()
{
DirectoryInfo tmpDir = CreateTempDir("TestNeverDelete");
BaseDirectoryWrapper d = NewFSDirectory(tmpDir);
// We want to "see" files removed if Lucene removed
// them. this is still worth running on Windows since
// some files the IR opens and closes.
if (d is MockDirectoryWrapper)
{
((MockDirectoryWrapper)d).NoDeleteOpenFile = false;
}
RandomIndexWriter w = new RandomIndexWriter(Random(), d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
w.w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 5, 30));
w.Commit();
ThreadClass[] indexThreads = new ThreadClass[Random().Next(4)];
long stopTime = Environment.TickCount + AtLeast(1000);
for (int x = 0; x < indexThreads.Length; x++)
{
indexThreads[x] = new ThreadAnonymousInnerClassHelper(w, stopTime, NewStringField, NewTextField);
indexThreads[x].Name = "Thread " + x;
indexThreads[x].Start();
}
HashSet <string> allFiles = new HashSet <string>();
DirectoryReader r = DirectoryReader.Open(d);
while (Environment.TickCount < stopTime)
{
IndexCommit ic = r.IndexCommit;
if (VERBOSE)
{
Console.WriteLine("TEST: check files: " + ic.FileNames);
}
allFiles.AddAll(ic.FileNames);
// Make sure no old files were removed
foreach (string fileName in allFiles)
{
Assert.IsTrue(SlowFileExists(d, fileName), "file " + fileName + " does not exist");
}
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Thread.Sleep(1);
}
r.Dispose();
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
w.Dispose();
d.Dispose();
System.IO.Directory.Delete(tmpDir.FullName, true);
}
public virtual void TestStressMultiThreading()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir, conf);
// create index
int numThreads = TestUtil.NextInt(Random(), 3, 6);
int numDocs = AtLeast(2000);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new StringField("id", "doc" + i, Store.NO));
double group = Random().NextDouble();
string g;
if (group < 0.1)
{
g = "g0";
}
else if (group < 0.5)
{
g = "g1";
}
else if (group < 0.8)
{
g = "g2";
}
else
{
g = "g3";
}
doc.Add(new StringField("updKey", g, Store.NO));
for (int j = 0; j < numThreads; j++)
{
long value = Random().Next();
doc.Add(new NumericDocValuesField("f" + j, value));
doc.Add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
}
writer.AddDocument(doc);
}
CountDownLatch done = new CountDownLatch(numThreads);
AtomicInteger numUpdates = new AtomicInteger(AtLeast(100));
// same thread updates a field as well as reopens
ThreadClass[] threads = new ThreadClass[numThreads];
for (int i = 0; i < threads.Length; i++)
{
string f = "f" + i;
string cf = "cf" + i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, "UpdateThread-" + i, writer, numDocs, done, numUpdates, f, cf);
}
foreach (ThreadClass t in threads)
{
t.Start();
}
done.@await();
writer.Dispose();
DirectoryReader reader = DirectoryReader.Open(dir);
foreach (AtomicReaderContext context in reader.Leaves)
{
AtomicReader r = context.AtomicReader;
for (int i = 0; i < numThreads; i++)
{
NumericDocValues ndv = r.GetNumericDocValues("f" + i);
NumericDocValues control = r.GetNumericDocValues("cf" + i);
Bits docsWithNdv = r.GetDocsWithField("f" + i);
Bits docsWithControl = r.GetDocsWithField("cf" + i);
Bits liveDocs = r.LiveDocs;
for (int j = 0; j < r.MaxDoc; j++)
{
if (liveDocs == null || liveDocs.Get(j))
{
Assert.AreEqual(docsWithNdv.Get(j), docsWithControl.Get(j));
if (docsWithNdv.Get(j))
{
Assert.AreEqual(control.Get(j), ndv.Get(j) * 2);
}
}
}
}
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestConcurrency()
{
int ncats = AtLeast(100000); // add many categories
int range = ncats * 3; // affects the categories selection
AtomicInteger numCats = new AtomicInteger(ncats);
Directory dir = NewDirectory();
var values = new ConcurrentDictionary<string, string>();
double d = Random().NextDouble();
ITaxonomyWriterCache cache;
if (d < 0.7)
{
// this is the fastest, yet most memory consuming
cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
}
else if (TEST_NIGHTLY && d > 0.98)
{
// this is the slowest, but tests the writer concurrency when no caching is done.
// only pick it during NIGHTLY tests, and even then, with very low chances.
cache = NO_OP_CACHE;
}
else
{
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
if (VERBOSE)
{
Console.WriteLine("TEST: use cache=" + cache);
}
var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
ThreadClass[] addThreads = new ThreadClass[AtLeast(4)];
for (int z = 0; z < addThreads.Length; z++)
{
addThreads[z] = new ThreadAnonymousInnerClassHelper(this, range, numCats, values, tw);
}
foreach (var t in addThreads)
{
t.Start();
}
foreach (var t in addThreads)
{
t.Join();
}
tw.Dispose();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
// +1 for root category
if (values.Count + 1 != dtr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (dtr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
Fail("mismatch number of categories");
}
int[] parents = dtr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.True(dtr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = new FacetLabel();
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = dtr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Close(dtr, dir);
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt32(Random, 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"), StringComparison.Ordinal);
IndexWriterConfig iwc = NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, new MockAnalyzer(Random));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TestNightly || RandomMultiplier > 1))
{
// Otherwise test can take way too long (> 2 hours)
//numTerms /= 2;
// LUCENENET specific - To keep this under the 1 hour free limit
// of Azure DevOps, this was reduced from /2 to /6.
numTerms /= 6;
}
if (Verbose)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i, CultureInfo.InvariantCulture);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList.Shuffle(Random);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir("bagofpostings"));
RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwc);
int threadCount = TestUtil.NextInt32(Random, 1, 5);
if (Verbose)
{
Console.WriteLine("config: " + iw.IndexWriter.Config);
Console.WriteLine("threadCount=" + threadCount);
}
ThreadJob[] threads = new ThreadJob[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, maxTermsPerDoc, postings, iw, startingGun);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadJob t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.GetReader();
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.GetTerms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
#pragma warning disable 612, 618
Assert.AreEqual(numTerms - 1, air.Fields.UniqueTermCount);
if (iwc.Codec is Lucene3xCodec == false)
#pragma warning restore 612, 618
{
Assert.AreEqual(numTerms - 1, terms.Count);
}
TermsEnum termsEnum = terms.GetEnumerator();
while (termsEnum.MoveNext())
{
int value = Convert.ToInt32(termsEnum.Term.Utf8ToString(), CultureInfo.InvariantCulture);
Assert.AreEqual(value, termsEnum.DocFreq);
// don't really need to check more than this, as CheckIndex
// will verify that docFreq == actual number of documents seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestDuringAddIndexes()
{
Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(2)));
// create the index
CreateIndexNoClose(false, "test", writer);
writer.Commit();
Directory[] dirs = new Directory[10];
for (int i = 0; i < 10; i++)
{
dirs[i] = new MockDirectoryWrapper(Random(), new RAMDirectory(dir1, NewIOContext(Random())));
}
DirectoryReader r = writer.Reader;
const float SECONDS = 0.5f;
long endTime = (long)(DateTime.Now.Millisecond + 1000.0 * SECONDS);
IList<Exception> excs = new ConcurrentList<Exception>(new List<Exception>());
// Only one thread can addIndexes at a time, because
// IndexWriter acquires a write lock in each directory:
ThreadClass[] threads = new ThreadClass[1];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new ThreadAnonymousInnerClassHelper(this, writer, dirs, endTime, excs);
threads[i].SetDaemon(true);
threads[i].Start();
}
int lastCount = 0;
while (DateTime.Now.Millisecond < endTime)
{
DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
if (r2 != null)
{
r.Dispose();
r = r2;
}
Query q = new TermQuery(new Term("indexname", "test"));
IndexSearcher searcher = NewSearcher(r);
int count = searcher.Search(q, 10).TotalHits;
Assert.IsTrue(count >= lastCount);
lastCount = count;
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
// final check
DirectoryReader dr2 = DirectoryReader.OpenIfChanged(r);
if (dr2 != null)
{
r.Dispose();
r = dr2;
}
Query q2 = new TermQuery(new Term("indexname", "test"));
IndexSearcher searcher_ = NewSearcher(r);
int count_ = searcher_.Search(q2, 10).TotalHits;
Assert.IsTrue(count_ >= lastCount);
Assert.AreEqual(0, excs.Count);
r.Dispose();
if (dir1 is MockDirectoryWrapper)
{
ICollection<string> openDeletedFiles = ((MockDirectoryWrapper)dir1).OpenDeletedFiles;
Assert.AreEqual(0, openDeletedFiles.Count, "openDeleted=" + openDeletedFiles);
}
writer.Dispose();
dir1.Dispose();
}
public virtual void TestStressMultiThreading()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
IndexWriter writer = new IndexWriter(dir, conf);
// create index
int numThreads = TestUtil.NextInt32(Random, 3, 6);
int numDocs = AtLeast(2000);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new StringField("id", "doc" + i, Store.NO));
double group = Random.NextDouble();
string g;
if (group < 0.1)
{
g = "g0";
}
else if (group < 0.5)
{
g = "g1";
}
else if (group < 0.8)
{
g = "g2";
}
else
{
g = "g3";
}
doc.Add(new StringField("updKey", g, Store.NO));
for (int j = 0; j < numThreads; j++)
{
long value = Random.Next();
doc.Add(new BinaryDocValuesField("f" + j, TestBinaryDocValuesUpdates.ToBytes(value)));
doc.Add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
}
writer.AddDocument(doc);
}
CountdownEvent done = new CountdownEvent(numThreads);
AtomicInt32 numUpdates = new AtomicInt32(AtLeast(100));
// same thread updates a field as well as reopens
ThreadJob[] threads = new ThreadJob[numThreads];
for (int i = 0; i < threads.Length; i++)
{
string f = "f" + i;
string cf = "cf" + i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, "UpdateThread-" + i, writer, numDocs, done, numUpdates, f, cf);
}
foreach (ThreadJob t in threads)
{
t.Start();
}
done.Wait();
writer.Dispose();
DirectoryReader reader = DirectoryReader.Open(dir);
BytesRef scratch = new BytesRef();
foreach (AtomicReaderContext context in reader.Leaves)
{
AtomicReader r = context.AtomicReader;
for (int i = 0; i < numThreads; i++)
{
BinaryDocValues bdv = r.GetBinaryDocValues("f" + i);
NumericDocValues control = r.GetNumericDocValues("cf" + i);
IBits docsWithBdv = r.GetDocsWithField("f" + i);
IBits docsWithControl = r.GetDocsWithField("cf" + i);
IBits liveDocs = r.LiveDocs;
for (int j = 0; j < r.MaxDoc; j++)
{
if (liveDocs == null || liveDocs.Get(j))
{
Assert.AreEqual(docsWithBdv.Get(j), docsWithControl.Get(j));
if (docsWithBdv.Get(j))
{
long ctrlValue = control.Get(j);
long bdvValue = TestBinaryDocValuesUpdates.GetValue(bdv, j, scratch) * 2;
// if (ctrlValue != bdvValue) {
// System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.Document(j).Get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
// }
Assert.AreEqual(ctrlValue, bdvValue);
}
}
}
}
}
reader.Dispose();
dir.Dispose();
}
public virtual void Test()
{
IList<string> postingsList = new List<string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"));
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1))
{
// Otherwise test can take way too long (> 2 hours)
numTerms /= 2;
}
if (VERBOSE)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList = CollectionsHelper.Shuffle(postingsList);
ConcurrentQueue<string> postings = new ConcurrentQueue<string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir(GetFullMethodName()));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int threadCount = TestUtil.NextInt(Random(), 1, 5);
if (VERBOSE)
{
Console.WriteLine("config: " + iw.w.Config);
Console.WriteLine("threadCount=" + threadCount);
}
Field prototype = NewTextField("field", "", Field.Store.NO);
FieldType fieldType = new FieldType((FieldType)prototype.FieldType);
if (Random().NextBoolean())
{
fieldType.OmitNorms = true;
}
int options = Random().Next(3);
if (options == 0)
{
fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS; // we dont actually need positions
fieldType.StoreTermVectors = true; // but enforce term vectors when we do this so we check SOMETHING
}
else if (options == 1 && !DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat("field")))
{
fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
// else just positions
ThreadClass[] threads = new ThreadClass[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
Random threadRandom = new Random(Random().Next());
Document document = new Document();
Field field = new Field("field", "", fieldType);
document.Add(field);
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, numTerms, maxTermsPerDoc, postings, iw, startingGun, threadRandom, document, field);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.Reader;
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.Terms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, terms.Size());
TermsEnum termsEnum = terms.Iterator(null);
BytesRef termBR;
while ((termBR = termsEnum.Next()) != null)
{
int value = Convert.ToInt32(termBR.Utf8ToString());
Assert.AreEqual(value, termsEnum.TotalTermFreq());
// don't really need to check more than this, as CheckIndex
// will verify that totalTermFreq == total number of positions seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
public virtual void TestNrt()
{
Store.Directory dir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
// Don't allow tiny maxBufferedDocs; it can make this
// test too slow:
iwc.SetMaxBufferedDocs(Math.Max(500, iwc.MaxBufferedDocs));
// MockRandom/AlcololicMergePolicy are too slow:
TieredMergePolicy tmp = new TieredMergePolicy();
tmp.FloorSegmentMB = .001;
iwc.SetMergePolicy(tmp);
IndexWriter w = new IndexWriter(dir, iwc);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
config.SetMultiValued("field", true);
AtomicBoolean stop = new AtomicBoolean();
// How many unique facets to index before stopping:
int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
var indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
var mgr = new SearcherTaxonomyManager(w, true, null, tw);
var reopener = new ThreadAnonymousInnerClassHelper(this, stop, mgr);
reopener.Name = "reopener";
reopener.Start();
indexer.Name = "indexer";
indexer.Start();
try
{
while (!stop.Get())
{
SearcherAndTaxonomy pair = mgr.Acquire();
try
{
//System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
FacetsCollector sfc = new FacetsCollector();
pair.searcher.Search(new MatchAllDocsQuery(), sfc);
Facets facets = GetTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
FacetResult result = facets.GetTopChildren(10, "field");
if (pair.searcher.IndexReader.NumDocs > 0)
{
//System.out.println(pair.taxonomyReader.getSize());
Assert.True(result.ChildCount > 0);
Assert.True(result.LabelValues.Length > 0);
}
//if (VERBOSE) {
//System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
//}
}
finally
{
mgr.Release(pair);
}
}
}
finally
{
indexer.Join();
reopener.Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: now stop");
}
IOUtils.Close(mgr, tw, w, taxoDir, dir);
}
public virtual void TestDeleteAllNoDeadLock()
{
Directory dir = NewDirectory();
RandomIndexWriter modifier = new RandomIndexWriter(Random(), dir);
int numThreads = AtLeast(2);
ThreadClass[] threads = new ThreadClass[numThreads];
CountDownLatch latch = new CountDownLatch(1);
CountDownLatch doneLatch = new CountDownLatch(numThreads);
for (int i = 0; i < numThreads; i++)
{
int offset = i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, modifier, latch, doneLatch, offset);
threads[i].Start();
}
latch.countDown();
//Wait for 1 millisecond
while (!doneLatch.@await(new TimeSpan(0, 0, 0, 0, 1)))
{
modifier.DeleteAll();
if (VERBOSE)
{
Console.WriteLine("del all");
}
}
modifier.DeleteAll();
foreach (ThreadClass thread in threads)
{
thread.Join();
}
modifier.Dispose();
DirectoryReader reader = DirectoryReader.Open(dir);
Assert.AreEqual(reader.MaxDoc, 0);
Assert.AreEqual(reader.NumDocs, 0);
Assert.AreEqual(reader.NumDeletedDocs, 0);
reader.Dispose();
dir.Dispose();
}
