|
public SetMergedSegmentWarmer ( IndexReaderWarmer mergeSegmentWarmer ) : |
||
| mergeSegmentWarmer | IndexReaderWarmer | |
| return | ||
/// <summary>
/// create a new index writer config with random defaults using the specified random </summary>
public static IndexWriterConfig NewIndexWriterConfig(Random r, LuceneVersion v, Analyzer a)
{
IndexWriterConfig c = new IndexWriterConfig(v, a);
c.SetSimilarity(ClassEnvRule.Similarity);
if (VERBOSE)
{
// Even though TestRuleSetupAndRestoreClassEnv calls
// InfoStream.setDefault, we do it again here so that
// the PrintStreamInfoStream.messageID increments so
// that when there are separate instances of
// IndexWriter created we see "IW 0", "IW 1", "IW 2",
// ... instead of just always "IW 0":
c.InfoStream = new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(Console.Out);
}
if (r.NextBoolean())
{
c.SetMergeScheduler(new SerialMergeScheduler());
}
else if (Rarely(r))
{
int maxThreadCount = TestUtil.NextInt(Random(), 1, 4);
int maxMergeCount = TestUtil.NextInt(Random(), maxThreadCount, maxThreadCount + 4);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
cms.SetMaxMergesAndThreads(maxMergeCount, maxThreadCount);
c.SetMergeScheduler(cms);
}
if (r.NextBoolean())
{
if (Rarely(r))
{
// crazy value
c.SetMaxBufferedDocs(TestUtil.NextInt(r, 2, 15));
}
else
{
// reasonable value
c.SetMaxBufferedDocs(TestUtil.NextInt(r, 16, 1000));
}
}
if (r.NextBoolean())
{
if (Rarely(r))
{
// crazy value
c.SetTermIndexInterval(r.NextBoolean() ? TestUtil.NextInt(r, 1, 31) : TestUtil.NextInt(r, 129, 1000));
}
else
{
// reasonable value
c.SetTermIndexInterval(TestUtil.NextInt(r, 32, 128));
}
}
if (r.NextBoolean())
{
int maxNumThreadStates = Rarely(r) ? TestUtil.NextInt(r, 5, 20) : TestUtil.NextInt(r, 1, 4); // reasonable value - crazy value
if (Rarely(r))
{
// Retrieve the package-private setIndexerThreadPool
// method:
MethodInfo setIndexerThreadPoolMethod = typeof(IndexWriterConfig).GetMethod("SetIndexerThreadPool", new Type[] { typeof(DocumentsWriterPerThreadPool) });
//setIndexerThreadPoolMethod.setAccessible(true);
Type clazz = typeof(RandomDocumentsWriterPerThreadPool);
ConstructorInfo ctor = clazz.GetConstructor(new[] { typeof(int), typeof(Random) });
//ctor.Accessible = true;
// random thread pool
setIndexerThreadPoolMethod.Invoke(c, new[] { ctor.Invoke(new object[] { maxNumThreadStates, r }) });
}
else
{
// random thread pool
c.SetMaxThreadStates(maxNumThreadStates);
}
}
c.SetMergePolicy(NewMergePolicy(r));
if (Rarely(r))
{
c.SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.InfoStream));
}
c.SetUseCompoundFile(r.NextBoolean());
c.SetReaderPooling(r.NextBoolean());
c.SetReaderTermsIndexDivisor(TestUtil.NextInt(r, 1, 4));
c.SetCheckIntegrityAtMerge(r.NextBoolean());
return c;
}
// Collections.synchronizedMap(new WeakHashMap<SegmentCoreReaders, bool?>());
public virtual void RunTest(string testName)
{
Failed.Set(false);
AddCount.Set(0);
DelCount.Set(0);
PackCount.Set(0);
DateTime t0 = DateTime.UtcNow;
Random random = new Random(Random().Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues());
DirectoryInfo tempDir = CreateTempDir(testName);
Dir = GetDirectory(NewMockFSDirectory(tempDir)); // some subclasses rely on this being MDW
if (Dir is BaseDirectoryWrapper)
{
((BaseDirectoryWrapper)Dir).CheckIndexOnClose = false; // don't double-checkIndex, we do it ourselves.
}
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetInfoStream(new FailOnNonBulkMergesInfoStream());
if (LuceneTestCase.TEST_NIGHTLY)
{
// newIWConfig makes smallish max seg size, which
// results in tons and tons of segments for this test
// when run nightly:
MergePolicy mp = conf.MergePolicy;
if (mp is TieredMergePolicy)
{
((TieredMergePolicy)mp).MaxMergedSegmentMB = 5000.0;
}
else if (mp is LogByteSizeMergePolicy)
{
((LogByteSizeMergePolicy)mp).MaxMergeMB = 1000.0;
}
else if (mp is LogMergePolicy)
{
((LogMergePolicy)mp).MaxMergeDocs = 100000;
}
}
conf.SetMergedSegmentWarmer(new IndexReaderWarmerAnonymousInnerClassHelper(this));
if (VERBOSE)
{
conf.InfoStream = new PrintStreamInfoStreamAnonymousInnerClassHelper(this, Console.Out);
}
Writer = new IndexWriter(Dir, conf);
TestUtil.ReduceOpenFiles(Writer);
//TaskScheduler es = Random().NextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory(testName));
TaskScheduler es = null;
DoAfterWriter(es);
int NUM_INDEX_THREADS = TestUtil.NextInt(Random(), 2, 4);
int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
ISet <string> delIDs = new ConcurrentHashSet <string>(new HashSet <string>());
ISet <string> delPackIDs = new ConcurrentHashSet <string>(new HashSet <string>());
IList <SubDocs> allSubDocs = new SynchronizedCollection <SubDocs>();
DateTime stopTime = DateTime.UtcNow.AddSeconds(RUN_TIME_SEC);
ThreadClass[] indexThreads = LaunchIndexingThreads(docs, NUM_INDEX_THREADS, stopTime, delIDs, delPackIDs, allSubDocs);
if (VERBOSE)
{
Console.WriteLine("TEST: DONE start " + NUM_INDEX_THREADS + " indexing threads [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
// Let index build up a bit
Thread.Sleep(100);
DoSearching(es, stopTime);
if (VERBOSE)
{
Console.WriteLine("TEST: all searching done [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
for (int thread = 0; thread < indexThreads.Length; thread++)
{
indexThreads[thread].Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: done join indexing threads [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]; addCount=" + AddCount + " delCount=" + DelCount);
}
IndexSearcher s = FinalSearcher;
if (VERBOSE)
{
Console.WriteLine("TEST: finalSearcher=" + s);
}
Assert.IsFalse(Failed.Get());
bool doFail = false;
// Verify: make sure delIDs are in fact deleted:
foreach (string id in delIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("doc id=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " hits; first docID=" + hits.ScoreDocs[0].Doc);
doFail = true;
}
}
// Verify: make sure delPackIDs are in fact deleted:
foreach (string id in delPackIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("packID=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " matches");
doFail = true;
}
}
// Verify: make sure each group of sub-docs are still in docID order:
foreach (SubDocs subDocs in allSubDocs.ToList())
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", subDocs.PackID)), 20);
if (!subDocs.Deleted)
{
// We sort by relevance but the scores should be identical so sort falls back to by docID:
if (hits.TotalHits != subDocs.SubIDs.Count)
{
Console.WriteLine("packID=" + subDocs.PackID + ": expected " + subDocs.SubIDs.Count + " hits but got " + hits.TotalHits);
doFail = true;
}
else
{
int lastDocID = -1;
int startDocID = -1;
foreach (ScoreDoc scoreDoc in hits.ScoreDocs)
{
int docID = scoreDoc.Doc;
if (lastDocID != -1)
{
Assert.AreEqual(1 + lastDocID, docID);
}
else
{
startDocID = docID;
}
lastDocID = docID;
Document doc = s.Doc(docID);
Assert.AreEqual(subDocs.PackID, doc.Get("packID"));
}
lastDocID = startDocID - 1;
foreach (string subID in subDocs.SubIDs)
{
hits = s.Search(new TermQuery(new Term("docid", subID)), 1);
Assert.AreEqual(1, hits.TotalHits);
int docID = hits.ScoreDocs[0].Doc;
if (lastDocID != -1)
{
Assert.AreEqual(1 + lastDocID, docID);
}
lastDocID = docID;
}
}
}
else
{
// Pack was deleted -- make sure its docs are
// deleted. We can't verify packID is deleted
// because we can re-use packID for update:
foreach (string subID in subDocs.SubIDs)
{
Assert.AreEqual(0, s.Search(new TermQuery(new Term("docid", subID)), 1).TotalHits);
}
}
}
// Verify: make sure all not-deleted docs are in fact
// not deleted:
int endID = Convert.ToInt32(docs.NextDoc().Get("docid"));
docs.Dispose();
for (int id = 0; id < endID; id++)
{
string stringID = "" + id;
if (!delIDs.Contains(stringID))
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", stringID)), 1);
if (hits.TotalHits != 1)
{
Console.WriteLine("doc id=" + stringID + " is not supposed to be deleted, but got hitCount=" + hits.TotalHits + "; delIDs=" + delIDs);
doFail = true;
}
}
}
Assert.IsFalse(doFail);
Assert.AreEqual(AddCount.Get() - DelCount.Get(), s.IndexReader.NumDocs, "index=" + Writer.SegString() + " addCount=" + AddCount + " delCount=" + DelCount);
ReleaseSearcher(s);
Writer.Commit();
Assert.AreEqual(AddCount.Get() - DelCount.Get(), Writer.NumDocs(), "index=" + Writer.SegString() + " addCount=" + AddCount + " delCount=" + DelCount);
DoClose();
Writer.Dispose(false);
// Cannot shutdown until after writer is closed because
// writer has merged segment warmer that uses IS to run
// searches, and that IS may be using this es!
/*if (es != null)
* {
* es.shutdown();
* es.awaitTermination(1, TimeUnit.SECONDS);
* }*/
TestUtil.CheckIndex(Dir);
Dir.Dispose();
System.IO.Directory.Delete(tempDir.FullName, true);
if (VERBOSE)
{
Console.WriteLine("TEST: done [" + (DateTime.UtcNow - t0).TotalMilliseconds + " ms]");
}
}
public virtual void RunTest(string testName)
{
m_failed.Value = (false);
m_addCount.Value = 0;
m_delCount.Value = 0;
m_packCount.Value = 0;
long t0 = J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond; // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
Random random = new J2N.Randomizer(Random.NextInt64());
using LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues);
DirectoryInfo tempDir = CreateTempDir(testName);
m_dir = GetDirectory(NewMockFSDirectory(tempDir)); // some subclasses rely on this being MDW
if (m_dir is BaseDirectoryWrapper baseDirectoryWrapper)
{
baseDirectoryWrapper.CheckIndexOnDispose = false; // don't double-checkIndex, we do it ourselves.
}
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetInfoStream(new FailOnNonBulkMergesInfoStream());
if (LuceneTestCase.TestNightly)
{
// newIWConfig makes smallish max seg size, which
// results in tons and tons of segments for this test
// when run nightly:
MergePolicy mp = conf.MergePolicy;
if (mp is TieredMergePolicy tieredMergePolicy)
{
//tieredMergePolicy.MaxMergedSegmentMB = 5000.0;
tieredMergePolicy.MaxMergedSegmentMB = 2500.0; // LUCENENET specific - reduced each number by 50% to keep testing time under 1 hour
}
else if (mp is LogByteSizeMergePolicy logByteSizeMergePolicy)
{
//logByteSizeMergePolicy.MaxMergeMB = 1000.0;
logByteSizeMergePolicy.MaxMergeMB = 500.0; // LUCENENET specific - reduced each number by 50% to keep testing time under 1 hour
}
else if (mp is LogMergePolicy logMergePolicy)
{
//logMergePolicy.MaxMergeDocs = 100000;
logMergePolicy.MaxMergeDocs = 50000; // LUCENENET specific - reduced each number by 50% to keep testing time under 1 hour
}
}
conf.SetMergedSegmentWarmer(new IndexReaderWarmerAnonymousClass(this));
if (Verbose)
{
conf.SetInfoStream(new PrintStreamInfoStreamAnonymousClass(Console.Out));
}
m_writer = new IndexWriter(m_dir, conf);
TestUtil.ReduceOpenFiles(m_writer);
TaskScheduler es = LuceneTestCase.Random.NextBoolean() ? null : TaskScheduler.Default;
DoAfterWriter(es);
int NUM_INDEX_THREADS = TestUtil.NextInt32(LuceneTestCase.Random, 2, 4);
//int RUN_TIME_SEC = LuceneTestCase.TestNightly ? 300 : RandomMultiplier;
// LUCENENET specific - lowered from 300 to 150 to reduce total time on Nightly
// build to less than 1 hour.
int RUN_TIME_SEC = LuceneTestCase.TestNightly ? 150 : RandomMultiplier;
ISet <string> delIDs = new ConcurrentHashSet <string>();
ISet <string> delPackIDs = new ConcurrentHashSet <string>();
ConcurrentQueue <SubDocs> allSubDocs = new ConcurrentQueue <SubDocs>();
long stopTime = (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) + (RUN_TIME_SEC * 1000); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
ThreadJob[] indexThreads = LaunchIndexingThreads(docs, NUM_INDEX_THREADS, stopTime, delIDs, delPackIDs, allSubDocs);
if (Verbose)
{
Console.WriteLine("TEST: DONE start " + NUM_INDEX_THREADS + " indexing threads [" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t0) + " ms]"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
// Let index build up a bit
Thread.Sleep(100);
DoSearching(es, stopTime);
if (Verbose)
{
Console.WriteLine("TEST: all searching done [" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t0) + " ms]"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
for (int thread = 0; thread < indexThreads.Length; thread++)
{
indexThreads[thread].Join();
}
if (Verbose)
{
Console.WriteLine("TEST: done join indexing threads [" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t0) + " ms]; addCount=" + m_addCount + " delCount=" + m_delCount); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
IndexSearcher s = GetFinalSearcher();
if (Verbose)
{
Console.WriteLine("TEST: finalSearcher=" + s);
}
assertFalse(m_failed);
bool doFail = false;
// Verify: make sure delIDs are in fact deleted:
foreach (string id in delIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("doc id=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " hits; first docID=" + hits.ScoreDocs[0].Doc);
doFail = true;
}
}
// Verify: make sure delPackIDs are in fact deleted:
foreach (string id in delPackIDs)
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", id)), 1);
if (hits.TotalHits != 0)
{
Console.WriteLine("packID=" + id + " is supposed to be deleted, but got " + hits.TotalHits + " matches");
doFail = true;
}
}
// Verify: make sure each group of sub-docs are still in docID order:
foreach (SubDocs subDocs in allSubDocs)
{
TopDocs hits = s.Search(new TermQuery(new Term("packID", subDocs.PackID)), 20);
if (!subDocs.Deleted)
{
// We sort by relevance but the scores should be identical so sort falls back to by docID:
if (hits.TotalHits != subDocs.SubIDs.Count)
{
Console.WriteLine("packID=" + subDocs.PackID + ": expected " + subDocs.SubIDs.Count + " hits but got " + hits.TotalHits);
doFail = true;
}
else
{
int lastDocID = -1;
int startDocID = -1;
foreach (ScoreDoc scoreDoc in hits.ScoreDocs)
{
int docID = scoreDoc.Doc;
if (lastDocID != -1)
{
assertEquals(1 + lastDocID, docID);
}
else
{
startDocID = docID;
}
lastDocID = docID;
Document doc = s.Doc(docID);
assertEquals(subDocs.PackID, doc.Get("packID"));
}
lastDocID = startDocID - 1;
foreach (string subID in subDocs.SubIDs)
{
hits = s.Search(new TermQuery(new Term("docid", subID)), 1);
assertEquals(1, hits.TotalHits);
int docID = hits.ScoreDocs[0].Doc;
if (lastDocID != -1)
{
assertEquals(1 + lastDocID, docID);
}
lastDocID = docID;
}
}
}
else
{
// Pack was deleted -- make sure its docs are
// deleted. We can't verify packID is deleted
// because we can re-use packID for update:
foreach (string subID in subDocs.SubIDs)
{
assertEquals(0, s.Search(new TermQuery(new Term("docid", subID)), 1).TotalHits);
}
}
}
// Verify: make sure all not-deleted docs are in fact
// not deleted:
int endID = Convert.ToInt32(docs.NextDoc().Get("docid"), CultureInfo.InvariantCulture);
docs.Dispose();
for (int id = 0; id < endID; id++)
{
string stringID = id.ToString(CultureInfo.InvariantCulture);
if (!delIDs.Contains(stringID))
{
TopDocs hits = s.Search(new TermQuery(new Term("docid", stringID)), 1);
if (hits.TotalHits != 1)
{
Console.WriteLine("doc id=" + stringID + " is not supposed to be deleted, but got hitCount=" + hits.TotalHits + "; delIDs=" + Collections.ToString(delIDs));
doFail = true;
}
}
}
assertFalse(doFail);
assertEquals("index=" + m_writer.SegString() + " addCount=" + m_addCount + " delCount=" + m_delCount, m_addCount - m_delCount, s.IndexReader.NumDocs);
ReleaseSearcher(s);
m_writer.Commit();
assertEquals("index=" + m_writer.SegString() + " addCount=" + m_addCount + " delCount=" + m_delCount, m_addCount - m_delCount, m_writer.NumDocs);
DoClose();
m_writer.Dispose(false);
// Cannot shutdown until after writer is closed because
// writer has merged segment warmer that uses IS to run
// searches, and that IS may be using this es!
/*if (es != null)
* {
* es.shutdown();
* es.awaitTermination(1, TimeUnit.SECONDS);
* }*/
TestUtil.CheckIndex(m_dir);
m_dir.Dispose();
//System.IO.Directory.Delete(tempDir.FullName, true);
TestUtil.Rm(tempDir);
if (Verbose)
{
Console.WriteLine("TEST: done [" + ((J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond) - t0) + " ms]"); // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
}
}