private void CheckExpecteds(BitArray expecteds)
{
IndexReader r = DirectoryReader.Open(Dir);
//Perhaps not the most efficient approach but meets our
//needs here.
IBits liveDocs = MultiFields.GetLiveDocs(r);
for (int i = 0; i < r.MaxDoc; i++)
{
if (liveDocs == null || liveDocs.Get(i))
{
string sval = r.Document(i).Get(FIELD_RECORD_ID);
if (sval != null)
{
int val = Convert.ToInt32(sval);
Assert.IsTrue(expecteds.SafeGet(val), "Did not expect document #" + val);
expecteds.SafeSet(val, false);
}
}
}
r.Dispose();
Assert.AreEqual(0, expecteds.Cardinality(), "Should have 0 docs remaining ");
}
public virtual void TestBasic()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = IndexOptions.DOCS_AND_FREQS;
Field f = NewField("foo", "this is a test test", ft);
doc.Add(f);
for (int i = 0; i < 100; i++)
{
w.AddDocument(doc);
}
IndexReader reader = w.GetReader();
w.Dispose();
Assert.IsNull(MultiFields.GetTermPositionsEnum(reader, null, "foo", new BytesRef("test")));
DocsEnum de = TestUtil.Docs(Random, reader, "foo", new BytesRef("test"), null, null, DocsFlags.FREQS);
while (de.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.AreEqual(2, de.Freq);
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestCloseWithThreads([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
int NUM_THREADS = 3;
int numIterations = TEST_NIGHTLY ? 7 : 3;
for (int iter = 0; iter < numIterations; iter++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
Directory dir = NewDirectory();
var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(10)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(4));
IndexWriter writer = new IndexWriter(dir, config);
var scheduler = config.mergeScheduler as IConcurrentMergeScheduler;
if (scheduler != null)
{
scheduler.SetSuppressExceptions();
}
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(writer, false, NewField)
// LUCENENET NOTE - ConcurrentMergeScheduler
// used to take too long for this test to index a single document
// so, increased the time from 200 to 300 ms.
// But it has now been restored to 200 ms like Lucene.
{
TimeToRunInMilliseconds = 200
};
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
bool done = false;
while (!done)
{
Thread.Sleep(100);
for (int i = 0; i < NUM_THREADS; i++)
// only stop when at least one thread has added a doc
{
if (threads[i].AddCount > 0)
{
done = true;
break;
}
else if (!threads[i].IsAlive)
{
Assert.Fail("thread failed before indexing a single document");
}
}
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: now close");
}
writer.Dispose(false);
// Make sure threads that are adding docs are not hung:
for (int i = 0; i < NUM_THREADS; i++)
{
// Without fix for LUCENE-1130: one of the
// threads will hang
threads[i].Join();
if (threads[i].IsAlive)
{
Assert.Fail("thread seems to be hung");
}
}
// Quick test to make sure index is not corrupt:
IndexReader reader = DirectoryReader.Open(dir);
DocsEnum tdocs = TestUtil.Docs(Random, reader, "field", new BytesRef("aaa"), MultiFields.GetLiveDocs(reader), null, 0);
int count = 0;
while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
count++;
}
Assert.IsTrue(count > 0);
reader.Dispose();
dir.Dispose();
}
}
public override void Run()
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (DateTime.UtcNow < StopTime)
{
try
{
IndexSearcher s = OuterInstance.CurrentSearcher;
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
Assert.IsNotNull(diagnostics);
string source = diagnostics["source"];
Assert.IsNotNull(source);
if (source.Equals("merge"))
{
Assert.IsTrue(!OuterInstance.AssertMergedSegmentsWarmed || OuterInstance.Warmed.ContainsKey((SegmentCoreReaders)segReader.CoreCacheKey), "sub reader " + sub + " wasn't warmed: warmed=" + OuterInstance.Warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo);
}
}
if (s.IndexReader.NumDocs > 0)
{
OuterInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.Terms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.Iterator(null);
int seenTermCount = 0;
int shift;
int trigger;
if (TotTermCount.Get() < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = TotTermCount.Get() / 30;
shift = Random().Next(trigger);
}
while (DateTime.UtcNow < StopTime)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
TotTermCount.Set(seenTermCount);
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
TotHits.AddAndGet(OuterInstance.RunQuery(s, new TermQuery(new Term("body", term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
OuterInstance.ReleaseSearcher(s);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
OuterInstance.Failed.Set(true);
Console.WriteLine(t.StackTrace);
throw new Exception(t.Message, t);
}
}
}
private void TestRandomSeeks(IndexReader r, params string[] validTermStrings)
{
BytesRef[] validTerms = new BytesRef[validTermStrings.Length];
for (int termIDX = 0; termIDX < validTermStrings.Length; termIDX++)
{
validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]);
}
Array.Sort(validTerms);
if (VERBOSE)
{
Console.WriteLine("TEST: " + validTerms.Length + " terms:");
foreach (BytesRef t in validTerms)
{
Console.WriteLine(" " + t.Utf8ToString() + " " + t);
}
}
TermsEnum te = MultiFields.GetTerms(r, FIELD).GetIterator(null);
int END_LOC = -validTerms.Length - 1;
IList <TermAndState> termStates = new List <TermAndState>();
for (int iter = 0; iter < 100 * RANDOM_MULTIPLIER; iter++)
{
BytesRef t;
int loc;
TermState termState;
if (Random.Next(6) == 4)
{
// pick term that doens't exist:
t = GetNonExistTerm(validTerms);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: invalid term=" + t.Utf8ToString());
}
loc = Array.BinarySearch(validTerms, t);
}
else if (termStates.Count != 0 && Random.Next(4) == 1)
{
TermAndState ts = termStates[Random.Next(termStates.Count)];
t = ts.Term;
loc = Array.BinarySearch(validTerms, t);
Assert.IsTrue(loc >= 0);
termState = ts.State;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid termState term=" + t.Utf8ToString());
}
}
else
{
// pick valid term
loc = Random.Next(validTerms.Length);
t = BytesRef.DeepCopyOf(validTerms[loc]);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid term=" + t.Utf8ToString());
}
}
// seekCeil or seekExact:
bool doSeekExact = Random.NextBoolean();
if (termState != null)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact termState");
}
te.SeekExact(t, termState);
}
else if (doSeekExact)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact");
}
Assert.AreEqual(loc >= 0, te.SeekExact(t));
}
else
{
if (VERBOSE)
{
Console.WriteLine(" seekCeil");
}
TermsEnum.SeekStatus result = te.SeekCeil(t);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (loc >= 0)
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, result);
}
else if (loc == END_LOC)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, result);
}
else
{
Debug.Assert(loc >= -validTerms.Length);
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, result);
}
}
if (loc >= 0)
{
Assert.AreEqual(t, te.Term);
}
else if (doSeekExact)
{
// TermsEnum is unpositioned if seekExact returns false
continue;
}
else if (loc == END_LOC)
{
continue;
}
else
{
loc = -loc - 1;
Assert.AreEqual(validTerms[loc], te.Term);
}
// Do a bunch of next's after the seek
int numNext = Random.Next(validTerms.Length);
for (int nextCount = 0; nextCount < numNext; nextCount++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: next loc=" + loc + " of " + validTerms.Length);
}
BytesRef t2 = te.Next();
loc++;
if (loc == validTerms.Length)
{
Assert.IsNull(t2);
break;
}
else
{
Assert.AreEqual(validTerms[loc], t2);
if (Random.Next(40) == 17 && termStates.Count < 100)
{
termStates.Add(new TermAndState(validTerms[loc], te.GetTermState()));
}
}
}
}
}
public virtual void Test()
{
Random random = new Random(Random.Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues);
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
LuceneTestCase.Random, d, analyzer);
int numDocs = AtLeast(10);
for (int docCount = 0; docCount < numDocs; docCount++)
{
w.AddDocument(docs.NextDoc());
}
IndexReader r = w.GetReader();
w.Dispose();
List <BytesRef> terms = new List <BytesRef>();
TermsEnum termsEnum = MultiFields.GetTerms(r, "body").GetIterator(null);
BytesRef term;
while ((term = termsEnum.Next()) != null)
{
terms.Add(BytesRef.DeepCopyOf(term));
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + terms.Count + " terms");
}
int upto = -1;
int iters = AtLeast(200);
for (int iter = 0; iter < iters; iter++)
{
bool isEnd;
if (upto != -1 && LuceneTestCase.Random.NextBoolean())
{
// next
if (VERBOSE)
{
Console.WriteLine("TEST: iter next");
}
isEnd = termsEnum.Next() == null;
upto++;
if (isEnd)
{
if (VERBOSE)
{
Console.WriteLine(" end");
}
Assert.AreEqual(upto, terms.Count);
upto = -1;
}
else
{
if (VERBOSE)
{
Console.WriteLine(" got term=" + termsEnum.Term.Utf8ToString() + " expected=" + terms[upto].Utf8ToString());
}
Assert.IsTrue(upto < terms.Count);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
BytesRef target;
string exists;
if (LuceneTestCase.Random.NextBoolean())
{
// likely fake term
if (LuceneTestCase.Random.NextBoolean())
{
target = new BytesRef(TestUtil.RandomSimpleString(LuceneTestCase.Random));
}
else
{
target = new BytesRef(TestUtil.RandomRealisticUnicodeString(LuceneTestCase.Random));
}
exists = "likely not";
}
else
{
// real term
target = terms[LuceneTestCase.Random.Next(terms.Count)];
exists = "yes";
}
upto = terms.BinarySearch(target);
if (LuceneTestCase.Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekCeil target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekCeil
TermsEnum.SeekStatus status = termsEnum.SeekCeil(target);
if (VERBOSE)
{
Console.WriteLine(" got " + status);
}
if (upto < 0)
{
upto = -(upto + 1);
if (upto >= terms.Count)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, status);
upto = -1;
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekExact target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekExact
bool result = termsEnum.SeekExact(target);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (upto < 0)
{
Assert.IsFalse(result);
upto = -1;
}
else
{
Assert.IsTrue(result);
Assert.AreEqual(target, termsEnum.Term);
}
}
}
}
r.Dispose();
d.Dispose();
docs.Dispose();
}
public override void Run()
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTimeMS) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
try
{
IndexSearcher s = outerInstance.GetCurrentSearcher();
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
assertNotNull(diagnostics);
diagnostics.TryGetValue("source", out string source);
assertNotNull(source);
if (source.Equals("merge", StringComparison.Ordinal))
{
#if !FEATURE_CONDITIONALWEAKTABLE_ADDORUPDATE
UninterruptableMonitor.Enter(outerInstance.warmedLock);
try
{
#endif
assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + outerInstance.warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo,
// LUCENENET: ConditionalWeakTable doesn't have ContainsKey, so we normalize to TryGetValue
!outerInstance.m_assertMergedSegmentsWarmed || outerInstance.warmed.TryGetValue(segReader.core, out BooleanRef _));
#if !FEATURE_CONDITIONALWEAKTABLE_ADDORUPDATE
}
finally
{
UninterruptableMonitor.Exit(outerInstance.warmedLock);
}
#endif
}
}
if (s.IndexReader.NumDocs > 0)
{
outerInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields is null)
{
continue;
}
Terms terms = fields.GetTerms("body");
if (terms is null)
{
continue;
}
TermsEnum termsEnum = terms.GetEnumerator();
int seenTermCount = 0;
int shift;
int trigger;
if (totTermCount < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = totTermCount / 30;
shift = Random.Next(trigger);
}
while (J2N.Time.NanoTime() / J2N.Time.MillisecondsPerNanosecond < stopTimeMS) // LUCENENET: Use NanoTime() rather than CurrentTimeMilliseconds() for more accurate/reliable results
{
if (!termsEnum.MoveNext())
{
totTermCount.Value = seenTermCount;
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
totHits.AddAndGet(outerInstance.RunQuery(s, new TermQuery(new Term("body", termsEnum.Term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
outerInstance.ReleaseSearcher(s);
}
}
catch (Exception t) when(t.IsThrowable())
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
outerInstance.m_failed.Value = (true);
Console.WriteLine(t.ToString());
throw RuntimeException.Create(t);
}
}
}
public virtual void VerifyEquals(DirectoryReader r1, DirectoryReader r2, string idField)
{
if (Verbose)
{
Console.WriteLine("\nr1 docs:");
PrintDocs(r1);
Console.WriteLine("\nr2 docs:");
PrintDocs(r2);
}
if (r1.NumDocs != r2.NumDocs)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, () => "r1.NumDocs=" + r1.NumDocs + " vs r2.NumDocs=" + r2.NumDocs);
}
}
bool hasDeletes = !(r1.MaxDoc == r2.MaxDoc && r1.NumDocs == r1.MaxDoc);
int[] r2r1 = new int[r2.MaxDoc]; // r2 id to r1 id mapping
// create mapping from id2 space to id2 based on idField
Fields f1 = MultiFields.GetFields(r1);
if (f1 == null)
{
// make sure r2 is empty
Assert.IsNull(MultiFields.GetFields(r2));
return;
}
Terms terms1 = f1.GetTerms(idField);
if (terms1 == null)
{
Assert.IsTrue(MultiFields.GetFields(r2) == null || MultiFields.GetFields(r2).GetTerms(idField) == null);
return;
}
TermsEnum termsEnum = terms1.GetIterator(null);
IBits liveDocs1 = MultiFields.GetLiveDocs(r1);
IBits liveDocs2 = MultiFields.GetLiveDocs(r2);
Fields fields = MultiFields.GetFields(r2);
if (fields == null)
{
// make sure r1 is in fact empty (eg has only all
// deleted docs):
IBits liveDocs = MultiFields.GetLiveDocs(r1);
DocsEnum docs = null;
while (termsEnum.Next() != null)
{
docs = TestUtil.Docs(Random, termsEnum, liveDocs, docs, DocsFlags.NONE);
while (docs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.Fail("r1 is not empty but r2 is");
}
}
return;
}
Terms terms2 = fields.GetTerms(idField);
TermsEnum termsEnum2 = terms2.GetIterator(null);
DocsEnum termDocs1 = null;
DocsEnum termDocs2 = null;
while (true)
{
BytesRef term = termsEnum.Next();
//System.out.println("TEST: match id term=" + term);
if (term == null)
{
break;
}
termDocs1 = TestUtil.Docs(Random, termsEnum, liveDocs1, termDocs1, DocsFlags.NONE);
if (termsEnum2.SeekExact(term))
{
termDocs2 = TestUtil.Docs(Random, termsEnum2, liveDocs2, termDocs2, DocsFlags.NONE);
}
else
{
termDocs2 = null;
}
if (termDocs1.NextDoc() == DocIdSetIterator.NO_MORE_DOCS)
{
// this doc is deleted and wasn't replaced
Assert.IsTrue(termDocs2 == null || termDocs2.NextDoc() == DocIdSetIterator.NO_MORE_DOCS);
continue;
}
int id1 = termDocs1.DocID;
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, termDocs1.NextDoc());
Assert.IsTrue(termDocs2.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int id2 = termDocs2.DocID;
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, termDocs2.NextDoc());
r2r1[id2] = id1;
// verify stored fields are equivalent
try
{
VerifyEquals(r1.Document(id1), r2.Document(id2));
}
catch (Exception /*t*/)
{
Console.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2 + " term=" + term);
Console.WriteLine(" d1=" + r1.Document(id1));
Console.WriteLine(" d2=" + r2.Document(id2));
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
try
{
// verify term vectors are equivalent
VerifyEquals(r1.GetTermVectors(id1), r2.GetTermVectors(id2));
}
catch (Exception /*e*/)
{
Console.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
Fields tv1 = r1.GetTermVectors(id1);
Console.WriteLine(" d1=" + tv1);
if (tv1 != null)
{
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
foreach (string field in tv1)
{
Console.WriteLine(" " + field + ":");
Terms terms3 = tv1.GetTerms(field);
Assert.IsNotNull(terms3);
TermsEnum termsEnum3 = terms3.GetIterator(null);
BytesRef term2;
while ((term2 = termsEnum3.Next()) != null)
{
Console.WriteLine(" " + term2.Utf8ToString() + ": freq=" + termsEnum3.TotalTermFreq);
dpEnum = termsEnum3.DocsAndPositions(null, dpEnum);
if (dpEnum != null)
{
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dpEnum.Freq;
Console.WriteLine(" doc=" + dpEnum.DocID + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++)
{
Console.WriteLine(" pos=" + dpEnum.NextPosition());
}
}
else
{
dEnum = TestUtil.Docs(Random, termsEnum3, null, dEnum, DocsFlags.FREQS);
Assert.IsNotNull(dEnum);
Assert.IsTrue(dEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dEnum.Freq;
Console.WriteLine(" doc=" + dEnum.DocID + " freq=" + freq);
}
}
}
}
Fields tv2 = r2.GetTermVectors(id2);
Console.WriteLine(" d2=" + tv2);
if (tv2 != null)
{
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
foreach (string field in tv2)
{
Console.WriteLine(" " + field + ":");
Terms terms3 = tv2.GetTerms(field);
Assert.IsNotNull(terms3);
TermsEnum termsEnum3 = terms3.GetIterator(null);
BytesRef term2;
while ((term2 = termsEnum3.Next()) != null)
{
Console.WriteLine(" " + term2.Utf8ToString() + ": freq=" + termsEnum3.TotalTermFreq);
dpEnum = termsEnum3.DocsAndPositions(null, dpEnum);
if (dpEnum != null)
{
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dpEnum.Freq;
Console.WriteLine(" doc=" + dpEnum.DocID + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++)
{
Console.WriteLine(" pos=" + dpEnum.NextPosition());
}
}
else
{
dEnum = TestUtil.Docs(Random, termsEnum3, null, dEnum, DocsFlags.FREQS);
Assert.IsNotNull(dEnum);
Assert.IsTrue(dEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dEnum.Freq;
Console.WriteLine(" doc=" + dEnum.DocID + " freq=" + freq);
}
}
}
}
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
}
//System.out.println("TEST: done match id");
// Verify postings
//System.out.println("TEST: create te1");
Fields fields1 = MultiFields.GetFields(r1);
IEnumerator <string> fields1Enum = fields1.GetEnumerator();
Fields fields2 = MultiFields.GetFields(r2);
IEnumerator <string> fields2Enum = fields2.GetEnumerator();
string field1 = null, field2 = null;
TermsEnum termsEnum1 = null;
termsEnum2 = null;
DocsEnum docs1 = null, docs2 = null;
// pack both doc and freq into single element for easy sorting
long[] info1 = new long[r1.NumDocs];
long[] info2 = new long[r2.NumDocs];
for (; ;)
{
BytesRef term1 = null, term2 = null;
// iterate until we get some docs
int len1;
for (; ;)
{
len1 = 0;
if (termsEnum1 == null)
{
if (!fields1Enum.MoveNext())
{
break;
}
field1 = fields1Enum.Current;
Terms terms = fields1.GetTerms(field1);
if (terms == null)
{
continue;
}
termsEnum1 = terms.GetIterator(null);
}
term1 = termsEnum1.Next();
if (term1 == null)
{
// no more terms in this field
termsEnum1 = null;
continue;
}
//System.out.println("TEST: term1=" + term1);
docs1 = TestUtil.Docs(Random, termsEnum1, liveDocs1, docs1, DocsFlags.FREQS);
while (docs1.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
int d = docs1.DocID;
int f = docs1.Freq;
info1[len1] = (((long)d) << 32) | (uint)f;
len1++;
}
if (len1 > 0)
{
break;
}
}
// iterate until we get some docs
int len2;
for (; ;)
{
len2 = 0;
if (termsEnum2 == null)
{
if (!fields2Enum.MoveNext())
{
break;
}
field2 = fields2Enum.Current;
Terms terms = fields2.GetTerms(field2);
if (terms == null)
{
continue;
}
termsEnum2 = terms.GetIterator(null);
}
term2 = termsEnum2.Next();
if (term2 == null)
{
// no more terms in this field
termsEnum2 = null;
continue;
}
//System.out.println("TEST: term1=" + term1);
docs2 = TestUtil.Docs(Random, termsEnum2, liveDocs2, docs2, DocsFlags.FREQS);
while (docs2.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
int d = r2r1[docs2.DocID];
int f = docs2.Freq;
info2[len2] = (((long)d) << 32) | (uint)f;
len2++;
}
if (len2 > 0)
{
break;
}
}
Assert.AreEqual(len1, len2);
if (len1 == 0) // no more terms
{
break;
}
Assert.AreEqual(field1, field2);
Assert.IsTrue(term1.BytesEquals(term2));
if (!hasDeletes)
{
Assert.AreEqual(termsEnum1.DocFreq, termsEnum2.DocFreq);
}
Assert.AreEqual(term1, term2, "len1=" + len1 + " len2=" + len2 + " deletes?=" + hasDeletes);
// sort info2 to get it into ascending docid
Array.Sort(info2, 0, len2);
// now compare
for (int i = 0; i < len1; i++)
{
Assert.AreEqual(info1[i], info2[i], "i=" + i + " len=" + len1 + " d1=" + ((long)((ulong)info1[i] >> 32)) + " f1=" + (info1[i] & int.MaxValue) + " d2=" + ((long)((ulong)info2[i] >> 32)) + " f2=" + (info2[i] & int.MaxValue) + " field=" + field1 + " term=" + term1.Utf8ToString());
}
}
}
public virtual void TestPositionIncrementGap()
{
Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper(this);
IndexWriter writer = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
doc.Add(NewTextField("repeated", "repeated one", Field.Store.YES));
doc.Add(NewTextField("repeated", "repeated two", Field.Store.YES));
writer.AddDocument(doc);
writer.Commit();
SegmentCommitInfo info = writer.NewestSegment();
writer.Dispose();
SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, NewIOContext(Random()));
DocsAndPositionsEnum termPositions = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), "repeated", new BytesRef("repeated"));
Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = termPositions.Freq;
Assert.AreEqual(2, freq);
Assert.AreEqual(0, termPositions.NextPosition());
Assert.AreEqual(502, termPositions.NextPosition());
reader.Dispose();
}
// builds an index with payloads in the given Directory and performs
// different tests to verify the payload encoding
private void PerformTest(Directory dir)
{
PayloadAnalyzer analyzer = new PayloadAnalyzer();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy()));
// should be in sync with value in TermInfosWriter
const int skipInterval = 16;
const int numTerms = 5;
const string fieldName = "f1";
int numDocs = skipInterval + 1;
// create content for the test documents with just a few terms
Term[] terms = GenerateTerms(fieldName, numTerms);
StringBuilder sb = new StringBuilder();
for (int i = 0; i < terms.Length; i++)
{
sb.Append(terms[i].Text());
sb.Append(" ");
}
string content = sb.ToString();
int payloadDataLength = numTerms * numDocs * 2 + numTerms * numDocs * (numDocs - 1) / 2;
var payloadData = GenerateRandomData(payloadDataLength);
Document d = new Document();
d.Add(NewTextField(fieldName, content, Field.Store.NO));
// add the same document multiple times to have the same payload lengths for all
// occurrences within two consecutive skip intervals
int offset = 0;
for (int i = 0; i < 2 * numDocs; i++)
{
analyzer = new PayloadAnalyzer(fieldName, payloadData, offset, 1);
offset += numTerms;
writer.AddDocument(d, analyzer);
}
// make sure we create more than one segment to test merging
writer.Commit();
// now we make sure to have different payload lengths next at the next skip point
for (int i = 0; i < numDocs; i++)
{
analyzer = new PayloadAnalyzer(fieldName, payloadData, offset, i);
offset += i * numTerms;
writer.AddDocument(d, analyzer);
}
writer.ForceMerge(1);
// flush
writer.Dispose();
/*
* Verify the index
* first we test if all payloads are stored correctly
*/
IndexReader reader = DirectoryReader.Open(dir);
var verifyPayloadData = new byte[payloadDataLength];
offset = 0;
var tps = new DocsAndPositionsEnum[numTerms];
for (int i = 0; i < numTerms; i++)
{
tps[i] = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[i].Field, new BytesRef(terms[i].Text()));
}
while (tps[0].NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
for (int i = 1; i < numTerms; i++)
{
tps[i].NextDoc();
}
int freq = tps[0].Freq;
for (int i = 0; i < freq; i++)
{
for (int j = 0; j < numTerms; j++)
{
tps[j].NextPosition();
BytesRef br = tps[j].GetPayload();
if (br != null)
{
Array.Copy(br.Bytes, br.Offset, verifyPayloadData, offset, br.Length);
offset += br.Length;
}
}
}
}
AssertByteArrayEquals(payloadData, verifyPayloadData);
/*
* test lazy skipping
*/
DocsAndPositionsEnum tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[0].Field, new BytesRef(terms[0].Text()));
tp.NextDoc();
tp.NextPosition();
// NOTE: prior rev of this test was failing to first
// call next here:
tp.NextDoc();
// now we don't read this payload
tp.NextPosition();
BytesRef payload = tp.GetPayload();
Assert.AreEqual(1, payload.Length, "Wrong payload length.");
Assert.AreEqual(payload.Bytes[payload.Offset], payloadData[numTerms]);
tp.NextDoc();
tp.NextPosition();
// we don't read this payload and skip to a different document
tp.Advance(5);
tp.NextPosition();
payload = tp.GetPayload();
Assert.AreEqual(1, payload.Length, "Wrong payload length.");
Assert.AreEqual(payload.Bytes[payload.Offset], payloadData[5 * numTerms]);
/*
* Test different lengths at skip points
*/
tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[1].Field, new BytesRef(terms[1].Text()));
tp.NextDoc();
tp.NextPosition();
Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
tp.Advance(skipInterval - 1);
tp.NextPosition();
Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
tp.Advance(2 * skipInterval - 1);
tp.NextPosition();
Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
tp.Advance(3 * skipInterval - 1);
tp.NextPosition();
Assert.AreEqual(3 * skipInterval - 2 * numDocs - 1, tp.GetPayload().Length, "Wrong payload length.");
reader.Dispose();
// test long payload
analyzer = new PayloadAnalyzer();
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.CREATE));
string singleTerm = "lucene";
d = new Document();
d.Add(NewTextField(fieldName, singleTerm, Field.Store.NO));
// add a payload whose length is greater than the buffer size of BufferedIndexOutput
payloadData = GenerateRandomData(2000);
analyzer.SetPayloadData(fieldName, payloadData, 100, 1500);
writer.AddDocument(d);
writer.ForceMerge(1);
// flush
writer.Dispose();
reader = DirectoryReader.Open(dir);
tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), fieldName, new BytesRef(singleTerm));
tp.NextDoc();
tp.NextPosition();
BytesRef bref = tp.GetPayload();
verifyPayloadData = new byte[bref.Length];
var portion = new byte[1500];
Array.Copy(payloadData, 100, portion, 0, 1500);
AssertByteArrayEquals(portion, bref.Bytes, bref.Offset, bref.Length);
reader.Dispose();
}
public virtual void TestCloseWithThreads()
{
int NUM_THREADS = 3;
int numIterations = TEST_NIGHTLY ? 7 : 3;
for (int iter = 0; iter < numIterations; iter++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergeScheduler(new ConcurrentMergeScheduler()).SetMergePolicy(NewLogMergePolicy(4)));
((ConcurrentMergeScheduler)writer.Config.MergeScheduler).SetSuppressExceptions();
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(this, writer, false);
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
bool done = false;
while (!done)
{
Thread.Sleep(100);
for (int i = 0; i < NUM_THREADS; i++)
// only stop when at least one thread has added a doc
{
if (threads[i].AddCount > 0)
{
done = true;
break;
}
else if (!threads[i].IsAlive)
{
Assert.Fail("thread failed before indexing a single document");
}
}
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: now close");
}
writer.Dispose(false);
// Make sure threads that are adding docs are not hung:
for (int i = 0; i < NUM_THREADS; i++)
{
// Without fix for LUCENE-1130: one of the
// threads will hang
threads[i].Join();
if (threads[i].IsAlive)
{
Assert.Fail("thread seems to be hung");
}
}
// Quick test to make sure index is not corrupt:
IndexReader reader = DirectoryReader.Open(dir);
DocsEnum tdocs = TestUtil.Docs(Random(), reader, "field", new BytesRef("aaa"), MultiFields.GetLiveDocs(reader), null, 0);
int count = 0;
while (tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
count++;
}
Assert.IsTrue(count > 0);
reader.Dispose();
dir.Dispose();
}
}
public virtual void TestMerge()
{
Codec codec = Codec.Default;
SegmentInfo si = new SegmentInfo(MergedDir, Constants.LUCENE_MAIN_VERSION, MergedSegment, -1, false, codec, null);
SegmentMerger merger = new SegmentMerger(Arrays.AsList <AtomicReader>(Reader1, Reader2), si, InfoStream.Default, MergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, CheckAbort.NONE, new FieldInfos.FieldNumbers(), NewIOContext(Random()), true);
MergeState mergeState = merger.Merge();
int docsMerged = mergeState.SegmentInfo.DocCount;
Assert.IsTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(new SegmentInfo(MergedDir, Constants.LUCENE_MAIN_VERSION, MergedSegment, docsMerged, false, codec, null), 0, -1L, -1L), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, NewIOContext(Random()));
Assert.IsTrue(mergedReader != null);
Assert.IsTrue(mergedReader.NumDocs == 2);
Document newDoc1 = mergedReader.Document(0);
Assert.IsTrue(newDoc1 != null);
//There are 2 unstored fields on the document
Assert.IsTrue(DocHelper.NumFields(newDoc1) == DocHelper.NumFields(Doc1) - DocHelper.Unstored.Count);
Document newDoc2 = mergedReader.Document(1);
Assert.IsTrue(newDoc2 != null);
Assert.IsTrue(DocHelper.NumFields(newDoc2) == DocHelper.NumFields(Doc2) - DocHelper.Unstored.Count);
DocsEnum termDocs = TestUtil.Docs(Random(), mergedReader, DocHelper.TEXT_FIELD_2_KEY, new BytesRef("field"), MultiFields.GetLiveDocs(mergedReader), null, 0);
Assert.IsTrue(termDocs != null);
Assert.IsTrue(termDocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int tvCount = 0;
foreach (FieldInfo fieldInfo in mergedReader.FieldInfos)
{
if (fieldInfo.HasVectors)
{
tvCount++;
}
}
//System.out.println("stored size: " + stored.Size());
Assert.AreEqual(3, tvCount, "We do not have 3 fields that were indexed with term vector");
Terms vector = mergedReader.GetTermVectors(0).GetTerms(DocHelper.TEXT_FIELD_2_KEY);
Assert.IsNotNull(vector);
Assert.AreEqual(3, vector.Count);
TermsEnum termsEnum = vector.GetIterator(null);
int i = 0;
while (termsEnum.Next() != null)
{
string term = termsEnum.Term.Utf8ToString();
int freq = (int)termsEnum.TotalTermFreq;
//System.out.println("Term: " + term + " Freq: " + freq);
Assert.IsTrue(DocHelper.FIELD_2_TEXT.IndexOf(term) != -1);
Assert.IsTrue(DocHelper.FIELD_2_FREQS[i] == freq);
i++;
}
TestSegmentReader.CheckNorms(mergedReader);
mergedReader.Dispose();
}
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef)
{
DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt32(Random, 2, 10));
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(r, "id", false);
/*
* for(int docID=0;docID<subR.MaxDoc;docID++) {
* System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
* }
*/
if (VERBOSE)
{
Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString()));
Console.WriteLine("TEST: all TERMS:");
TermsEnum allTE = MultiFields.GetTerms(r, "field").GetIterator(null);
int ord = 0;
while (allTE.Next() != null)
{
Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term.Utf8ToString());
}
}
//final TermsEnum te = subR.Fields.Terms("field").iterator();
TermsEnum te = dto.GetOrdTermsEnum(r);
if (dto.NumTerms == 0)
{
if (prefixRef == null)
{
Assert.IsNull(MultiFields.GetTerms(r, "field"));
}
else
{
Terms terms = MultiFields.GetTerms(r, "field");
if (terms != null)
{
TermsEnum termsEnum = terms.GetIterator(null);
TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef);
if (result != TermsEnum.SeekStatus.END)
{
Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term, prefixRef), "term=" + termsEnum.Term.Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString());
}
else
{
// ok
}
}
else
{
// ok
}
}
return;
}
if (VERBOSE)
{
Console.WriteLine("TEST: TERMS:");
te.SeekExact(0);
while (true)
{
Console.WriteLine(" ord=" + te.Ord + " term=" + te.Term.Utf8ToString());
if (te.Next() == null)
{
break;
}
}
}
SortedSetDocValues iter = dto.GetIterator(r);
for (int docID = 0; docID < r.MaxDoc; docID++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")");
}
iter.SetDocument(docID);
int[] answers = idToOrds[docIDToID.Get(docID)];
int upto = 0;
long ord;
while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
te.SeekExact(ord);
BytesRef expected = termsArray[answers[upto++]];
if (VERBOSE)
{
Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString());
}
Assert.AreEqual(expected, te.Term, "expected=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString() + " ord=" + ord);
}
Assert.AreEqual(answers.Length, upto);
}
}
public virtual void TestSkipTo(int indexDivisor)
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
Term ta = new Term("content", "aaa");
for (int i = 0; i < 10; i++)
{
AddDoc(writer, "aaa aaa aaa aaa");
}
Term tb = new Term("content", "bbb");
for (int i = 0; i < 16; i++)
{
AddDoc(writer, "bbb bbb bbb bbb");
}
Term tc = new Term("content", "ccc");
for (int i = 0; i < 50; i++)
{
AddDoc(writer, "ccc ccc ccc ccc");
}
// assure that we deal with a single segment
writer.ForceMerge(1);
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir, indexDivisor);
DocsEnum tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);
// without optimization (assumption skipInterval == 16)
// with next
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(0, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.Advance(2) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(2, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(4, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(9, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);
// without next
tdocs = TestUtil.Docs(Random(), reader, ta.Field(), new BytesRef(ta.Text()), MultiFields.GetLiveDocs(reader), null, 0);
Assert.IsTrue(tdocs.Advance(0) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(0, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(4) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(4, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(9) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(9, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(10) != DocIdSetIterator.NO_MORE_DOCS);
// exactly skipInterval documents and therefore with optimization
// with next
tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(10, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(11, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.Advance(12) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(12, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(15, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(24, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(25, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);
// without next
tdocs = TestUtil.Docs(Random(), reader, tb.Field(), new BytesRef(tb.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);
Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(10, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(15) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(15, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(24) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(24, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(25) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(25, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(26) != DocIdSetIterator.NO_MORE_DOCS);
// much more than skipInterval documents and therefore with optimization
// with next
tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(26, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(27, tdocs.DocID());
Assert.AreEqual(4, tdocs.Freq());
Assert.IsTrue(tdocs.Advance(28) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(28, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(40, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(57, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(74, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(75, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);
//without next
tdocs = TestUtil.Docs(Random(), reader, tc.Field(), new BytesRef(tc.Text()), MultiFields.GetLiveDocs(reader), null, 0);
Assert.IsTrue(tdocs.Advance(5) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(26, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(40) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(40, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(57) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(57, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(74) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(74, tdocs.DocID());
Assert.IsTrue(tdocs.Advance(75) != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(75, tdocs.DocID());
Assert.IsFalse(tdocs.Advance(76) != DocIdSetIterator.NO_MORE_DOCS);
reader.Dispose();
dir.Dispose();
}
public override void Run()
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (Environment.TickCount < stopTimeMS)
{
try
{
IndexSearcher s = outerInstance.CurrentSearcher;
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
assertNotNull(diagnostics);
string source;
diagnostics.TryGetValue("source", out source);
assertNotNull(source);
if (source.Equals("merge", StringComparison.Ordinal))
{
assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + outerInstance.warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo,
!outerInstance.assertMergedSegmentsWarmed || outerInstance.warmed.ContainsKey(segReader.core));
}
}
if (s.IndexReader.NumDocs > 0)
{
outerInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.GetTerms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.GetIterator(null);
int seenTermCount = 0;
int shift;
int trigger;
if (totTermCount.Get() < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = totTermCount.Get() / 30;
shift = Random().Next(trigger);
}
while (Environment.TickCount < stopTimeMS)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
totTermCount.Set(seenTermCount);
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
totHits.AddAndGet(outerInstance.RunQuery(s, new TermQuery(new Term("body", term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
outerInstance.ReleaseSearcher(s);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
outerInstance.failed.Set(true);
Console.WriteLine(t.ToString());
throw new Exception(t.ToString(), t);
}
}
}
// Runs test, with multiple threads, using the specific
// failure to trigger an IOException
public virtual void TestMultipleThreadsFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure)
{
int NUM_THREADS = 3;
for (int iter = 0; iter < 2; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
MockDirectoryWrapper dir = NewMockDirectory();
var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(2)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(4));
IndexWriter writer = new IndexWriter(dir, config);
var scheduler = config.mergeScheduler as IConcurrentMergeScheduler;
if (scheduler != null)
{
scheduler.SetSuppressExceptions();
}
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(writer, true, NewField);
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
Thread.Sleep(10);
dir.FailOn(failure);
failure.SetDoFail();
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable");
}
bool success = false;
try
{
writer.Dispose(false);
success = true;
}
catch (IOException)
{
failure.ClearDoFail();
writer.Dispose(false);
}
if (VERBOSE)
{
Console.WriteLine("TEST: success=" + success);
}
if (success)
{
IndexReader reader = DirectoryReader.Open(dir);
IBits delDocs = MultiFields.GetLiveDocs(reader);
for (int j = 0; j < reader.MaxDoc; j++)
{
if (delDocs == null || !delDocs.Get(j))
{
reader.Document(j);
reader.GetTermVectors(j);
}
}
reader.Dispose();
}
dir.Dispose();
}
}
public virtual void SearchIndex(Directory dir, string oldName)
{
//QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
//Query query = parser.parse("handle:1");
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = NewSearcher(reader);
TestUtil.CheckIndex(dir);
// true if this is a 4.0+ index
bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null;
// true if this is a 4.2+ index
bool is42Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("dvSortedSet") != null;
Debug.Assert(is40Index); // NOTE: currently we can only do this on trunk!
Bits liveDocs = MultiFields.GetLiveDocs(reader);
for (int i = 0; i < 35; i++)
{
if (liveDocs.Get(i))
{
Document d = reader.Document(i);
IList <IndexableField> fields = d.Fields;
bool isProxDoc = d.GetField("content3") == null;
if (isProxDoc)
{
int numFields = is40Index ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
IndexableField f = d.GetField("id");
Assert.AreEqual("" + i, f.StringValue);
f = d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue);
f = d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue);
}
Fields tfvFields = reader.GetTermVectors(i);
Assert.IsNotNull(tfvFields, "i=" + i);
Terms tfv = tfvFields.Terms("utf8");
Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName);
}
else
{
// Only ID 7 is deleted
Assert.AreEqual(7, i);
}
}
if (is40Index)
{
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort");
SortedSetDocValues dvSortedSet = null;
if (is42Index)
{
dvSortedSet = MultiDocValues.GetSortedSetValues(reader, "dvSortedSet");
}
for (int i = 0; i < 35; i++)
{
int id = Convert.ToInt32(reader.Document(i).Get("id"));
Assert.AreEqual(id, dvByte.Get(i));
sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id };
BytesRef expectedRef = new BytesRef((byte[])(Array)bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesDerefVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
Assert.AreEqual((double)id, BitConverter.Int64BitsToDouble(dvDouble.Get(i)), 0D);
Assert.AreEqual((float)id, Number.IntBitsToFloat((int)dvFloat.Get(i)), 0F);
Assert.AreEqual(id, dvInt.Get(i));
Assert.AreEqual(id, dvLong.Get(i));
Assert.AreEqual(id, dvPacked.Get(i));
Assert.AreEqual(id, dvShort.Get(i));
if (is42Index)
{
dvSortedSet.Document = i;
long ord = dvSortedSet.NextOrd();
Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.NextOrd());
dvSortedSet.LookupOrd(ord, scratch);
Assert.AreEqual(expectedRef, scratch);
}
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #0
Document doc = searcher.IndexReader.Document(hits[0].Doc);
assertEquals("didn't get the right document first", "0", doc.Get("id"));
DoTestHits(hits, 34, searcher.IndexReader);
if (is40Index)
{
hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
}
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
reader.Dispose();
}
public virtual void TestLongPostings_Mem()
{
// Don't use TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random.NextInt64()));
int NUM_DOCS = AtLeast(2000);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
string s1 = GetRandomTerm(null);
string s2 = GetRandomTerm(s1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2);
/*
* for(int idx=0;idx<s1.Length();idx++) {
* System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
* }
* for(int idx=0;idx<s2.Length();idx++) {
* System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
* }
*/
}
FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
if (Random.NextBoolean())
{
isS1.Set(idx);
}
}
IndexReader r;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy());
iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random.NextDouble());
iwc.SetMaxBufferedDocs(-1);
RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
Document doc = new Document();
string s = isS1.Get(idx) ? s1 : s2;
Field f = NewTextField("field", s, Field.Store.NO);
int count = TestUtil.NextInt32(Random, 1, 4);
for (int ct = 0; ct < count; ct++)
{
doc.Add(f);
}
riw.AddDocument(doc);
}
r = riw.GetReader();
riw.Dispose();
/*
* if (VERBOSE) {
* System.out.println("TEST: terms");
* TermEnum termEnum = r.Terms();
* while(termEnum.Next()) {
* System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().Length());
* Assert.IsTrue(termEnum.DocFreq() > 0);
* System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.Length());
* System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.Length());
* final String s = termEnum.Term().Text();
* for(int idx=0;idx<s.Length();idx++) {
* System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx)));
* }
* }
* }
*/
Assert.AreEqual(NUM_DOCS, r.NumDocs);
Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0);
Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0);
int num = AtLeast(1000);
for (int iter = 0; iter < num; iter++)
{
string term;
bool doS1;
if (Random.NextBoolean())
{
term = s1;
doS1 = true;
}
else
{
term = s2;
doS1 = false;
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1);
}
DocsAndPositionsEnum postings = MultiFields.GetTermPositionsEnum(r, null, "field", new BytesRef(term));
int docID = -1;
while (docID < DocIdSetIterator.NO_MORE_DOCS)
{
int what = Random.Next(3);
if (what == 0)
{
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do next()");
}
// nextDoc
int expected = docID + 1;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = postings.NextDoc();
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4);
for (int pos = 0; pos < freq; pos++)
{
Assert.AreEqual(pos, postings.NextPosition());
if (Random.NextBoolean())
{
var dummy = postings.GetPayload();
if (Random.NextBoolean())
{
dummy = postings.GetPayload(); // get it again
}
}
}
}
}
else
{
// advance
int targetDocID;
if (docID == -1)
{
targetDocID = Random.Next(NUM_DOCS + 1);
}
else
{
targetDocID = docID + TestUtil.NextInt32(Random, 1, NUM_DOCS - docID);
}
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
}
int expected = targetDocID;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = postings.Advance(targetDocID);
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4);
for (int pos = 0; pos < freq; pos++)
{
Assert.AreEqual(pos, postings.NextPosition());
if (Random.NextBoolean())
{
var dummy = postings.GetPayload();
if (Random.NextBoolean())
{
dummy = postings.GetPayload(); // get it again
}
}
}
}
}
}
}
r.Dispose();
dir.Dispose();
}
public virtual void DoTestNumbers(bool withPayloads)
{
Directory dir = NewDirectory();
Analyzer analyzer = withPayloads ? (Analyzer) new MockPayloadAnalyzer() : new MockAnalyzer(Random);
iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
iwc.SetMergePolicy(NewLogMergePolicy()); // will rely on docids a bit for skipping
RandomIndexWriter w = new RandomIndexWriter(Random, dir, iwc);
FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
if (Random.NextBoolean())
{
ft.StoreTermVectors = true;
ft.StoreTermVectorOffsets = Random.NextBoolean();
ft.StoreTermVectorPositions = Random.NextBoolean();
}
int numDocs = AtLeast(500);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new Field("numbers", English.Int32ToEnglish(i), ft));
doc.Add(new Field("oddeven", (i % 2) == 0 ? "even" : "odd", ft));
doc.Add(new StringField("id", "" + i, Field.Store.NO));
w.AddDocument(doc);
}
IndexReader reader = w.GetReader();
w.Dispose();
string[] terms = new string[] { "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "hundred" };
foreach (string term in terms)
{
DocsAndPositionsEnum dp = MultiFields.GetTermPositionsEnum(reader, null, "numbers", new BytesRef(term));
int doc;
while ((doc = dp.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
string storedNumbers = reader.Document(doc).Get("numbers");
int freq = dp.Freq;
for (int i = 0; i < freq; i++)
{
dp.NextPosition();
int start = dp.StartOffset;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(start >= 0);
}
int end = dp.EndOffset;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(end >= 0 && end >= start);
}
// check that the offsets correspond to the term in the src text
Assert.IsTrue(storedNumbers.Substring(start, end - start).Equals(term, StringComparison.Ordinal));
if (withPayloads)
{
// check that we have a payload and it starts with "pos"
Assert.IsNotNull(dp.GetPayload());
BytesRef payload = dp.GetPayload();
Assert.IsTrue(payload.Utf8ToString().StartsWith("pos:", StringComparison.Ordinal));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
}
}
}
// check we can skip correctly
int numSkippingTests = AtLeast(50);
for (int j = 0; j < numSkippingTests; j++)
{
int num = TestUtil.NextInt32(Random, 100, Math.Min(numDocs - 1, 999));
DocsAndPositionsEnum dp = MultiFields.GetTermPositionsEnum(reader, null, "numbers", new BytesRef("hundred"));
int doc = dp.Advance(num);
Assert.AreEqual(num, doc);
int freq = dp.Freq;
for (int i = 0; i < freq; i++)
{
string storedNumbers = reader.Document(doc).Get("numbers");
dp.NextPosition();
int start = dp.StartOffset;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(start >= 0);
}
int end = dp.EndOffset;
if (Debugging.AssertsEnabled)
{
Debugging.Assert(end >= 0 && end >= start);
}
// check that the offsets correspond to the term in the src text
Assert.IsTrue(storedNumbers.Substring(start, end - start).Equals("hundred", StringComparison.Ordinal));
if (withPayloads)
{
// check that we have a payload and it starts with "pos"
Assert.IsNotNull(dp.GetPayload());
BytesRef payload = dp.GetPayload();
Assert.IsTrue(payload.Utf8ToString().StartsWith("pos:", StringComparison.Ordinal));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
}
}
// check that other fields (without offsets) work correctly
for (int i = 0; i < numDocs; i++)
{
DocsEnum dp = MultiFields.GetTermDocsEnum(reader, null, "id", new BytesRef("" + i), 0);
Assert.AreEqual(i, dp.NextDoc());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dp.NextDoc());
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestRandom()
{
int num = AtLeast(2);
for (int iter = 0; iter < num; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NoMergePolicy.COMPOUND_FILES));
// we can do this because we use NoMergePolicy (and dont merge to "nothing")
w.KeepFullyDeletedSegments = true;
IDictionary <BytesRef, IList <int?> > docs = new Dictionary <BytesRef, IList <int?> >();
ISet <int?> deleted = new JCG.HashSet <int?>();
IList <BytesRef> terms = new List <BytesRef>();
int numDocs = TestUtil.NextInt32(Random, 1, 100 * RANDOM_MULTIPLIER);
Documents.Document doc = new Documents.Document();
Field f = NewStringField("field", "", Field.Store.NO);
doc.Add(f);
Field id = NewStringField("id", "", Field.Store.NO);
doc.Add(id);
bool onlyUniqueTerms = Random.NextBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: onlyUniqueTerms=" + onlyUniqueTerms + " numDocs=" + numDocs);
}
ISet <BytesRef> uniqueTerms = new JCG.HashSet <BytesRef>();
for (int i = 0; i < numDocs; i++)
{
if (!onlyUniqueTerms && Random.NextBoolean() && terms.Count > 0)
{
// re-use existing term
BytesRef term = terms[Random.Next(terms.Count)];
docs[term].Add(i);
f.SetStringValue(term.Utf8ToString());
}
else
{
string s = TestUtil.RandomUnicodeString(Random, 10);
BytesRef term = new BytesRef(s);
if (!docs.TryGetValue(term, out IList <int?> docsTerm))
{
docs[term] = docsTerm = new List <int?>();
}
docsTerm.Add(i);
terms.Add(term);
uniqueTerms.Add(term);
f.SetStringValue(s);
}
id.SetStringValue("" + i);
w.AddDocument(doc);
if (Random.Next(4) == 1)
{
w.Commit();
}
if (i > 0 && Random.Next(20) == 1)
{
int delID = Random.Next(i);
deleted.Add(delID);
w.DeleteDocuments(new Term("id", "" + delID));
if (VERBOSE)
{
Console.WriteLine("TEST: delete " + delID);
}
}
}
if (VERBOSE)
{
List <BytesRef> termsList = new List <BytesRef>(uniqueTerms);
#pragma warning disable 612, 618
termsList.Sort(BytesRef.UTF8SortedAsUTF16Comparer);
#pragma warning restore 612, 618
Console.WriteLine("TEST: terms in UTF16 order:");
foreach (BytesRef b in termsList)
{
Console.WriteLine(" " + UnicodeUtil.ToHexString(b.Utf8ToString()) + " " + b);
foreach (int docID in docs[b])
{
if (deleted.Contains(docID))
{
Console.WriteLine(" " + docID + " (deleted)");
}
else
{
Console.WriteLine(" " + docID);
}
}
}
}
IndexReader reader = w.GetReader();
w.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: reader=" + reader);
}
IBits liveDocs = MultiFields.GetLiveDocs(reader);
foreach (int delDoc in deleted)
{
Assert.IsFalse(liveDocs.Get(delDoc));
}
for (int i = 0; i < 100; i++)
{
BytesRef term = terms[Random.Next(terms.Count)];
if (VERBOSE)
{
Console.WriteLine("TEST: seek term=" + UnicodeUtil.ToHexString(term.Utf8ToString()) + " " + term);
}
DocsEnum docsEnum = TestUtil.Docs(Random, reader, "field", term, liveDocs, null, DocsFlags.NONE);
Assert.IsNotNull(docsEnum);
foreach (int docID in docs[term])
{
if (!deleted.Contains(docID))
{
Assert.AreEqual(docID, docsEnum.NextDoc());
}
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, docsEnum.NextDoc());
}
reader.Dispose();
dir.Dispose();
}
}
public virtual void TestTerms()
{
Fields fields = MultiFields.GetFields(Reader);
foreach (string field in fields)
{
Terms terms = fields.GetTerms(field);
Assert.IsNotNull(terms);
TermsEnum termsEnum = terms.GetIterator(null);
while (termsEnum.Next() != null)
{
BytesRef term = termsEnum.Term;
Assert.IsTrue(term != null);
string fieldValue = (string)DocHelper.NameValues[field];
Assert.IsTrue(fieldValue.IndexOf(term.Utf8ToString(), StringComparison.Ordinal) != -1);
}
}
DocsEnum termDocs = TestUtil.Docs(Random, Reader, DocHelper.TEXT_FIELD_1_KEY, new BytesRef("field"), MultiFields.GetLiveDocs(Reader), null, 0);
Assert.IsTrue(termDocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
termDocs = TestUtil.Docs(Random, Reader, DocHelper.NO_NORMS_KEY, new BytesRef(DocHelper.NO_NORMS_TEXT), MultiFields.GetLiveDocs(Reader), null, 0);
Assert.IsTrue(termDocs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
DocsAndPositionsEnum positions = MultiFields.GetTermPositionsEnum(Reader, MultiFields.GetLiveDocs(Reader), DocHelper.TEXT_FIELD_1_KEY, new BytesRef("field"));
// NOTE: prior rev of this test was failing to first
// call next here:
Assert.IsTrue(positions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.IsTrue(positions.DocID == 0);
Assert.IsTrue(positions.NextPosition() >= 0);
}
public virtual void TestIntersectRandom()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numTerms = AtLeast(300);
//final int numTerms = 50;
HashSet <string> terms = new HashSet <string>();
ICollection <string> pendingTerms = new List <string>();
IDictionary <BytesRef, int?> termToID = new Dictionary <BytesRef, int?>();
int id = 0;
while (terms.Count != numTerms)
{
string s = RandomString;
if (!terms.Contains(s))
{
terms.Add(s);
pendingTerms.Add(s);
if (Random.Next(20) == 7)
{
AddDoc(w, pendingTerms, termToID, id++);
}
}
}
AddDoc(w, pendingTerms, termToID, id++);
BytesRef[] termsArray = new BytesRef[terms.Count];
HashSet <BytesRef> termsSet = new HashSet <BytesRef>();
{
int upto = 0;
foreach (string s in terms)
{
BytesRef b = new BytesRef(s);
termsArray[upto++] = b;
termsSet.Add(b);
}
Array.Sort(termsArray);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: indexed terms (unicode order):");
foreach (BytesRef t in termsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + " -> id:" + termToID[t]);
}
}
IndexReader r = w.GetReader();
w.Dispose();
// NOTE: intentional insanity!!
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
{
// TODO: can we also test infinite As here...?
// From the random terms, pick some ratio and compile an
// automaton:
HashSet <string> acceptTerms = new HashSet <string>();
SortedSet <BytesRef> sortedAcceptTerms = new SortedSet <BytesRef>();
double keepPct = Random.NextDouble();
Automaton a;
if (iter == 0)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: empty automaton");
}
a = BasicAutomata.MakeEmpty();
}
else
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: keepPct=" + keepPct);
}
foreach (string s in terms)
{
string s2;
if (Random.NextDouble() <= keepPct)
{
s2 = s;
}
else
{
s2 = RandomString;
}
acceptTerms.Add(s2);
sortedAcceptTerms.Add(new BytesRef(s2));
}
a = BasicAutomata.MakeStringUnion(sortedAcceptTerms);
}
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: reduce the automaton");
}
a.Reduce();
}
CompiledAutomaton c = new CompiledAutomaton(a, true, false);
BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.Count];
HashSet <BytesRef> acceptTermsSet = new HashSet <BytesRef>();
int upto = 0;
foreach (string s in acceptTerms)
{
BytesRef b = new BytesRef(s);
acceptTermsArray[upto++] = b;
acceptTermsSet.Add(b);
Assert.IsTrue(Accepts(c, b));
}
Array.Sort(acceptTermsArray);
if (VERBOSE)
{
Console.WriteLine("\nTEST: accept terms (unicode order):");
foreach (BytesRef t in acceptTermsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + (termsSet.Contains(t) ? " (exists)" : ""));
}
Console.WriteLine(a.ToDot());
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
BytesRef startTerm = acceptTermsArray.Length == 0 || Random.NextBoolean() ? null : acceptTermsArray[Random.Next(acceptTermsArray.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.Utf8ToString()));
if (startTerm != null)
{
int state = c.RunAutomaton.InitialState;
for (int idx = 0; idx < startTerm.Length; idx++)
{
int label = startTerm.Bytes[startTerm.Offset + idx] & 0xff;
Console.WriteLine(" state=" + state + " label=" + label);
state = c.RunAutomaton.Step(state, label);
Assert.IsTrue(state != -1);
}
Console.WriteLine(" state=" + state);
}
}
TermsEnum te = MultiFields.GetTerms(r, "f").Intersect(c, startTerm);
int loc;
if (startTerm == null)
{
loc = 0;
}
else
{
loc = Array.BinarySearch(termsArray, BytesRef.DeepCopyOf(startTerm));
if (loc < 0)
{
loc = -(loc + 1);
}
else
{
// startTerm exists in index
loc++;
}
}
while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]))
{
loc++;
}
DocsEnum docsEnum = null;
while (loc < termsArray.Length)
{
BytesRef expected = termsArray[loc];
BytesRef actual = te.Next();
if (VERBOSE)
{
Console.WriteLine("TEST: next() expected=" + expected.Utf8ToString() + " actual=" + (actual == null ? "null" : actual.Utf8ToString()));
}
Assert.AreEqual(expected, actual);
Assert.AreEqual(1, te.DocFreq);
docsEnum = TestUtil.Docs(Random, te, null, docsEnum, DocsFlags.NONE);
int docID = docsEnum.NextDoc();
Assert.IsTrue(docID != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(docIDToID.Get(docID), (int)termToID[expected]);
do
{
loc++;
} while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]));
}
Assert.IsNull(te.Next());
}
}
r.Dispose();
dir.Dispose();
}
