public IntDocValuesAnonymousInnerClassHelper(TermFreqValueSource outerInstance, TermFreqValueSource @this, Terms terms)
: base(@this)
{
this.outerInstance = outerInstance;
this.terms = terms;
lastDocRequested = -1;
Reset();
}
public override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
if (_terms.Size() == 0)
{
return TermsEnum.EMPTY;
}
return new SeekingTermSetTermsEnum(terms.Iterator(null), _terms, _ords);
}
private void CheckTerms(Terms terms, Bits liveDocs, params string[] termsList)
{
Assert.IsNotNull(terms);
TermsEnum te = terms.Iterator(null);
foreach (string t in termsList)
{
BytesRef b = te.Next();
Assert.IsNotNull(b);
Assert.AreEqual(t, b.Utf8ToString());
DocsEnum td = TestUtil.Docs(Random(), te, liveDocs, null, DocsEnum.FLAG_NONE);
Assert.IsTrue(td.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(0, td.DocID());
Assert.AreEqual(td.NextDoc(), DocIdSetIterator.NO_MORE_DOCS);
}
Assert.IsNull(te.Next());
}
/// <summary>
/// Sole constructor.
/// </summary>
/// <param name="subs"> The <seealso cref="Terms"/> instances of all sub-readers. </param>
/// <param name="subSlices"> A parallel array (matching {@code
/// subs}) describing the sub-reader slices. </param>
public MultiTerms(Terms[] subs, ReaderSlice[] subSlices)
{
this.Subs = subs;
this.SubSlices = subSlices;
IComparer<BytesRef> _termComp = null;
Debug.Assert(subs.Length > 0, "inefficient: don't use MultiTerms over one sub");
bool _hasFreqs = true;
bool _hasOffsets = true;
bool _hasPositions = true;
bool _hasPayloads = false;
for (int i = 0; i < subs.Length; i++)
{
if (_termComp == null)
{
_termComp = subs[i].Comparator;
}
else
{
// We cannot merge sub-readers that have
// different TermComps
IComparer<BytesRef> subTermComp = subs[i].Comparator;
if (subTermComp != null && !subTermComp.Equals(_termComp))
{
throw new InvalidOperationException("sub-readers have different BytesRef.Comparators; cannot merge");
}
}
_hasFreqs &= subs[i].HasFreqs();
_hasOffsets &= subs[i].HasOffsets();
_hasPositions &= subs[i].HasPositions();
_hasPayloads |= subs[i].HasPayloads();
}
TermComp = _termComp;
HasFreqs_Renamed = _hasFreqs;
HasOffsets_Renamed = _hasOffsets;
HasPositions_Renamed = _hasPositions;
HasPayloads_Renamed = HasPositions_Renamed && _hasPayloads; // if all subs have pos, and at least one has payloads.
}
// following code is almost an exact dup of code from TestDuelingCodecs: sorry!
public virtual void AssertTerms(Terms leftTerms, Terms rightTerms, bool deep)
{
if (leftTerms == null || rightTerms == null)
{
Assert.IsNull(leftTerms);
Assert.IsNull(rightTerms);
return;
}
AssertTermsStatistics(leftTerms, rightTerms);
// NOTE: we don't assert hasOffsets/hasPositions/hasPayloads because they are allowed to be different
TermsEnum leftTermsEnum = leftTerms.Iterator(null);
TermsEnum rightTermsEnum = rightTerms.Iterator(null);
AssertTermsEnum(leftTermsEnum, rightTermsEnum, true);
AssertTermsSeeking(leftTerms, rightTerms);
if (deep)
{
int numIntersections = AtLeast(3);
for (int i = 0; i < numIntersections; i++)
{
string re = AutomatonTestUtil.RandomRegexp(Random());
CompiledAutomaton automaton = new CompiledAutomaton((new RegExp(re, RegExp.NONE)).ToAutomaton());
if (automaton.Type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL)
{
// TODO: test start term too
TermsEnum leftIntersection = leftTerms.Intersect(automaton, null);
TermsEnum rightIntersection = rightTerms.Intersect(automaton, null);
AssertTermsEnum(leftIntersection, rightIntersection, Rarely());
}
}
}
}
internal void AddField(string fieldName, Terms terms)
{
Fields[fieldName] = terms;
}
public override TermsEnum GetTermsEnum(Terms terms, AttributeSource atts)
{
if (!termLongEnough)
{ // can only match if it's exact
return new SingleTermsEnum(terms.Iterator(null), term.Bytes);
}
return new SlowFuzzyTermsEnum(terms, atts, Term, minimumSimilarity, prefixLength);
}
/// <summary>
/// checks collection-level statistics on Terms
/// </summary>
public void AssertTermsStatisticsEquals(string info, Terms leftTerms, Terms rightTerms)
{
Debug.Assert(leftTerms.Comparator == rightTerms.Comparator);
if (leftTerms.DocCount != -1 && rightTerms.DocCount != -1)
{
Assert.AreEqual(leftTerms.DocCount, rightTerms.DocCount, info);
}
if (leftTerms.SumDocFreq != -1 && rightTerms.SumDocFreq != -1)
{
Assert.AreEqual(leftTerms.SumDocFreq, rightTerms.SumDocFreq, info);
}
if (leftTerms.SumTotalTermFreq != -1 && rightTerms.SumTotalTermFreq != -1)
{
Assert.AreEqual(leftTerms.SumTotalTermFreq, rightTerms.SumTotalTermFreq, info);
}
if (leftTerms.Size() != -1 && rightTerms.Size() != -1)
{
Assert.AreEqual(leftTerms.Size(), rightTerms.Size(), info);
}
}
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef)
{
DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt(Random(), 2, 10));
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(r, "id", false);
/*
* for(int docID=0;docID<subR.MaxDoc;docID++) {
* System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
* }
*/
if (VERBOSE)
{
Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString()));
Console.WriteLine("TEST: all TERMS:");
TermsEnum allTE = MultiFields.GetTerms(r, "field").GetIterator(null);
int ord = 0;
while (allTE.Next() != null)
{
Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term.Utf8ToString());
}
}
//final TermsEnum te = subR.Fields.Terms("field").iterator();
TermsEnum te = dto.GetOrdTermsEnum(r);
if (dto.NumTerms == 0)
{
if (prefixRef == null)
{
Assert.IsNull(MultiFields.GetTerms(r, "field"));
}
else
{
Terms terms = MultiFields.GetTerms(r, "field");
if (terms != null)
{
TermsEnum termsEnum = terms.GetIterator(null);
TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef);
if (result != TermsEnum.SeekStatus.END)
{
Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term, prefixRef), "term=" + termsEnum.Term.Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString());
}
else
{
// ok
}
}
else
{
// ok
}
}
return;
}
if (VERBOSE)
{
Console.WriteLine("TEST: TERMS:");
te.SeekExact(0);
while (true)
{
Console.WriteLine(" ord=" + te.Ord + " term=" + te.Term.Utf8ToString());
if (te.Next() == null)
{
break;
}
}
}
SortedSetDocValues iter = dto.GetIterator(r);
for (int docID = 0; docID < r.MaxDoc; docID++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")");
}
iter.SetDocument(docID);
int[] answers = idToOrds[docIDToID.Get(docID)];
int upto = 0;
long ord;
while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
te.SeekExact(ord);
BytesRef expected = termsArray[answers[upto++]];
if (VERBOSE)
{
Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString());
}
Assert.AreEqual(expected, te.Term, "expected=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString() + " ord=" + ord);
}
Assert.AreEqual(answers.Length, upto);
}
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"));
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1))
{
// Otherwise test can take way too long (> 2 hours)
numTerms /= 2;
}
if (VERBOSE)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
postingsList = CollectionsHelper.Shuffle(postingsList);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir("bagofpostings"));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int threadCount = TestUtil.NextInt(Random(), 1, 5);
if (VERBOSE)
{
Console.WriteLine("config: " + iw.w.Config);
Console.WriteLine("threadCount=" + threadCount);
}
ThreadClass[] threads = new ThreadClass[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, maxTermsPerDoc, postings, iw, startingGun);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.Reader;
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.Terms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, air.Fields.UniqueTermCount);
if (iwc.Codec is Lucene3xCodec == false)
{
Assert.AreEqual(numTerms - 1, terms.Size());
}
TermsEnum termsEnum = terms.Iterator(null);
BytesRef term_;
while ((term_ = termsEnum.Next()) != null)
{
int value = Convert.ToInt32(term_.Utf8ToString());
Assert.AreEqual(value, termsEnum.DocFreq());
// don't really need to check more than this, as CheckIndex
// will verify that docFreq == actual number of documents seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
public static void VerifyEquals(Fields d1, Fields d2)
{
if (d1 == null)
{
Assert.IsTrue(d2 == null || d2.Count == 0);
return;
}
Assert.IsTrue(d2 != null);
IEnumerator <string> fieldsEnum2 = d2.GetEnumerator();
foreach (string field1 in d1)
{
fieldsEnum2.MoveNext();
string field2 = fieldsEnum2.Current;
Assert.AreEqual(field1, field2);
Terms terms1 = d1.GetTerms(field1);
Assert.IsNotNull(terms1);
TermsEnum termsEnum1 = terms1.GetIterator(null);
Terms terms2 = d2.GetTerms(field2);
Assert.IsNotNull(terms2);
TermsEnum termsEnum2 = terms2.GetIterator(null);
DocsAndPositionsEnum dpEnum1 = null;
DocsAndPositionsEnum dpEnum2 = null;
DocsEnum dEnum1 = null;
DocsEnum dEnum2 = null;
BytesRef term1;
while ((term1 = termsEnum1.Next()) != null)
{
BytesRef term2 = termsEnum2.Next();
Assert.AreEqual(term1, term2);
Assert.AreEqual(termsEnum1.TotalTermFreq, termsEnum2.TotalTermFreq);
dpEnum1 = termsEnum1.DocsAndPositions(null, dpEnum1);
dpEnum2 = termsEnum2.DocsAndPositions(null, dpEnum2);
if (dpEnum1 != null)
{
Assert.IsNotNull(dpEnum2);
int docID1 = dpEnum1.NextDoc();
dpEnum2.NextDoc();
// docIDs are not supposed to be equal
//int docID2 = dpEnum2.NextDoc();
//Assert.AreEqual(docID1, docID2);
Assert.IsTrue(docID1 != DocIdSetIterator.NO_MORE_DOCS);
int freq1 = dpEnum1.Freq;
int freq2 = dpEnum2.Freq;
Assert.AreEqual(freq1, freq2);
IOffsetAttribute offsetAtt1 = dpEnum1.Attributes.HasAttribute <IOffsetAttribute>() ? dpEnum1.Attributes.GetAttribute <IOffsetAttribute>() : null;
IOffsetAttribute offsetAtt2 = dpEnum2.Attributes.HasAttribute <IOffsetAttribute>() ? dpEnum2.Attributes.GetAttribute <IOffsetAttribute>() : null;
if (offsetAtt1 != null)
{
Assert.IsNotNull(offsetAtt2);
}
else
{
Assert.IsNull(offsetAtt2);
}
for (int posUpto = 0; posUpto < freq1; posUpto++)
{
int pos1 = dpEnum1.NextPosition();
int pos2 = dpEnum2.NextPosition();
Assert.AreEqual(pos1, pos2);
if (offsetAtt1 != null)
{
Assert.AreEqual(offsetAtt1.StartOffset, offsetAtt2.StartOffset);
Assert.AreEqual(offsetAtt1.EndOffset, offsetAtt2.EndOffset);
}
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum1.NextDoc());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum2.NextDoc());
}
else
{
dEnum1 = TestUtil.Docs(Random(), termsEnum1, null, dEnum1, DocsFlags.FREQS);
dEnum2 = TestUtil.Docs(Random(), termsEnum2, null, dEnum2, DocsFlags.FREQS);
Assert.IsNotNull(dEnum1);
Assert.IsNotNull(dEnum2);
int docID1 = dEnum1.NextDoc();
dEnum2.NextDoc();
// docIDs are not supposed to be equal
//int docID2 = dEnum2.NextDoc();
//Assert.AreEqual(docID1, docID2);
Assert.IsTrue(docID1 != DocIdSetIterator.NO_MORE_DOCS);
int freq1 = dEnum1.Freq;
int freq2 = dEnum2.Freq;
Assert.AreEqual(freq1, freq2);
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dEnum1.NextDoc());
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dEnum2.NextDoc());
}
}
Assert.IsNull(termsEnum2.Next());
}
Assert.IsFalse(fieldsEnum2.MoveNext());
}
public virtual void VerifyEquals(DirectoryReader r1, DirectoryReader r2, string idField)
{
if (VERBOSE)
{
Console.WriteLine("\nr1 docs:");
PrintDocs(r1);
Console.WriteLine("\nr2 docs:");
PrintDocs(r2);
}
if (r1.NumDocs != r2.NumDocs)
{
Debug.Assert(false, "r1.NumDocs=" + r1.NumDocs + " vs r2.NumDocs=" + r2.NumDocs);
}
bool hasDeletes = !(r1.MaxDoc == r2.MaxDoc && r1.NumDocs == r1.MaxDoc);
int[] r2r1 = new int[r2.MaxDoc]; // r2 id to r1 id mapping
// create mapping from id2 space to id2 based on idField
Fields f1 = MultiFields.GetFields(r1);
if (f1 == null)
{
// make sure r2 is empty
Assert.IsNull(MultiFields.GetFields(r2));
return;
}
Terms terms1 = f1.GetTerms(idField);
if (terms1 == null)
{
Assert.IsTrue(MultiFields.GetFields(r2) == null || MultiFields.GetFields(r2).GetTerms(idField) == null);
return;
}
TermsEnum termsEnum = terms1.GetIterator(null);
IBits liveDocs1 = MultiFields.GetLiveDocs(r1);
IBits liveDocs2 = MultiFields.GetLiveDocs(r2);
Fields fields = MultiFields.GetFields(r2);
if (fields == null)
{
// make sure r1 is in fact empty (eg has only all
// deleted docs):
IBits liveDocs = MultiFields.GetLiveDocs(r1);
DocsEnum docs = null;
while (termsEnum.Next() != null)
{
docs = TestUtil.Docs(Random(), termsEnum, liveDocs, docs, DocsFlags.NONE);
while (docs.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
Assert.Fail("r1 is not empty but r2 is");
}
}
return;
}
Terms terms2 = fields.GetTerms(idField);
TermsEnum termsEnum2 = terms2.GetIterator(null);
DocsEnum termDocs1 = null;
DocsEnum termDocs2 = null;
while (true)
{
BytesRef term = termsEnum.Next();
//System.out.println("TEST: match id term=" + term);
if (term == null)
{
break;
}
termDocs1 = TestUtil.Docs(Random(), termsEnum, liveDocs1, termDocs1, DocsFlags.NONE);
if (termsEnum2.SeekExact(term))
{
termDocs2 = TestUtil.Docs(Random(), termsEnum2, liveDocs2, termDocs2, DocsFlags.NONE);
}
else
{
termDocs2 = null;
}
if (termDocs1.NextDoc() == DocIdSetIterator.NO_MORE_DOCS)
{
// this doc is deleted and wasn't replaced
Assert.IsTrue(termDocs2 == null || termDocs2.NextDoc() == DocIdSetIterator.NO_MORE_DOCS);
continue;
}
int id1 = termDocs1.DocID;
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, termDocs1.NextDoc());
Assert.IsTrue(termDocs2.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int id2 = termDocs2.DocID;
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, termDocs2.NextDoc());
r2r1[id2] = id1;
// verify stored fields are equivalent
try
{
VerifyEquals(r1.Document(id1), r2.Document(id2));
}
catch (Exception t)
{
Console.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2 + " term=" + term);
Console.WriteLine(" d1=" + r1.Document(id1));
Console.WriteLine(" d2=" + r2.Document(id2));
throw t;
}
try
{
// verify term vectors are equivalent
VerifyEquals(r1.GetTermVectors(id1), r2.GetTermVectors(id2));
}
catch (Exception e)
{
Console.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
Fields tv1 = r1.GetTermVectors(id1);
Console.WriteLine(" d1=" + tv1);
if (tv1 != null)
{
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
foreach (string field in tv1)
{
Console.WriteLine(" " + field + ":");
Terms terms3 = tv1.GetTerms(field);
Assert.IsNotNull(terms3);
TermsEnum termsEnum3 = terms3.GetIterator(null);
BytesRef term2;
while ((term2 = termsEnum3.Next()) != null)
{
Console.WriteLine(" " + term2.Utf8ToString() + ": freq=" + termsEnum3.TotalTermFreq);
dpEnum = termsEnum3.DocsAndPositions(null, dpEnum);
if (dpEnum != null)
{
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dpEnum.Freq;
Console.WriteLine(" doc=" + dpEnum.DocID + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++)
{
Console.WriteLine(" pos=" + dpEnum.NextPosition());
}
}
else
{
dEnum = TestUtil.Docs(Random(), termsEnum3, null, dEnum, DocsFlags.FREQS);
Assert.IsNotNull(dEnum);
Assert.IsTrue(dEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dEnum.Freq;
Console.WriteLine(" doc=" + dEnum.DocID + " freq=" + freq);
}
}
}
}
Fields tv2 = r2.GetTermVectors(id2);
Console.WriteLine(" d2=" + tv2);
if (tv2 != null)
{
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
foreach (string field in tv2)
{
Console.WriteLine(" " + field + ":");
Terms terms3 = tv2.GetTerms(field);
Assert.IsNotNull(terms3);
TermsEnum termsEnum3 = terms3.GetIterator(null);
BytesRef term2;
while ((term2 = termsEnum3.Next()) != null)
{
Console.WriteLine(" " + term2.Utf8ToString() + ": freq=" + termsEnum3.TotalTermFreq);
dpEnum = termsEnum3.DocsAndPositions(null, dpEnum);
if (dpEnum != null)
{
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dpEnum.Freq;
Console.WriteLine(" doc=" + dpEnum.DocID + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++)
{
Console.WriteLine(" pos=" + dpEnum.NextPosition());
}
}
else
{
dEnum = TestUtil.Docs(Random(), termsEnum3, null, dEnum, DocsFlags.FREQS);
Assert.IsNotNull(dEnum);
Assert.IsTrue(dEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = dEnum.Freq;
Console.WriteLine(" doc=" + dEnum.DocID + " freq=" + freq);
}
}
}
}
throw e;
}
}
//System.out.println("TEST: done match id");
// Verify postings
//System.out.println("TEST: create te1");
Fields fields1 = MultiFields.GetFields(r1);
IEnumerator <string> fields1Enum = fields1.GetEnumerator();
Fields fields2 = MultiFields.GetFields(r2);
IEnumerator <string> fields2Enum = fields2.GetEnumerator();
string field1 = null, field2 = null;
TermsEnum termsEnum1 = null;
termsEnum2 = null;
DocsEnum docs1 = null, docs2 = null;
// pack both doc and freq into single element for easy sorting
long[] info1 = new long[r1.NumDocs];
long[] info2 = new long[r2.NumDocs];
for (; ;)
{
BytesRef term1 = null, term2 = null;
// iterate until we get some docs
int len1;
for (; ;)
{
len1 = 0;
if (termsEnum1 == null)
{
if (!fields1Enum.MoveNext())
{
break;
}
field1 = fields1Enum.Current;
Terms terms = fields1.GetTerms(field1);
if (terms == null)
{
continue;
}
termsEnum1 = terms.GetIterator(null);
}
term1 = termsEnum1.Next();
if (term1 == null)
{
// no more terms in this field
termsEnum1 = null;
continue;
}
//System.out.println("TEST: term1=" + term1);
docs1 = TestUtil.Docs(Random(), termsEnum1, liveDocs1, docs1, DocsFlags.FREQS);
while (docs1.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
int d = docs1.DocID;
int f = docs1.Freq;
info1[len1] = (((long)d) << 32) | (uint)f;
len1++;
}
if (len1 > 0)
{
break;
}
}
// iterate until we get some docs
int len2;
for (; ;)
{
len2 = 0;
if (termsEnum2 == null)
{
if (!fields2Enum.MoveNext())
{
break;
}
field2 = fields2Enum.Current;
Terms terms = fields2.GetTerms(field2);
if (terms == null)
{
continue;
}
termsEnum2 = terms.GetIterator(null);
}
term2 = termsEnum2.Next();
if (term2 == null)
{
// no more terms in this field
termsEnum2 = null;
continue;
}
//System.out.println("TEST: term1=" + term1);
docs2 = TestUtil.Docs(Random(), termsEnum2, liveDocs2, docs2, DocsFlags.FREQS);
while (docs2.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
int d = r2r1[docs2.DocID];
int f = docs2.Freq;
info2[len2] = (((long)d) << 32) | (uint)f;
len2++;
}
if (len2 > 0)
{
break;
}
}
Assert.AreEqual(len1, len2);
if (len1 == 0) // no more terms
{
break;
}
Assert.AreEqual(field1, field2);
Assert.IsTrue(term1.BytesEquals(term2));
if (!hasDeletes)
{
Assert.AreEqual(termsEnum1.DocFreq, termsEnum2.DocFreq);
}
Assert.AreEqual(term1, term2, "len1=" + len1 + " len2=" + len2 + " deletes?=" + hasDeletes);
// sort info2 to get it into ascending docid
Array.Sort(info2, 0, len2);
// now compare
for (int i = 0; i < len1; i++)
{
Assert.AreEqual(info1[i], info2[i], "i=" + i + " len=" + len1 + " d1=" + ((long)((ulong)info1[i] >> 32)) + " f1=" + (info1[i] & int.MaxValue) + " d2=" + ((long)((ulong)info2[i] >> 32)) + " f2=" + (info2[i] & int.MaxValue) + " field=" + field1 + " term=" + term1.Utf8ToString());
}
}
}
public TermsEnum TermsEnum(Terms terms)
{
return terms.Iterator(null);
}
public TermsEnum TermsEnum(Terms terms)
{
return NumericUtils.FilterPrefixCodedLongs(terms.Iterator(null));
}
/// <summary>
/// checks collection-level statistics on Terms
/// </summary>
public virtual void AssertTermsStatistics(Terms leftTerms, Terms rightTerms)
{
Debug.Assert(leftTerms.Comparator == rightTerms.Comparator);
if (leftTerms.DocCount != -1 && rightTerms.DocCount != -1)
{
Assert.AreEqual(leftTerms.DocCount, rightTerms.DocCount);
}
if (leftTerms.SumDocFreq != -1 && rightTerms.SumDocFreq != -1)
{
Assert.AreEqual(leftTerms.SumDocFreq, rightTerms.SumDocFreq);
}
if (leftTerms.SumTotalTermFreq != -1 && rightTerms.SumTotalTermFreq != -1)
{
Assert.AreEqual(leftTerms.SumTotalTermFreq, rightTerms.SumTotalTermFreq);
}
if (leftTerms.Size() != -1 && rightTerms.Size() != -1)
{
Assert.AreEqual(leftTerms.Size(), rightTerms.Size());
}
}
public FloatDocValuesAnonymousInnerClassHelper(TFValueSource outerInstance, TFValueSource @this, Terms terms, TFIDFSimilarity similarity)
: base(@this)
{
this.outerInstance = outerInstance;
this.terms = terms;
this.similarity = similarity;
lastDocRequested = -1;
Reset();
}
public virtual void TestDoubleOffsetCounting()
{
Directory dir = NewDirectory();
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Document doc = new Document();
FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
Field f = NewField("field", "abcd", customType);
doc.Add(f);
doc.Add(f);
Field f2 = NewField("field", "", customType);
doc.Add(f2);
doc.Add(f);
w.AddDocument(doc);
w.Dispose();
IndexReader r = DirectoryReader.Open(dir);
Terms vector = r.GetTermVectors(0).GetTerms("field");
Assert.IsNotNull(vector);
TermsEnum termsEnum = vector.GetIterator(null);
Assert.IsNotNull(termsEnum.Next());
Assert.AreEqual("", termsEnum.Term.Utf8ToString());
// Token "" occurred once
Assert.AreEqual(1, termsEnum.TotalTermFreq);
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(8, dpEnum.StartOffset);
Assert.AreEqual(8, dpEnum.EndOffset);
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
// Token "abcd" occurred three times
Assert.AreEqual(new BytesRef("abcd"), termsEnum.Next());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.AreEqual(3, termsEnum.TotalTermFreq);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
dpEnum.NextPosition();
Assert.AreEqual(0, dpEnum.StartOffset);
Assert.AreEqual(4, dpEnum.EndOffset);
dpEnum.NextPosition();
Assert.AreEqual(4, dpEnum.StartOffset);
Assert.AreEqual(8, dpEnum.EndOffset);
dpEnum.NextPosition();
Assert.AreEqual(8, dpEnum.StartOffset);
Assert.AreEqual(12, dpEnum.EndOffset);
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
Assert.IsNull(termsEnum.Next());
r.Dispose();
dir.Dispose();
}
public override void Run()
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": launch search thread");
}
while (Environment.TickCount < stopTimeMS)
{
try
{
IndexSearcher s = outerInstance.GetCurrentSearcher();
try
{
// Verify 1) IW is correctly setting
// diagnostics, and 2) segment warming for
// merged segments is actually happening:
foreach (AtomicReaderContext sub in s.IndexReader.Leaves)
{
SegmentReader segReader = (SegmentReader)sub.Reader;
IDictionary <string, string> diagnostics = segReader.SegmentInfo.Info.Diagnostics;
assertNotNull(diagnostics);
string source;
diagnostics.TryGetValue("source", out source);
assertNotNull(source);
if (source.Equals("merge", StringComparison.Ordinal))
{
assertTrue("sub reader " + sub + " wasn't warmed: warmed=" + outerInstance.warmed + " diagnostics=" + diagnostics + " si=" + segReader.SegmentInfo,
!outerInstance.m_assertMergedSegmentsWarmed || outerInstance.warmed.ContainsKey(segReader.core));
}
}
if (s.IndexReader.NumDocs > 0)
{
outerInstance.SmokeTestSearcher(s);
Fields fields = MultiFields.GetFields(s.IndexReader);
if (fields == null)
{
continue;
}
Terms terms = fields.GetTerms("body");
if (terms == null)
{
continue;
}
TermsEnum termsEnum = terms.GetIterator(null);
int seenTermCount = 0;
int shift;
int trigger;
if (totTermCount.Get() < 30)
{
shift = 0;
trigger = 1;
}
else
{
trigger = totTermCount.Get() / 30;
shift = Random.Next(trigger);
}
while (Environment.TickCount < stopTimeMS)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
totTermCount.Set(seenTermCount);
break;
}
seenTermCount++;
// search 30 terms
if ((seenTermCount + shift) % trigger == 0)
{
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.Utf8ToString());
//}
totHits.AddAndGet(outerInstance.RunQuery(s, new TermQuery(new Term("body", term))));
}
}
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + ": search done");
//}
}
}
finally
{
outerInstance.ReleaseSearcher(s);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
outerInstance.m_failed.Set(true);
Console.WriteLine(t.ToString());
throw new Exception(t.ToString(), t);
}
}
}
protected virtual void AssertEquals(RandomTokenStream tk, FieldType ft, Terms terms)
{
Assert.AreEqual(1, terms.DocCount);
int termCount = (new HashSet <string>(Arrays.AsList(tk.terms))).Count;
Assert.AreEqual(termCount, terms.Count);
Assert.AreEqual(termCount, terms.SumDocFreq);
Assert.AreEqual(ft.StoreTermVectorPositions, terms.HasPositions);
Assert.AreEqual(ft.StoreTermVectorOffsets, terms.HasOffsets);
Assert.AreEqual(ft.StoreTermVectorPayloads && tk.HasPayloads(), terms.HasPayloads);
HashSet <BytesRef> uniqueTerms = new HashSet <BytesRef>();
foreach (string term in tk.freqs.Keys)
{
uniqueTerms.Add(new BytesRef(term));
}
BytesRef[] sortedTerms = uniqueTerms.ToArray(/*new BytesRef[0]*/);
Array.Sort(sortedTerms, terms.Comparer);
TermsEnum termsEnum = terms.GetIterator(Random.NextBoolean() ? null : this.termsEnum.Value);
this.termsEnum.Value = termsEnum;
for (int i = 0; i < sortedTerms.Length; ++i)
{
BytesRef nextTerm = termsEnum.Next();
Assert.AreEqual(sortedTerms[i], nextTerm);
Assert.AreEqual(sortedTerms[i], termsEnum.Term);
Assert.AreEqual(1, termsEnum.DocFreq);
FixedBitSet bits = new FixedBitSet(1);
DocsEnum docsEnum = termsEnum.Docs(bits, Random.NextBoolean() ? null : this.docsEnum.Value);
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
bits.Set(0);
docsEnum = termsEnum.Docs(Random.NextBoolean() ? bits : null, Random.NextBoolean() ? null : docsEnum);
Assert.IsNotNull(docsEnum);
Assert.AreEqual(0, docsEnum.NextDoc());
Assert.AreEqual(0, docsEnum.DocID);
Assert.AreEqual(tk.freqs[termsEnum.Term.Utf8ToString()], (int?)docsEnum.Freq);
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
this.docsEnum.Value = docsEnum;
bits.Clear(0);
DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.DocsAndPositions(bits, Random.NextBoolean() ? null : this.docsAndPositionsEnum.Value);
Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
if (docsAndPositionsEnum != null)
{
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
}
bits.Set(0);
docsAndPositionsEnum = termsEnum.DocsAndPositions(Random.NextBoolean() ? bits : null, Random.NextBoolean() ? null : docsAndPositionsEnum);
Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
if (terms.HasPositions || terms.HasOffsets)
{
Assert.AreEqual(0, docsAndPositionsEnum.NextDoc());
int freq = docsAndPositionsEnum.Freq;
Assert.AreEqual(tk.freqs[termsEnum.Term.Utf8ToString()], (int?)freq);
if (docsAndPositionsEnum != null)
{
for (int k = 0; k < freq; ++k)
{
int position = docsAndPositionsEnum.NextPosition();
ISet <int?> indexes;
if (terms.HasPositions)
{
indexes = tk.positionToTerms[position];
Assert.IsNotNull(indexes);
}
else
{
indexes = tk.startOffsetToTerms[docsAndPositionsEnum.StartOffset];
Assert.IsNotNull(indexes);
}
if (terms.HasPositions)
{
bool foundPosition = false;
foreach (int index in indexes)
{
if (tk.termBytes[index].Equals(termsEnum.Term) && tk.positions[index] == position)
{
foundPosition = true;
break;
}
}
Assert.IsTrue(foundPosition);
}
if (terms.HasOffsets)
{
bool foundOffset = false;
foreach (int index in indexes)
{
if (tk.termBytes[index].Equals(termsEnum.Term) && tk.startOffsets[index] == docsAndPositionsEnum.StartOffset && tk.endOffsets[index] == docsAndPositionsEnum.EndOffset)
{
foundOffset = true;
break;
}
}
Assert.IsTrue(foundOffset);
}
if (terms.HasPayloads)
{
bool foundPayload = false;
foreach (int index in indexes)
{
if (tk.termBytes[index].Equals(termsEnum.Term) && Equals(tk.payloads[index], docsAndPositionsEnum.GetPayload()))
{
foundPayload = true;
break;
}
}
Assert.IsTrue(foundPayload);
}
}
try
{
docsAndPositionsEnum.NextPosition();
Assert.Fail();
}
#pragma warning disable 168
catch (Exception e)
#pragma warning restore 168
{
// ok
}
}
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
}
this.docsAndPositionsEnum.Value = docsAndPositionsEnum;
}
Assert.IsNull(termsEnum.Next());
for (int i = 0; i < 5; ++i)
{
if (Random.NextBoolean())
{
Assert.IsTrue(termsEnum.SeekExact(RandomPicks.RandomFrom(Random, tk.termBytes)));
}
else
{
Assert.AreEqual(SeekStatus.FOUND, termsEnum.SeekCeil(RandomPicks.RandomFrom(Random, tk.termBytes)));
}
}
}
internal TestTerms(Terms @in)
: base(@in)
{
}
// Delete by Term
private long ApplyTermDeletes(IEnumerable <Term> termsIter, ReadersAndUpdates rld, SegmentReader reader)
{
lock (this)
{
long delCount = 0;
Fields fields = reader.Fields;
if (fields == null)
{
// this reader has no postings
return(0);
}
TermsEnum termsEnum = null;
string currentField = null;
DocsEnum docs = null;
Debug.Assert(CheckDeleteTerm(null));
bool any = false;
//System.out.println(Thread.currentThread().getName() + " del terms reader=" + reader);
foreach (Term term in termsIter)
{
// Since we visit terms sorted, we gain performance
// by re-using the same TermsEnum and seeking only
// forwards
if (!string.Equals(term.Field, currentField, StringComparison.Ordinal))
{
Debug.Assert(currentField == null || currentField.CompareToOrdinal(term.Field) < 0);
currentField = term.Field;
Terms terms = fields.GetTerms(currentField);
if (terms != null)
{
termsEnum = terms.GetIterator(termsEnum);
}
else
{
termsEnum = null;
}
}
if (termsEnum == null)
{
continue;
}
Debug.Assert(CheckDeleteTerm(term));
// System.out.println(" term=" + term);
if (termsEnum.SeekExact(term.Bytes))
{
// we don't need term frequencies for this
DocsEnum docsEnum = termsEnum.Docs(rld.LiveDocs, docs, DocsFlags.NONE);
//System.out.println("BDS: got docsEnum=" + docsEnum);
if (docsEnum != null)
{
while (true)
{
int docID = docsEnum.NextDoc();
//System.out.println(Thread.currentThread().getName() + " del term=" + term + " doc=" + docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (!any)
{
rld.InitWritableLiveDocs();
any = true;
}
// NOTE: there is no limit check on the docID
// when deleting by Term (unlike by Query)
// because on flush we apply all Term deletes to
// each segment. So all Term deleting here is
// against prior segments:
if (rld.Delete(docID))
{
delCount++;
}
}
}
}
}
return(delCount);
}
}
public AutomatonTermsEnumAnonymousInnerClassHelper(Terms outerInstance, Lucene.Net.Index.TermsEnum iterator, CompiledAutomaton compiled, BytesRef startTerm)
: base(iterator, compiled)
{
this.OuterInstance = outerInstance;
this.StartTerm = startTerm;
}
// DocValues updates
private void ApplyDocValuesUpdates <T1>(IEnumerable <T1> updates, ReadersAndUpdates rld, SegmentReader reader, DocValuesFieldUpdates.Container dvUpdatesContainer) where T1 : DocValuesUpdate
{
lock (this)
{
Fields fields = reader.Fields;
if (fields == null)
{
// this reader has no postings
return;
}
// TODO: we can process the updates per DV field, from last to first so that
// if multiple terms affect same document for the same field, we add an update
// only once (that of the last term). To do that, we can keep a bitset which
// marks which documents have already been updated. So e.g. if term T1
// updates doc 7, and then we process term T2 and it updates doc 7 as well,
// we don't apply the update since we know T1 came last and therefore wins
// the update.
// We can also use that bitset as 'liveDocs' to pass to TermEnum.docs(), so
// that these documents aren't even returned.
string currentField = null;
TermsEnum termsEnum = null;
DocsEnum docs = null;
//System.out.println(Thread.currentThread().getName() + " numericDVUpdate reader=" + reader);
foreach (DocValuesUpdate update in updates)
{
Term term = update.term;
int limit = update.docIDUpto;
// TODO: we traverse the terms in update order (not term order) so that we
// apply the updates in the correct order, i.e. if two terms udpate the
// same document, the last one that came in wins, irrespective of the
// terms lexical order.
// we can apply the updates in terms order if we keep an updatesGen (and
// increment it with every update) and attach it to each NumericUpdate. Note
// that we cannot rely only on docIDUpto because an app may send two updates
// which will get same docIDUpto, yet will still need to respect the order
// those updates arrived.
if (!string.Equals(term.Field, currentField, StringComparison.Ordinal))
{
// if we change the code to process updates in terms order, enable this assert
// assert currentField == null || currentField.CompareToOrdinal(term.Field) < 0;
currentField = term.Field;
Terms terms = fields.GetTerms(currentField);
if (terms != null)
{
termsEnum = terms.GetIterator(termsEnum);
}
else
{
termsEnum = null;
continue; // no terms in that field
}
}
if (termsEnum == null)
{
continue;
}
// System.out.println(" term=" + term);
if (termsEnum.SeekExact(term.Bytes))
{
// we don't need term frequencies for this
DocsEnum docsEnum = termsEnum.Docs(rld.LiveDocs, docs, DocsFlags.NONE);
//System.out.println("BDS: got docsEnum=" + docsEnum);
DocValuesFieldUpdates dvUpdates = dvUpdatesContainer.GetUpdates(update.field, update.type);
if (dvUpdates == null)
{
dvUpdates = dvUpdatesContainer.NewUpdates(update.field, update.type, reader.MaxDoc);
}
int doc;
while ((doc = docsEnum.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
//System.out.println(Thread.currentThread().getName() + " numericDVUpdate term=" + term + " doc=" + docID);
if (doc >= limit)
{
break; // no more docs that can be updated for this term
}
dvUpdates.Add(doc, update.value);
}
}
}
}
}
public virtual void SearchIndex(Directory dir, string oldName)
{
//QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
//Query query = parser.parse("handle:1");
IndexReader reader = DirectoryReader.Open(dir);
IndexSearcher searcher = NewSearcher(reader);
TestUtil.CheckIndex(dir);
// true if this is a 4.0+ index
bool is40Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("content5") != null;
// true if this is a 4.2+ index
bool is42Index = MultiFields.GetMergedFieldInfos(reader).FieldInfo("dvSortedSet") != null;
Debug.Assert(is40Index); // NOTE: currently we can only do this on trunk!
IBits liveDocs = MultiFields.GetLiveDocs(reader);
for (int i = 0; i < 35; i++)
{
if (liveDocs.Get(i))
{
Document d = reader.Document(i);
IList <IIndexableField> fields = d.Fields;
bool isProxDoc = d.GetField("content3") == null;
if (isProxDoc)
{
int numFields = is40Index ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
IIndexableField f = d.GetField("id");
Assert.AreEqual("" + i, f.GetStringValue());
f = d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue());
f = d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.GetStringValue());
f = d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.GetStringValue());
f = d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.GetStringValue());
}
Fields tfvFields = reader.GetTermVectors(i);
Assert.IsNotNull(tfvFields, "i=" + i);
Terms tfv = tfvFields.GetTerms("utf8");
Assert.IsNotNull(tfv, "docID=" + i + " index=" + oldName);
}
else
{
// Only ID 7 is deleted
Assert.AreEqual(7, i);
}
}
if (is40Index)
{
// check docvalues fields
NumericDocValues dvByte = MultiDocValues.GetNumericValues(reader, "dvByte");
BinaryDocValues dvBytesDerefFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefFixed");
BinaryDocValues dvBytesDerefVar = MultiDocValues.GetBinaryValues(reader, "dvBytesDerefVar");
SortedDocValues dvBytesSortedFixed = MultiDocValues.GetSortedValues(reader, "dvBytesSortedFixed");
SortedDocValues dvBytesSortedVar = MultiDocValues.GetSortedValues(reader, "dvBytesSortedVar");
BinaryDocValues dvBytesStraightFixed = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightFixed");
BinaryDocValues dvBytesStraightVar = MultiDocValues.GetBinaryValues(reader, "dvBytesStraightVar");
NumericDocValues dvDouble = MultiDocValues.GetNumericValues(reader, "dvDouble");
NumericDocValues dvFloat = MultiDocValues.GetNumericValues(reader, "dvFloat");
NumericDocValues dvInt = MultiDocValues.GetNumericValues(reader, "dvInt");
NumericDocValues dvLong = MultiDocValues.GetNumericValues(reader, "dvLong");
NumericDocValues dvPacked = MultiDocValues.GetNumericValues(reader, "dvPacked");
NumericDocValues dvShort = MultiDocValues.GetNumericValues(reader, "dvShort");
SortedSetDocValues dvSortedSet = null;
if (is42Index)
{
dvSortedSet = MultiDocValues.GetSortedSetValues(reader, "dvSortedSet");
}
for (int i = 0; i < 35; i++)
{
int id = Convert.ToInt32(reader.Document(i).Get("id"));
Assert.AreEqual(id, dvByte.Get(i));
sbyte[] bytes = new sbyte[] { (sbyte)((int)((uint)id >> 24)), (sbyte)((int)((uint)id >> 16)), (sbyte)((int)((uint)id >> 8)), (sbyte)id };
BytesRef expectedRef = new BytesRef((byte[])(Array)bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesDerefVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesSortedVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightFixed.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
dvBytesStraightVar.Get(i, scratch);
Assert.AreEqual(expectedRef, scratch);
Assert.AreEqual((double)id, J2N.BitConversion.Int64BitsToDouble(dvDouble.Get(i)), 0D);
Assert.AreEqual((float)id, J2N.BitConversion.Int32BitsToSingle((int)dvFloat.Get(i)), 0F);
Assert.AreEqual(id, dvInt.Get(i));
Assert.AreEqual(id, dvLong.Get(i));
Assert.AreEqual(id, dvPacked.Get(i));
Assert.AreEqual(id, dvShort.Get(i));
if (is42Index)
{
dvSortedSet.SetDocument(i);
long ord = dvSortedSet.NextOrd();
Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.NextOrd());
dvSortedSet.LookupOrd(ord, scratch);
Assert.AreEqual(expectedRef, scratch);
}
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #0
Document doc = searcher.IndexReader.Document(hits[0].Doc);
assertEquals("didn't get the right document first", "0", doc.Get("id"));
DoTestHits(hits, 34, searcher.IndexReader);
if (is40Index)
{
hits = searcher.Search(new TermQuery(new Term("content5", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
hits = searcher.Search(new TermQuery(new Term("content6", "aaa")), null, 1000).ScoreDocs;
DoTestHits(hits, 34, searcher.IndexReader);
}
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
reader.Dispose();
}
/// <summary>
/// Call this only once (if you subclass!) </summary>
protected virtual void Uninvert(AtomicReader reader, IBits liveDocs, BytesRef termPrefix)
{
FieldInfo info = reader.FieldInfos.FieldInfo(m_field);
if (info != null && info.HasDocValues)
{
throw new InvalidOperationException("Type mismatch: " + m_field + " was indexed as " + info.DocValuesType);
}
//System.out.println("DTO uninvert field=" + field + " prefix=" + termPrefix);
long startTime = Environment.TickCount;
m_prefix = termPrefix == null ? null : BytesRef.DeepCopyOf(termPrefix);
int maxDoc = reader.MaxDoc;
int[] index = new int[maxDoc]; // immediate term numbers, or the index into the byte[] representing the last number
int[] lastTerm = new int[maxDoc]; // last term we saw for this document
var bytes = new sbyte[maxDoc][]; // list of term numbers for the doc (delta encoded vInts)
Fields fields = reader.Fields;
if (fields == null)
{
// No terms
return;
}
Terms terms = fields.GetTerms(m_field);
if (terms == null)
{
// No terms
return;
}
TermsEnum te = terms.GetIterator(null);
BytesRef seekStart = termPrefix != null ? termPrefix : new BytesRef();
//System.out.println("seekStart=" + seekStart.utf8ToString());
if (te.SeekCeil(seekStart) == TermsEnum.SeekStatus.END)
{
// No terms match
return;
}
// If we need our "term index wrapper", these will be
// init'd below:
IList <BytesRef> indexedTerms = null;
PagedBytes indexedTermsBytes = null;
bool testedOrd = false;
// we need a minimum of 9 bytes, but round up to 12 since the space would
// be wasted with most allocators anyway.
var tempArr = new sbyte[12];
//
// enumerate all terms, and build an intermediate form of the un-inverted field.
//
// During this intermediate form, every document has a (potential) byte[]
// and the int[maxDoc()] array either contains the termNumber list directly
// or the *end* offset of the termNumber list in it's byte array (for faster
// appending and faster creation of the final form).
//
// idea... if things are too large while building, we could do a range of docs
// at a time (but it would be a fair amount slower to build)
// could also do ranges in parallel to take advantage of multiple CPUs
// OPTIONAL: remap the largest df terms to the lowest 128 (single byte)
// values. this requires going over the field first to find the most
// frequent terms ahead of time.
int termNum = 0;
m_docsEnum = null;
// Loop begins with te positioned to first term (we call
// seek above):
for (; ;)
{
BytesRef t = te.Term;
if (t == null || (termPrefix != null && !StringHelper.StartsWith(t, termPrefix)))
{
break;
}
//System.out.println("visit term=" + t.utf8ToString() + " " + t + " termNum=" + termNum);
if (!testedOrd)
{
try
{
m_ordBase = (int)te.Ord;
//System.out.println("got ordBase=" + ordBase);
}
#pragma warning disable 168
catch (NotSupportedException uoe)
#pragma warning restore 168
{
// Reader cannot provide ord support, so we wrap
// our own support by creating our own terms index:
indexedTerms = new List <BytesRef>();
indexedTermsBytes = new PagedBytes(15);
//System.out.println("NO ORDS");
}
testedOrd = true;
}
VisitTerm(te, termNum);
if (indexedTerms != null && (termNum & indexIntervalMask) == 0)
{
// Index this term
m_sizeOfIndexedStrings += t.Length;
BytesRef indexedTerm = new BytesRef();
indexedTermsBytes.Copy(t, indexedTerm);
// TODO: really should 1) strip off useless suffix,
// and 2) use FST not array/PagedBytes
indexedTerms.Add(indexedTerm);
}
int df = te.DocFreq;
if (df <= m_maxTermDocFreq)
{
m_docsEnum = te.Docs(liveDocs, m_docsEnum, DocsFlags.NONE);
// dF, but takes deletions into account
int actualDF = 0;
for (; ;)
{
int doc = m_docsEnum.NextDoc();
if (doc == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
//System.out.println(" chunk=" + chunk + " docs");
actualDF++;
m_termInstances++;
//System.out.println(" docID=" + doc);
// add TNUM_OFFSET to the term number to make room for special reserved values:
// 0 (end term) and 1 (index into byte array follows)
int delta = termNum - lastTerm[doc] + TNUM_OFFSET;
lastTerm[doc] = termNum;
int val = index[doc];
if ((val & 0xff) == 1)
{
// index into byte array (actually the end of
// the doc-specific byte[] when building)
int pos = (int)((uint)val >> 8);
int ilen = VInt32Size(delta);
var arr = bytes[doc];
int newend = pos + ilen;
if (newend > arr.Length)
{
// We avoid a doubling strategy to lower memory usage.
// this faceting method isn't for docs with many terms.
// In hotspot, objects have 2 words of overhead, then fields, rounded up to a 64-bit boundary.
// TODO: figure out what array lengths we can round up to w/o actually using more memory
// (how much space does a byte[] take up? Is data preceded by a 32 bit length only?
// It should be safe to round up to the nearest 32 bits in any case.
int newLen = (newend + 3) & unchecked ((int)0xfffffffc); // 4 byte alignment
var newarr = new sbyte[newLen];
Array.Copy(arr, 0, newarr, 0, pos);
arr = newarr;
bytes[doc] = newarr;
}
pos = WriteInt32(delta, arr, pos);
index[doc] = (pos << 8) | 1; // update pointer to end index in byte[]
}
else
{
// OK, this int has data in it... find the end (a zero starting byte - not
// part of another number, hence not following a byte with the high bit set).
int ipos;
if (val == 0)
{
ipos = 0;
}
else if ((val & 0x0000ff80) == 0)
{
ipos = 1;
}
else if ((val & 0x00ff8000) == 0)
{
ipos = 2;
}
else if ((val & 0xff800000) == 0)
{
ipos = 3;
}
else
{
ipos = 4;
}
//System.out.println(" ipos=" + ipos);
int endPos = WriteInt32(delta, tempArr, ipos);
//System.out.println(" endpos=" + endPos);
if (endPos <= 4)
{
//System.out.println(" fits!");
// value will fit in the integer... move bytes back
for (int j = ipos; j < endPos; j++)
{
val |= (tempArr[j] & 0xff) << (j << 3);
}
index[doc] = val;
}
else
{
// value won't fit... move integer into byte[]
for (int j = 0; j < ipos; j++)
{
tempArr[j] = (sbyte)val;
val = (int)((uint)val >> 8);
}
// point at the end index in the byte[]
index[doc] = (endPos << 8) | 1;
bytes[doc] = tempArr;
tempArr = new sbyte[12];
}
}
}
SetActualDocFreq(termNum, actualDF);
}
termNum++;
if (te.Next() == null)
{
break;
}
}
m_numTermsInField = termNum;
long midPoint = Environment.TickCount;
if (m_termInstances == 0)
{
// we didn't invert anything
// lower memory consumption.
m_tnums = null;
}
else
{
this.m_index = index;
//
// transform intermediate form into the final form, building a single byte[]
// at a time, and releasing the intermediate byte[]s as we go to avoid
// increasing the memory footprint.
//
for (int pass = 0; pass < 256; pass++)
{
var target = m_tnums[pass];
var pos = 0; // end in target;
if (target != null)
{
pos = target.Length;
}
else
{
target = new sbyte[4096];
}
// loop over documents, 0x00ppxxxx, 0x01ppxxxx, 0x02ppxxxx
// where pp is the pass (which array we are building), and xx is all values.
// each pass shares the same byte[] for termNumber lists.
for (int docbase = pass << 16; docbase < maxDoc; docbase += (1 << 24))
{
int lim = Math.Min(docbase + (1 << 16), maxDoc);
for (int doc = docbase; doc < lim; doc++)
{
//System.out.println(" pass="******" process docID=" + doc);
int val = index[doc];
if ((val & 0xff) == 1)
{
int len = (int)((uint)val >> 8);
//System.out.println(" ptr pos=" + pos);
index[doc] = (pos << 8) | 1; // change index to point to start of array
if ((pos & 0xff000000) != 0)
{
// we only have 24 bits for the array index
throw new InvalidOperationException("Too many values for UnInvertedField faceting on field " + m_field);
}
var arr = bytes[doc];
/*
* for(byte b : arr) {
* //System.out.println(" b=" + Integer.toHexString((int) b));
* }
*/
bytes[doc] = null; // IMPORTANT: allow GC to avoid OOM
if (target.Length <= pos + len)
{
int newlen = target.Length;
//* we don't have to worry about the array getting too large
// since the "pos" param will overflow first (only 24 bits available)
// if ((newlen<<1) <= 0) {
// // overflow...
// newlen = Integer.MAX_VALUE;
// if (newlen <= pos + len) {
// throw new SolrException(400,"Too many terms to uninvert field!");
// }
// } else {
// while (newlen <= pos + len) newlen<<=1; // doubling strategy
// }
//
while (newlen <= pos + len) // doubling strategy
{
newlen <<= 1;
}
var newtarget = new sbyte[newlen];
Array.Copy(target, 0, newtarget, 0, pos);
target = newtarget;
}
Array.Copy(arr, 0, target, pos, len);
pos += len + 1; // skip single byte at end and leave it 0 for terminator
}
}
}
// shrink array
if (pos < target.Length)
{
var newtarget = new sbyte[pos];
Array.Copy(target, 0, newtarget, 0, pos);
target = newtarget;
}
m_tnums[pass] = target;
if ((pass << 16) > maxDoc)
{
break;
}
}
}
if (indexedTerms != null)
{
m_indexedTermsArray = new BytesRef[indexedTerms.Count];
indexedTerms.CopyTo(m_indexedTermsArray, 0);
}
long endTime = Environment.TickCount;
m_total_time = (int)(endTime - startTime);
m_phase1_time = (int)(midPoint - startTime);
}
public AssertingTerms(Terms @in)
: base(@in)
{
}
public virtual void TestIntersectEmptyString()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergePolicy(new LogDocMergePolicy());
RandomIndexWriter w = new RandomIndexWriter(Random, dir, iwc);
Document doc = new Document();
doc.Add(NewStringField("field", "", Field.Store.NO));
doc.Add(NewStringField("field", "abc", Field.Store.NO));
w.AddDocument(doc);
doc = new Document();
// add empty string to both documents, so that singletonDocID == -1.
// For a FST-based term dict, we'll expect to see the first arc is
// flaged with HAS_FINAL_OUTPUT
doc.Add(NewStringField("field", "abc", Field.Store.NO));
doc.Add(NewStringField("field", "", Field.Store.NO));
w.AddDocument(doc);
w.ForceMerge(1);
DirectoryReader r = w.GetReader();
w.Dispose();
AtomicReader sub = GetOnlySegmentReader(r);
Terms terms = sub.Fields.GetTerms("field");
Automaton automaton = (new RegExp(".*", RegExpSyntax.NONE)).ToAutomaton(); // accept ALL
CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false);
TermsEnum te = terms.Intersect(ca, null);
DocsEnum de;
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("", te.Term.Utf8ToString());
de = te.Docs(null, null, DocsFlags.NONE);
Assert.AreEqual(0, de.NextDoc());
Assert.AreEqual(1, de.NextDoc());
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("abc", te.Term.Utf8ToString());
de = te.Docs(null, null, DocsFlags.NONE);
Assert.AreEqual(0, de.NextDoc());
Assert.AreEqual(1, de.NextDoc());
Assert.IsFalse(te.MoveNext());
// pass empty string
te = terms.Intersect(ca, new BytesRef(""));
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("abc", te.Term.Utf8ToString());
de = te.Docs(null, null, DocsFlags.NONE);
Assert.AreEqual(0, de.NextDoc());
Assert.AreEqual(1, de.NextDoc());
Assert.IsFalse(te.MoveNext());
r.Dispose();
dir.Dispose();
}
/// <summary>
/// Creates a new FilterTerms </summary>
/// <param name="in"> the underlying Terms instance. </param>
public FilterTerms(Terms @in)
{
this.@in = @in;
}
public virtual void TestIntersectStartTerm()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMergePolicy(new LogDocMergePolicy());
RandomIndexWriter w = new RandomIndexWriter(Random, dir, iwc);
Document doc = new Document();
doc.Add(NewStringField("field", "abc", Field.Store.NO));
w.AddDocument(doc);
doc = new Document();
doc.Add(NewStringField("field", "abd", Field.Store.NO));
w.AddDocument(doc);
doc = new Document();
doc.Add(NewStringField("field", "acd", Field.Store.NO));
w.AddDocument(doc);
doc = new Document();
doc.Add(NewStringField("field", "bcd", Field.Store.NO));
w.AddDocument(doc);
w.ForceMerge(1);
DirectoryReader r = w.GetReader();
w.Dispose();
AtomicReader sub = GetOnlySegmentReader(r);
Terms terms = sub.Fields.GetTerms("field");
Automaton automaton = (new RegExp(".*d", RegExpSyntax.NONE)).ToAutomaton();
CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false);
TermsEnum te;
// should seek to startTerm
te = terms.Intersect(ca, new BytesRef("aad"));
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("abd", te.Term.Utf8ToString());
Assert.AreEqual(1, te.Docs(null, null, DocsFlags.NONE).NextDoc());
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("acd", te.Term.Utf8ToString());
Assert.AreEqual(2, te.Docs(null, null, DocsFlags.NONE).NextDoc());
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("bcd", te.Term.Utf8ToString());
Assert.AreEqual(3, te.Docs(null, null, DocsFlags.NONE).NextDoc());
Assert.IsFalse(te.MoveNext());
// should fail to find ceil label on second arc, rewind
te = terms.Intersect(ca, new BytesRef("add"));
Assert.IsTrue(te.MoveNext());
Assert.AreEqual("bcd", te.Term.Utf8ToString());
Assert.AreEqual(3, te.Docs(null, null, DocsFlags.NONE).NextDoc());
Assert.IsFalse(te.MoveNext());
// should reach end
te = terms.Intersect(ca, new BytesRef("bcd"));
Assert.IsFalse(te.MoveNext());
te = terms.Intersect(ca, new BytesRef("ddd"));
Assert.IsFalse(te.MoveNext());
r.Dispose();
dir.Dispose();
}
private void AssertTermsSeeking(Terms leftTerms, Terms rightTerms)
{
TermsEnum leftEnum = null;
TermsEnum rightEnum = null;
// just an upper bound
int numTests = AtLeast(20);
Random random = Random();
// collect this number of terms from the left side
HashSet<BytesRef> tests = new HashSet<BytesRef>();
int numPasses = 0;
while (numPasses < 10 && tests.Count < numTests)
{
leftEnum = leftTerms.Iterator(leftEnum);
BytesRef term = null;
while ((term = leftEnum.Next()) != null)
{
int code = random.Next(10);
if (code == 0)
{
// the term
tests.Add(BytesRef.DeepCopyOf(term));
}
else if (code == 1)
{
// truncated subsequence of term
term = BytesRef.DeepCopyOf(term);
if (term.Length > 0)
{
// truncate it
term.Length = random.Next(term.Length);
}
}
else if (code == 2)
{
// term, but ensure a non-zero offset
var newbytes = new byte[term.Length + 5];
Array.Copy(term.Bytes, term.Offset, newbytes, 5, term.Length);
tests.Add(new BytesRef(newbytes, 5, term.Length));
}
}
numPasses++;
}
List<BytesRef> shuffledTests = new List<BytesRef>(tests);
shuffledTests = (List<BytesRef>)CollectionsHelper.Shuffle(shuffledTests);
foreach (BytesRef b in shuffledTests)
{
leftEnum = leftTerms.Iterator(leftEnum);
rightEnum = rightTerms.Iterator(rightEnum);
Assert.AreEqual(leftEnum.SeekExact(b), rightEnum.SeekExact(b));
Assert.AreEqual(leftEnum.SeekExact(b), rightEnum.SeekExact(b));
SeekStatus leftStatus;
SeekStatus rightStatus;
leftStatus = leftEnum.SeekCeil(b);
rightStatus = rightEnum.SeekCeil(b);
Assert.AreEqual(leftStatus, rightStatus);
if (leftStatus != SeekStatus.END)
{
Assert.AreEqual(leftEnum.Term(), rightEnum.Term());
}
leftStatus = leftEnum.SeekCeil(b);
rightStatus = rightEnum.SeekCeil(b);
Assert.AreEqual(leftStatus, rightStatus);
if (leftStatus != SeekStatus.END)
{
Assert.AreEqual(leftEnum.Term(), rightEnum.Term());
}
}
}
internal void AddField(string fieldName, Terms terms)
{
fields[fieldName] = terms;
}
protected internal virtual void AssertEquals(RandomTokenStream tk, FieldType ft, Terms terms)
{
Assert.AreEqual(1, terms.DocCount);
int termCount = (new HashSet<string>(Arrays.AsList(tk.Terms))).Count;
Assert.AreEqual(termCount, terms.Size());
Assert.AreEqual(termCount, terms.SumDocFreq);
Assert.AreEqual(ft.StoreTermVectorPositions, terms.HasPositions());
Assert.AreEqual(ft.StoreTermVectorOffsets, terms.HasOffsets());
Assert.AreEqual(ft.StoreTermVectorPayloads && tk.HasPayloads(), terms.HasPayloads());
HashSet<BytesRef> uniqueTerms = new HashSet<BytesRef>();
foreach (string term in tk.Freqs.Keys)
{
uniqueTerms.Add(new BytesRef(term));
}
BytesRef[] sortedTerms = uniqueTerms.ToArray(/*new BytesRef[0]*/);
Array.Sort(sortedTerms, terms.Comparator);
TermsEnum termsEnum = terms.Iterator(Random().NextBoolean() ? null : this.termsEnum.Value);
this.termsEnum.Value = termsEnum;
for (int i = 0; i < sortedTerms.Length; ++i)
{
BytesRef nextTerm = termsEnum.Next();
Assert.AreEqual(sortedTerms[i], nextTerm);
Assert.AreEqual(sortedTerms[i], termsEnum.Term());
Assert.AreEqual(1, termsEnum.DocFreq());
FixedBitSet bits = new FixedBitSet(1);
DocsEnum docsEnum = termsEnum.Docs(bits, Random().NextBoolean() ? null : this.docsEnum.Value);
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
bits.Set(0);
docsEnum = termsEnum.Docs(Random().NextBoolean() ? bits : null, Random().NextBoolean() ? null : docsEnum);
Assert.IsNotNull(docsEnum);
Assert.AreEqual(0, docsEnum.NextDoc());
Assert.AreEqual(0, docsEnum.DocID());
Assert.AreEqual(tk.Freqs[termsEnum.Term().Utf8ToString()], (int?)docsEnum.Freq());
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
this.docsEnum.Value = docsEnum;
bits.Clear(0);
DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.DocsAndPositions(bits, Random().NextBoolean() ? null : this.docsAndPositionsEnum.Value);
Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
if (docsAndPositionsEnum != null)
{
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
}
bits.Set(0);
docsAndPositionsEnum = termsEnum.DocsAndPositions(Random().NextBoolean() ? bits : null, Random().NextBoolean() ? null : docsAndPositionsEnum);
Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
if (terms.HasPositions() || terms.HasOffsets())
{
Assert.AreEqual(0, docsAndPositionsEnum.NextDoc());
int freq = docsAndPositionsEnum.Freq();
Assert.AreEqual(tk.Freqs[termsEnum.Term().Utf8ToString()], (int?)freq);
if (docsAndPositionsEnum != null)
{
for (int k = 0; k < freq; ++k)
{
int position = docsAndPositionsEnum.NextPosition();
ISet<int?> indexes;
if (terms.HasPositions())
{
indexes = tk.PositionToTerms[position];
Assert.IsNotNull(indexes);
}
else
{
indexes = tk.StartOffsetToTerms[docsAndPositionsEnum.StartOffset()];
Assert.IsNotNull(indexes);
}
if (terms.HasPositions())
{
bool foundPosition = false;
foreach (int index in indexes)
{
if (tk.TermBytes[index].Equals(termsEnum.Term()) && tk.Positions[index] == position)
{
foundPosition = true;
break;
}
}
Assert.IsTrue(foundPosition);
}
if (terms.HasOffsets())
{
bool foundOffset = false;
foreach (int index in indexes)
{
if (tk.TermBytes[index].Equals(termsEnum.Term()) && tk.StartOffsets[index] == docsAndPositionsEnum.StartOffset() && tk.EndOffsets[index] == docsAndPositionsEnum.EndOffset())
{
foundOffset = true;
break;
}
}
Assert.IsTrue(foundOffset);
}
if (terms.HasPayloads())
{
bool foundPayload = false;
foreach (int index in indexes)
{
if (tk.TermBytes[index].Equals(termsEnum.Term()) && Equals(tk.Payloads[index], docsAndPositionsEnum.Payload))
{
foundPayload = true;
break;
}
}
Assert.IsTrue(foundPayload);
}
}
try
{
docsAndPositionsEnum.NextPosition();
Assert.Fail();
}
catch (Exception e)
{
// ok
}
}
Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
}
this.docsAndPositionsEnum.Value = docsAndPositionsEnum;
}
Assert.IsNull(termsEnum.Next());
for (int i = 0; i < 5; ++i)
{
if (Random().NextBoolean())
{
Assert.IsTrue(termsEnum.SeekExact(RandomInts.RandomFrom(Random(), tk.TermBytes)));
}
else
{
Assert.AreEqual(SeekStatus.FOUND, termsEnum.SeekCeil(RandomInts.RandomFrom(Random(), tk.TermBytes)));
}
}
}
public virtual void Test()
{
IList <string> postingsList = new List <string>();
int numTerms = AtLeast(300);
int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20);
bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field"));
IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1))
{
// Otherwise test can take way too long (> 2 hours)
numTerms /= 2;
}
if (VERBOSE)
{
Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc);
Console.WriteLine("numTerms=" + numTerms);
}
for (int i = 0; i < numTerms; i++)
{
string term = Convert.ToString(i);
for (int j = 0; j < i; j++)
{
postingsList.Add(term);
}
}
Collections.Shuffle(postingsList);
ConcurrentQueue <string> postings = new ConcurrentQueue <string>(postingsList);
Directory dir = NewFSDirectory(CreateTempDir(GetFullMethodName()));
RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);
int threadCount = TestUtil.NextInt(Random(), 1, 5);
if (VERBOSE)
{
Console.WriteLine("config: " + iw.w.Config);
Console.WriteLine("threadCount=" + threadCount);
}
Field prototype = NewTextField("field", "", Field.Store.NO);
FieldType fieldType = new FieldType(prototype.FieldType);
if (Random().NextBoolean())
{
fieldType.OmitNorms = true;
}
int options = Random().Next(3);
if (options == 0)
{
fieldType.IndexOptions = IndexOptions.DOCS_AND_FREQS; // we dont actually need positions
fieldType.StoreTermVectors = true; // but enforce term vectors when we do this so we check SOMETHING
}
else if (options == 1 && !DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat("field")))
{
fieldType.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
// else just positions
ThreadClass[] threads = new ThreadClass[threadCount];
CountdownEvent startingGun = new CountdownEvent(1);
for (int threadID = 0; threadID < threadCount; threadID++)
{
Random threadRandom = new Random(Random().Next());
Document document = new Document();
Field field = new Field("field", "", fieldType);
document.Add(field);
threads[threadID] = new ThreadAnonymousInnerClassHelper(this, numTerms, maxTermsPerDoc, postings, iw, startingGun, threadRandom, document, field);
threads[threadID].Start();
}
startingGun.Signal();
foreach (ThreadClass t in threads)
{
t.Join();
}
iw.ForceMerge(1);
DirectoryReader ir = iw.Reader;
Assert.AreEqual(1, ir.Leaves.Count);
AtomicReader air = (AtomicReader)ir.Leaves[0].Reader;
Terms terms = air.GetTerms("field");
// numTerms-1 because there cannot be a term 0 with 0 postings:
Assert.AreEqual(numTerms - 1, terms.Count);
TermsEnum termsEnum = terms.GetIterator(null);
BytesRef termBR;
while ((termBR = termsEnum.Next()) != null)
{
int value = Convert.ToInt32(termBR.Utf8ToString());
Assert.AreEqual(value, termsEnum.TotalTermFreq);
// don't really need to check more than this, as CheckIndex
// will verify that totalTermFreq == total number of positions seen
// from a docsAndPositionsEnum.
}
ir.Dispose();
iw.Dispose();
dir.Dispose();
}
/// <summary>
/// Terms api equivalency
/// </summary>
public void AssertTermsEquals(string info, IndexReader leftReader, Terms leftTerms, Terms rightTerms, bool deep)
{
if (leftTerms == null || rightTerms == null)
{
Assert.IsNull(leftTerms, info);
Assert.IsNull(rightTerms, info);
return;
}
AssertTermsStatisticsEquals(info, leftTerms, rightTerms);
Assert.AreEqual(leftTerms.HasOffsets(), rightTerms.HasOffsets());
Assert.AreEqual(leftTerms.HasPositions(), rightTerms.HasPositions());
Assert.AreEqual(leftTerms.HasPayloads(), rightTerms.HasPayloads());
TermsEnum leftTermsEnum = leftTerms.Iterator(null);
TermsEnum rightTermsEnum = rightTerms.Iterator(null);
AssertTermsEnumEquals(info, leftReader, leftTermsEnum, rightTermsEnum, true);
AssertTermsSeekingEquals(info, leftTerms, rightTerms);
if (deep)
{
int numIntersections = AtLeast(3);
for (int i = 0; i < numIntersections; i++)
{
string re = AutomatonTestUtil.RandomRegexp(Random());
CompiledAutomaton automaton = new CompiledAutomaton((new RegExp(re, RegExp.NONE)).ToAutomaton());
if (automaton.Type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL)
{
// TODO: test start term too
TermsEnum leftIntersection = leftTerms.Intersect(automaton, null);
TermsEnum rightIntersection = rightTerms.Intersect(automaton, null);
AssertTermsEnumEquals(info, leftReader, leftIntersection, rightIntersection, Rarely());
}
}
}
}
public virtual void TestPositionReader()
{
TermVectorsReader reader = Codec.Default.TermVectorsFormat().VectorsReader(Dir, Seg.Info, FieldInfos, NewIOContext(Random()));
BytesRef[] terms;
Terms vector = reader.Get(0).Terms(TestFields[0]);
Assert.IsNotNull(vector);
Assert.AreEqual(TestTerms.Length, vector.Size());
TermsEnum termsEnum = vector.Iterator(null);
DocsAndPositionsEnum dpEnum = null;
for (int i = 0; i < TestTerms.Length; i++)
{
BytesRef text = termsEnum.Next();
Assert.IsNotNull(text);
string term = text.Utf8ToString();
//System.out.println("Term: " + term);
Assert.AreEqual(TestTerms[i], term);
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsNotNull(dpEnum);
int doc = dpEnum.DocID();
Assert.AreEqual(-1, doc);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(dpEnum.Freq(), Positions[i].Length);
for (int j = 0; j < Positions[i].Length; j++)
{
Assert.AreEqual(Positions[i][j], dpEnum.NextPosition());
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
doc = dpEnum.DocID();
Assert.AreEqual(-1, doc);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.IsNotNull(dpEnum);
Assert.AreEqual(dpEnum.Freq(), Positions[i].Length);
for (int j = 0; j < Positions[i].Length; j++)
{
Assert.AreEqual(Positions[i][j], dpEnum.NextPosition());
Assert.AreEqual(j * 10, dpEnum.StartOffset());
Assert.AreEqual(j * 10 + TestTerms[i].Length, dpEnum.EndOffset());
}
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, dpEnum.NextDoc());
}
Terms freqVector = reader.Get(0).Terms(TestFields[1]); //no pos, no offset
Assert.IsNotNull(freqVector);
Assert.AreEqual(TestTerms.Length, freqVector.Size());
termsEnum = freqVector.Iterator(null);
Assert.IsNotNull(termsEnum);
for (int i = 0; i < TestTerms.Length; i++)
{
BytesRef text = termsEnum.Next();
Assert.IsNotNull(text);
string term = text.Utf8ToString();
//System.out.println("Term: " + term);
Assert.AreEqual(TestTerms[i], term);
Assert.IsNotNull(termsEnum.Docs(null, null));
Assert.IsNull(termsEnum.DocsAndPositions(null, null)); // no pos
}
reader.Dispose();
}
private void AssertTermsSeekingEquals(string info, Terms leftTerms, Terms rightTerms)
{
TermsEnum leftEnum = null;
TermsEnum rightEnum = null;
// just an upper bound
int numTests = AtLeast(20);
Random random = Random();
// collect this number of terms from the left side
HashSet<BytesRef> tests = new HashSet<BytesRef>();
int numPasses = 0;
while (numPasses < 10 && tests.Count < numTests)
{
leftEnum = leftTerms.Iterator(leftEnum);
BytesRef term = null;
while ((term = leftEnum.Next()) != null)
{
int code = random.Next(10);
if (code == 0)
{
// the term
tests.Add(BytesRef.DeepCopyOf(term));
}
else if (code == 1)
{
// truncated subsequence of term
term = BytesRef.DeepCopyOf(term);
if (term.Length > 0)
{
// truncate it
term.Length = random.Next(term.Length);
}
}
else if (code == 2)
{
// term, but ensure a non-zero offset
var newbytes = new byte[term.Length + 5];
Array.Copy(term.Bytes, term.Offset, newbytes, 5, term.Length);
tests.Add(new BytesRef(newbytes, 5, term.Length));
}
else if (code == 3)
{
switch (Random().Next(3))
{
case 0:
tests.Add(new BytesRef()); // before the first term
break;
case 1:
tests.Add(new BytesRef(new byte[] { unchecked((byte)0xFF), unchecked((byte)0xFF) })); // past the last term
break;
case 2:
tests.Add(new BytesRef(TestUtil.RandomSimpleString(Random()))); // random term
break;
default:
throw new InvalidOperationException();
}
}
}
numPasses++;
}
rightEnum = rightTerms.Iterator(rightEnum);
IList<BytesRef> shuffledTests = new List<BytesRef>(tests);
shuffledTests = CollectionsHelper.Shuffle(shuffledTests);
foreach (BytesRef b in shuffledTests)
{
if (Rarely())
{
// reuse the enums
leftEnum = leftTerms.Iterator(leftEnum);
rightEnum = rightTerms.Iterator(rightEnum);
}
bool seekExact = Random().NextBoolean();
if (seekExact)
{
Assert.AreEqual(leftEnum.SeekExact(b), rightEnum.SeekExact(b), info);
}
else
{
TermsEnum.SeekStatus leftStatus = leftEnum.SeekCeil(b);
TermsEnum.SeekStatus rightStatus = rightEnum.SeekCeil(b);
Assert.AreEqual(leftStatus, rightStatus, info);
if (leftStatus != TermsEnum.SeekStatus.END)
{
Assert.AreEqual(leftEnum.Term(), rightEnum.Term(), info);
AssertTermStatsEquals(info, leftEnum, rightEnum);
}
}
}
}
public AutomatonTermsEnumAnonymousInnerClassHelper(Terms outerInstance, Lucene.Net.Index.TermsEnum iterator, CompiledAutomaton compiled, BytesRef startTerm)
: base(iterator, compiled)
{
this.outerInstance = outerInstance;
this.startTerm = startTerm;
}