/// <summary>
/// Detect mis-use, where provided parent query in fact sometimes returns child documents.
/// </summary>
private void ValidateParentDoc()
{
if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc))
{
throw new InvalidOperationException(InvalidQueryMessage + _parentDoc);
}
}
protected override void Swap(int i, int j)
{
long tmpDoc = docs.Get(j);
docs.Set(j, docs.Get(i));
docs.Set(i, tmpDoc);
long tmpVal = values.Get(j);
values.Set(j, values.Get(i));
values.Set(i, tmpVal);
bool tmpBool = docsWithField.Get(j);
if (docsWithField.Get(i))
{
docsWithField.Set(j);
}
else
{
docsWithField.Clear(j);
}
if (tmpBool)
{
docsWithField.Set(i);
}
else
{
docsWithField.Clear(i);
}
}
protected internal override void Swap(int i, int j)
{
long tmpDoc = Docs.Get(j);
Docs.Set(j, Docs.Get(i));
Docs.Set(i, tmpDoc);
long tmpVal = Values.Get(j);
Values.Set(j, Values.Get(i));
Values.Set(i, tmpVal);
bool tmpBool = DocsWithField.Get(j);
if (DocsWithField.Get(i))
{
DocsWithField.Set(j);
}
else
{
DocsWithField.Clear(j);
}
if (tmpBool)
{
DocsWithField.Set(i);
}
else
{
DocsWithField.Clear(i);
}
}
/// <summary>
/// Detect mis-use, where provided parent query in fact sometimes returns child documents.
/// </summary>
private void ValidateParentDoc()
{
if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc))
{
throw IllegalStateException.Create(INVALID_QUERY_MESSAGE + _parentDoc);
}
}
/// <summary>
/// Detect mis-use, where provided parent query in fact sometimes returns child documents.
/// </summary>
private void ValidateParentDoc()
{
if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc))
{
throw new InvalidOperationException(INVALID_QUERY_MESSAGE + _parentDoc);
}
}
internal IBits RandomLiveDocs(int maxDoc)
{
if (Rarely())
{
if (Random.nextBoolean())
{
return(null);
}
else
{
return(new Bits.MatchNoBits(maxDoc));
}
}
FixedBitSet bits = new FixedBitSet(maxDoc);
int bitsSet = TestUtil.NextInt32(Random, 1, maxDoc - 1);
for (int i = 0; i < bitsSet; ++i)
{
while (true)
{
int index = Random.nextInt(maxDoc);
if (!bits.Get(index))
{
bits.Set(index);
break;
}
}
}
return(bits);
}
/// <summary>
/// Useful from an assert. </summary>
internal virtual bool IsConsistent(int maxDoc)
{
FixedBitSet targets = new FixedBitSet(maxDoc);
for (int i = 0; i < maxDoc; ++i)
{
int target = Map(i);
if (target < 0 || target >= maxDoc)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, "out of range: {0} not in [0-{1}[", target, maxDoc);
}
return(false);
}
else if (targets.Get(target))
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, "{0} is already taken ({1})", target, i);
}
return(false);
}
}
return(true);
}
protected override void FillDocsAndScores(FixedBitSet matchingDocs, IBits acceptDocs,
TermsEnum termsEnum)
{
BytesRef spare = new BytesRef();
DocsEnum docsEnum = null;
for (int i = 0; i < m_outerInstance._terms.Count; i++)
{
if (termsEnum.SeekExact(m_outerInstance._terms.Get(m_outerInstance._ords[i], spare)))
{
docsEnum = termsEnum.Docs(acceptDocs, docsEnum, DocsFlags.NONE);
float score = m_outerInstance._scores[m_outerInstance._ords[i]];
for (int doc = docsEnum.NextDoc();
doc != NO_MORE_DOCS;
doc = docsEnum.NextDoc())
{
// I prefer this:
/*if (scores[doc] < score) {
* scores[doc] = score;
* matchingDocs.set(doc);
* }*/
// But this behaves the same as MVInnerScorer and only then the tests will pass:
if (!matchingDocs.Get(doc))
{
scores[doc] = score;
matchingDocs.Set(doc);
}
}
}
}
}
private bool OpenBitSetContains(int[] expectedDocs, FixedBitSet actual, int maxDoc)
{
if (expectedDocs.Length != actual.Cardinality())
{
return(false);
}
FixedBitSet expected = new FixedBitSet(maxDoc);
foreach (int expectedDoc in expectedDocs)
{
expected.Set(expectedDoc);
}
int docId;
DocIdSetIterator iterator = expected.GetIterator();
while ((docId = iterator.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS)
{
if (!actual.Get(docId))
{
return(false);
}
}
return(true);
}
public int NextDoc()
{
if (Idx >= Size)
{
Offset = -1;
return(Doc_Renamed = DocIdSetIterator.NO_MORE_DOCS);
}
Doc_Renamed = (int)Docs.Get(Idx);
++Idx;
while (Idx < Size && Docs.Get(Idx) == Doc_Renamed)
{
++Idx;
}
// idx points to the "next" element
long prevIdx = Idx - 1;
if (!DocsWithField.Get((int)prevIdx))
{
Offset = -1;
}
else
{
// cannot change 'value' here because nextDoc is called before the
// value is used, and it's a waste to clone the BytesRef when we
// obtain the value
Offset = (int)Offsets.Get(prevIdx);
Length = (int)Lengths.Get(prevIdx);
}
return(Doc_Renamed);
}
private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam)
{
// Use yield return instead of ucsom IEnumerable
AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = Lengths.GetIterator();
int size = (int)Lengths.Size();
DataInput bytesIterator = Bytes.DataInput;
int maxDoc = maxDocParam;
int upto = 0;
while (upto < maxDoc)
{
BytesRef v = null;
if (upto < size)
{
int length = (int)lengthsIterator.Next();
var value = new BytesRef();
value.Grow(length);
value.Length = length;
bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length);
if (DocsWithField.Get(upto))
{
v = value;
}
}
upto++;
yield return(v);
}
}
/// <summary>
/// Useful from an assert. </summary>
internal virtual bool IsConsistent(int maxDoc)
{
FixedBitSet targets = new FixedBitSet(maxDoc);
for (int i = 0; i < maxDoc; ++i)
{
int target = Map(i);
if (target < 0 || target >= maxDoc)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, () => "out of range: " + target + " not in [0-" + maxDoc + "[");
}
return(false);
}
else if (targets.Get(target))
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(false, () => target + " is already taken (" + i + ")");
}
return(false);
}
}
return(true);
}
public override int NextDoc()
{
if (idx >= size)
{
offset = -1;
return(doc = DocIdSetIterator.NO_MORE_DOCS);
}
doc = (int)docs.Get(idx);
++idx;
while (idx < size && docs.Get(idx) == doc)
{
++idx;
}
// idx points to the "next" element
long prevIdx = idx - 1;
if (!docsWithField.Get((int)prevIdx))
{
offset = -1;
}
else
{
// cannot change 'value' here because nextDoc is called before the
// value is used, and it's a waste to clone the BytesRef when we
// obtain the value
offset = (int)offsets.Get(prevIdx);
length = (int)lengths.Get(prevIdx);
}
return(doc);
}
private IEnumerable <long?> GetNumericIterator(int maxDoc)
{
// LUCENENET specific: using yield return instead of custom iterator type. Much less code.
AbstractAppendingInt64Buffer.Iterator iter = pending.GetIterator();
int size = (int)pending.Count;
int upto = 0;
while (upto < maxDoc)
{
long?value;
if (upto < size)
{
var v = iter.Next();
if (docsWithField == null || docsWithField.Get(upto))
{
value = v;
}
else
{
value = null;
}
}
else
{
value = docsWithField != null ? (long?)null : MISSING;
}
upto++;
// TODO: make reusable Number
yield return(value);
}
}
public void testEquals()
{
// This test can't handle numBits==0:
int numBits = rnd.Next(2000) + 1;
FixedBitSet b1 = new FixedBitSet(numBits);
FixedBitSet b2 = new FixedBitSet(numBits);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
for (int iter = 0; iter < 10 * rnd.Next(500); iter++)
{
int idx = rnd.Next(numBits);
if (!b1.Get(idx))
{
b1.Set(idx);
Assert.IsFalse(b1.Equals(b2));
Assert.IsFalse(b2.Equals(b1));
b2.Set(idx);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
}
}
// try different type of object
Assert.IsFalse(b1.Equals(new Object()));
}
private ContainsResult MayContainValue(int positiveHash)
{
Debug.Assert((positiveHash >= 0));
// Bloom sizes are always base 2 and so can be ANDed for a fast modulo
var pos = positiveHash & _bloomSize;
return(_filter.Get(pos) ? ContainsResult.MAYBE : ContainsResult.NO);
}
void doGet(BitArray a, FixedBitSet b)
{
int max = b.Length();
for (int i = 0; i < max; i++)
{
if (a.Get(i) != b.Get(i))
{
Assert.Fail("mismatch: BitSet=[" + i + "]=" + a.Get(i));
}
}
}
void doGet(BitArray a, FixedBitSet b)
{
int max = b.Length();
for (int i = 0; i < max; i++)
{
if (a.Get(i) != b.Get(i))
{
Assert.Fail("mismatch: BitSet=[" + i + "]=" + a.Get(i));
}
}
}
public virtual void TestBuildDocMap()
{
int maxDoc = TestUtil.NextInt(Random(), 1, 128);
int numDocs = TestUtil.NextInt(Random(), 0, maxDoc);
int numDeletedDocs = maxDoc - numDocs;
FixedBitSet liveDocs = new FixedBitSet(maxDoc);
for (int i = 0; i < numDocs; ++i)
{
while (true)
{
int docID = Random().Next(maxDoc);
if (!liveDocs.Get(docID))
{
liveDocs.Set(docID);
break;
}
}
}
MergeState.DocMap docMap = MergeState.DocMap.Build(maxDoc, liveDocs);
Assert.AreEqual(maxDoc, docMap.MaxDoc);
Assert.AreEqual(numDocs, docMap.NumDocs);
Assert.AreEqual(numDeletedDocs, docMap.NumDeletedDocs);
// assert the mapping is compact
for (int i = 0, del = 0; i < maxDoc; ++i)
{
if (!liveDocs.Get(i))
{
Assert.AreEqual(-1, docMap.Get(i));
++del;
}
else
{
Assert.AreEqual(i - del, docMap.Get(i));
}
}
}
protected internal override sealed bool MatchDoc(int doc)
{
int ord = fcsi.GetOrd(doc);
if (ord == -1)
{
// missing
return(false);
}
else
{
return(bits.Get(ord));
}
}
protected override void Swap(int i, int j)
{
long tmpDoc = docs.Get(j);
docs.Set(j, docs.Get(i));
docs.Set(i, tmpDoc);
long tmpOffset = offsets.Get(j);
offsets.Set(j, offsets.Get(i));
offsets.Set(i, tmpOffset);
long tmpLength = lengths.Get(j);
lengths.Set(j, lengths.Get(i));
lengths.Set(i, tmpLength);
bool tmpBool = docsWithField.Get(j);
if (docsWithField.Get(i))
{
docsWithField.Set(j);
}
else
{
docsWithField.Clear(j);
}
if (tmpBool)
{
docsWithField.Set(i);
}
else
{
docsWithField.Clear(i);
}
}
protected override void Swap(int i, int j)
{
long tmpDoc = Docs.Get(j);
Docs.Set(j, Docs.Get(i));
Docs.Set(i, tmpDoc);
long tmpOffset = Offsets.Get(j);
Offsets.Set(j, Offsets.Get(i));
Offsets.Set(i, tmpOffset);
long tmpLength = Lengths.Get(j);
Lengths.Set(j, Lengths.Get(i));
Lengths.Set(i, tmpLength);
bool tmpBool = DocsWithField.Get(j);
if (DocsWithField.Get(i))
{
DocsWithField.Set(j);
}
else
{
DocsWithField.Clear(j);
}
if (tmpBool)
{
DocsWithField.Set(i);
}
else
{
DocsWithField.Clear(i);
}
}
/// <summary>
/// pp was just advanced. If that caused a repeater collision, resolve by advancing the lesser
/// of the two colliding pps. Note that there can only be one collision, as by the initialization
/// there were no collisions before pp was advanced.
/// </summary>
private bool AdvanceRpts(PhrasePositions pp)
{
if (pp.rptGroup < 0)
{
return(true); // not a repeater
}
PhrasePositions[] rg = rptGroups[pp.rptGroup];
FixedBitSet bits = new FixedBitSet(rg.Length); // for re-queuing after collisions are resolved
int k0 = pp.rptInd;
int k;
while ((k = Collide(pp)) >= 0)
{
pp = Lesser(pp, rg[k]); // always advance the lesser of the (only) two colliding pps
if (!AdvancePP(pp))
{
return(false); // exhausted
}
if (k != k0) // careful: mark only those currently in the queue
{
bits = FixedBitSet.EnsureCapacity(bits, k);
bits.Set(k); // mark that pp2 need to be re-queued
}
}
// collisions resolved, now re-queue
// empty (partially) the queue until seeing all pps advanced for resolving collisions
int n = 0;
// TODO would be good if we can avoid calling cardinality() in each iteration!
int numBits = bits.Length; // larges bit we set
while (bits.Cardinality() > 0)
{
PhrasePositions pp2 = pq.Pop();
rptStack[n++] = pp2;
if (pp2.rptGroup >= 0 && pp2.rptInd < numBits && bits.Get(pp2.rptInd)) // this bit may not have been set
{
bits.Clear(pp2.rptInd);
}
}
// add back to queue
for (int i = n - 1; i >= 0; i--)
{
pq.Add(rptStack[i]);
}
return(true);
}
private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam)
{
// Use yield return instead of ucsom IEnumerable
var value = new BytesRef();
AppendingDeltaPackedInt64Buffer.Iterator lengthsIterator = lengths.GetIterator();
int size = (int)lengths.Count;
DataInput bytesIterator = bytes.GetDataInput();
int maxDoc = maxDocParam;
int upto = 0;
while (upto < maxDoc)
{
BytesRef v = null;
if (upto < size)
{
int length = (int)lengthsIterator.Next();
value.Grow(length);
value.Length = length;
try
{
bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length);
}
catch (Exception ioe) when(ioe.IsIOException())
{
// Should never happen!
throw RuntimeException.Create(ioe);
}
if (docsWithField.Get(upto))
{
v = value;
}
}
upto++;
yield return(v);
}
}
private BigSegmentedArray GetCollapsedCounts()
{
if (m_collapsedCounts == null)
{
m_collapsedCounts = new LazyBigInt32Array(m_bucketValues.Count);
FacetDataCache dataCache = m_subCollector.DataCache;
ITermValueList subList = dataCache.ValArray;
BigSegmentedArray subcounts = m_subCollector.Count;
FixedBitSet indexSet = new FixedBitSet(subcounts.Length);
int c = 0;
int i = 0;
foreach (string val in m_bucketValues)
{
if (val.Length > 0)
{
string[] subVals = m_predefinedBuckets.Get(val);
int count = 0;
foreach (string subVal in subVals)
{
int index = subList.IndexOf(subVal);
if (index > 0)
{
int subcount = subcounts.Get(index);
count += subcount;
if (!indexSet.Get(index))
{
indexSet.Set(index);
c += dataCache.Freqs[index];
}
}
}
m_collapsedCounts.Add(i, count);
}
i++;
}
m_collapsedCounts.Add(0, (m_numdocs - c));
}
return(m_collapsedCounts);
}
public void testHashCodeEquals()
{
// This test can't handle numBits==0:
int numBits = rnd.Next(2000) + 1;
FixedBitSet b1 = new FixedBitSet(numBits);
FixedBitSet b2 = new FixedBitSet(numBits);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
for (int iter = 0; iter < 10 * rnd.Next(500); iter++)
{
int idx = rnd.Next(numBits);
if (!b1.Get(idx))
{
b1.Set(idx);
Assert.IsFalse(b1.Equals(b2));
Assert.AreNotEqual(b1.GetHashCode(), b2.GetHashCode());
b2.Set(idx);
Assert.AreEqual(b1, b2);
Assert.AreEqual(b1.GetHashCode(), b2.GetHashCode());
}
}
}
public override int NextDoc()
{
if (idx >= size)
{
value = null;
return(doc = DocIdSetIterator.NO_MORE_DOCS);
}
doc = (int)docs.Get(idx);
++idx;
while (idx < size && docs.Get(idx) == doc)
{
++idx;
}
if (!docsWithField.Get((int)(idx - 1)))
{
value = null;
}
else
{
// idx points to the "next" element
value = Convert.ToInt64(values.Get(idx - 1));
}
return(doc);
}
public int NextDoc()
{
if (Idx >= Size)
{
Value_Renamed = null;
return(Doc_Renamed = DocIdSetIterator.NO_MORE_DOCS);
}
Doc_Renamed = (int)Docs.Get(Idx);
++Idx;
while (Idx < Size && Docs.Get(Idx) == Doc_Renamed)
{
++Idx;
}
if (!DocsWithField.Get((int)(Idx - 1)))
{
Value_Renamed = null;
}
else
{
// idx points to the "next" element
Value_Renamed = Convert.ToInt64(Values.Get(Idx - 1));
}
return(Doc_Renamed);
}
public virtual void TestBuildDocMap()
{
int maxDoc = TestUtil.NextInt(Random(), 1, 128);
int numDocs = TestUtil.NextInt(Random(), 0, maxDoc);
int numDeletedDocs = maxDoc - numDocs;
FixedBitSet liveDocs = new FixedBitSet(maxDoc);
for (int i = 0; i < numDocs; ++i)
{
while (true)
{
int docID = Random().Next(maxDoc);
if (!liveDocs.Get(docID))
{
liveDocs.Set(docID);
break;
}
}
}
MergeState.DocMap docMap = MergeState.DocMap.Build(maxDoc, liveDocs);
Assert.AreEqual(maxDoc, docMap.MaxDoc);
Assert.AreEqual(numDocs, docMap.NumDocs);
Assert.AreEqual(numDeletedDocs, docMap.NumDeletedDocs);
// assert the mapping is compact
for (int i = 0, del = 0; i < maxDoc; ++i)
{
if (!liveDocs.Get(i))
{
Assert.AreEqual(-1, docMap.Get(i));
++del;
}
else
{
Assert.AreEqual(i - del, docMap.Get(i));
}
}
}
public void testEquals()
{
// This test can't handle numBits==0:
int numBits = rnd.Next(2000) + 1;
FixedBitSet b1 = new FixedBitSet(numBits);
FixedBitSet b2 = new FixedBitSet(numBits);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
for (int iter = 0; iter < 10 * rnd.Next(500); iter++)
{
int idx = rnd.Next(numBits);
if (!b1.Get(idx))
{
b1.Set(idx);
Assert.IsFalse(b1.Equals(b2));
Assert.IsFalse(b2.Equals(b1));
b2.Set(idx);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
}
}
// try different type of object
Assert.IsFalse(b1.Equals(new Object()));
}
public bool Get(int index)
{
return(Bits.Get(index));
}
public void testHashCodeEquals()
{
// This test can't handle numBits==0:
int numBits = rnd.Next(2000) + 1;
FixedBitSet b1 = new FixedBitSet(numBits);
FixedBitSet b2 = new FixedBitSet(numBits);
Assert.IsTrue(b1.Equals(b2));
Assert.IsTrue(b2.Equals(b1));
for (int iter = 0; iter < 10 * rnd.Next(500); iter++)
{
int idx = rnd.Next(numBits);
if (!b1.Get(idx))
{
b1.Set(idx);
Assert.IsFalse(b1.Equals(b2));
Assert.AreNotEqual(b1.GetHashCode(), b2.GetHashCode());
b2.Set(idx);
Assert.AreEqual(b1, b2);
Assert.AreEqual(b1.GetHashCode(), b2.GetHashCode());
}
}
}
public virtual void DoTestLongPostingsNoPositions(IndexOptions options)
{
// Don't use TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random.NextInt64()));
int NUM_DOCS = AtLeast(2000);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
string s1 = GetRandomTerm(null);
string s2 = GetRandomTerm(s1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2);
/*
* for(int idx=0;idx<s1.Length();idx++) {
* System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
* }
* for(int idx=0;idx<s2.Length();idx++) {
* System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
* }
*/
}
FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
if (Random.NextBoolean())
{
isS1.Set(idx);
}
}
IndexReader r;
if (true)
{
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy());
iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random.NextDouble());
iwc.SetMaxBufferedDocs(-1);
RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc);
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = options;
for (int idx = 0; idx < NUM_DOCS; idx++)
{
Document doc = new Document();
string s = isS1.Get(idx) ? s1 : s2;
Field f = NewField("field", s, ft);
int count = TestUtil.NextInt32(Random, 1, 4);
for (int ct = 0; ct < count; ct++)
{
doc.Add(f);
}
riw.AddDocument(doc);
}
r = riw.GetReader();
riw.Dispose();
}
else
{
#pragma warning disable 162
r = DirectoryReader.Open(dir);
#pragma warning restore 162
}
/*
* if (VERBOSE) {
* System.out.println("TEST: terms");
* TermEnum termEnum = r.Terms();
* while(termEnum.Next()) {
* System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().Length());
* Assert.IsTrue(termEnum.DocFreq() > 0);
* System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.Length());
* System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.Length());
* final String s = termEnum.Term().Text();
* for(int idx=0;idx<s.Length();idx++) {
* System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx)));
* }
* }
* }
*/
Assert.AreEqual(NUM_DOCS, r.NumDocs);
Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0);
Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0);
int num = AtLeast(1000);
for (int iter = 0; iter < num; iter++)
{
string term;
bool doS1;
if (Random.NextBoolean())
{
term = s1;
doS1 = true;
}
else
{
term = s2;
doS1 = false;
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1 + " term=" + term);
}
DocsEnum docs;
DocsEnum postings;
if (options == IndexOptions.DOCS_ONLY)
{
docs = TestUtil.Docs(Random, r, "field", new BytesRef(term), null, null, DocsFlags.NONE);
postings = null;
}
else
{
docs = postings = TestUtil.Docs(Random, r, "field", new BytesRef(term), null, null, DocsFlags.FREQS);
Debug.Assert(postings != null);
}
Debug.Assert(docs != null);
int docID = -1;
while (docID < DocIdSetIterator.NO_MORE_DOCS)
{
int what = Random.Next(3);
if (what == 0)
{
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do next()");
}
// nextDoc
int expected = docID + 1;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = docs.NextDoc();
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3 && postings != null)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4);
}
}
else
{
// advance
int targetDocID;
if (docID == -1)
{
targetDocID = Random.Next(NUM_DOCS + 1);
}
else
{
targetDocID = docID + TestUtil.NextInt32(Random, 1, NUM_DOCS - docID);
}
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
}
int expected = targetDocID;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = docs.Advance(targetDocID);
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3 && postings != null)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4, "got invalid freq=" + freq);
}
}
}
}
r.Dispose();
dir.Dispose();
}
public virtual void DoTestLongPostingsNoPositions(FieldInfo.IndexOptions options)
{
// Don't use TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random().NextLong()));
int NUM_DOCS = AtLeast(2000);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
string s1 = GetRandomTerm(null);
string s2 = GetRandomTerm(s1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2);
/*
for(int idx=0;idx<s1.Length();idx++) {
System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
}
for(int idx=0;idx<s2.Length();idx++) {
System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
}
*/
}
FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
if (Random().NextBoolean())
{
isS1.Set(idx);
}
}
IndexReader r;
if (true)
{
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE).SetMergePolicy(NewLogMergePolicy());
iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random().NextDouble());
iwc.SetMaxBufferedDocs(-1);
RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, iwc);
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.IndexOptions = options;
for (int idx = 0; idx < NUM_DOCS; idx++)
{
Document doc = new Document();
string s = isS1.Get(idx) ? s1 : s2;
Field f = NewField("field", s, ft);
int count = TestUtil.NextInt(Random(), 1, 4);
for (int ct = 0; ct < count; ct++)
{
doc.Add(f);
}
riw.AddDocument(doc);
}
r = riw.Reader;
riw.Dispose();
}
else
{
r = DirectoryReader.Open(dir);
}
/*
if (VERBOSE) {
System.out.println("TEST: terms");
TermEnum termEnum = r.Terms();
while(termEnum.Next()) {
System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().Length());
Assert.IsTrue(termEnum.DocFreq() > 0);
System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.Length());
System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.Length());
final String s = termEnum.Term().Text();
for(int idx=0;idx<s.Length();idx++) {
System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx)));
}
}
}
*/
Assert.AreEqual(NUM_DOCS, r.NumDocs);
Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0);
Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0);
int num = AtLeast(1000);
for (int iter = 0; iter < num; iter++)
{
string term;
bool doS1;
if (Random().NextBoolean())
{
term = s1;
doS1 = true;
}
else
{
term = s2;
doS1 = false;
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1 + " term=" + term);
}
DocsEnum docs;
DocsEnum postings;
if (options == FieldInfo.IndexOptions.DOCS_ONLY)
{
docs = TestUtil.Docs(Random(), r, "field", new BytesRef(term), null, null, DocsEnum.FLAG_NONE);
postings = null;
}
else
{
docs = postings = TestUtil.Docs(Random(), r, "field", new BytesRef(term), null, null, DocsEnum.FLAG_FREQS);
Debug.Assert(postings != null);
}
Debug.Assert(docs != null);
int docID = -1;
while (docID < DocIdSetIterator.NO_MORE_DOCS)
{
int what = Random().Next(3);
if (what == 0)
{
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do next()");
}
// nextDoc
int expected = docID + 1;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = docs.NextDoc();
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random().Next(6) == 3 && postings != null)
{
int freq = postings.Freq();
Assert.IsTrue(freq >= 1 && freq <= 4);
}
}
else
{
// advance
int targetDocID;
if (docID == -1)
{
targetDocID = Random().Next(NUM_DOCS + 1);
}
else
{
targetDocID = docID + TestUtil.NextInt(Random(), 1, NUM_DOCS - docID);
}
if (VERBOSE)
{
Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
}
int expected = targetDocID;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = docs.Advance(targetDocID);
if (VERBOSE)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random().Next(6) == 3 && postings != null)
{
int freq = postings.Freq();
Assert.IsTrue(freq >= 1 && freq <= 4, "got invalid freq=" + freq);
}
}
}
}
r.Dispose();
dir.Dispose();
}
public virtual void TestLongPostings_Mem()
{
// Don't use TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random.NextInt64()));
int NUM_DOCS = AtLeast(2000);
if (Verbose)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
string s1 = GetRandomTerm(null);
string s2 = GetRandomTerm(s1);
if (Verbose)
{
Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2);
/*
* for(int idx=0;idx<s1.length();idx++) {
* System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx)));
* }
* for(int idx=0;idx<s2.length();idx++) {
* System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx)));
* }
*/
}
FixedBitSet isS1 = new FixedBitSet(NUM_DOCS);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
if (Random.NextBoolean())
{
isS1.Set(idx);
}
}
IndexReader r;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy());
iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random.NextDouble());
iwc.SetMaxBufferedDocs(-1);
RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc);
for (int idx = 0; idx < NUM_DOCS; idx++)
{
Document doc = new Document();
string s = isS1.Get(idx) ? s1 : s2;
Field f = NewTextField("field", s, Field.Store.NO);
int count = TestUtil.NextInt32(Random, 1, 4);
for (int ct = 0; ct < count; ct++)
{
doc.Add(f);
}
riw.AddDocument(doc);
}
r = riw.GetReader();
riw.Dispose();
/*
* if (VERBOSE) {
* System.out.println("TEST: terms");
* TermEnum termEnum = r.Terms();
* while(termEnum.Next()) {
* System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().length());
* Assert.IsTrue(termEnum.DocFreq() > 0);
* System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.length());
* System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.length());
* final String s = termEnum.Term().Text();
* for(int idx=0;idx<s.length();idx++) {
* System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx)));
* }
* }
* }
*/
Assert.AreEqual(NUM_DOCS, r.NumDocs);
Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0);
Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0);
int num = AtLeast(1000);
for (int iter = 0; iter < num; iter++)
{
string term;
bool doS1;
if (Random.NextBoolean())
{
term = s1;
doS1 = true;
}
else
{
term = s2;
doS1 = false;
}
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1);
}
DocsAndPositionsEnum postings = MultiFields.GetTermPositionsEnum(r, null, "field", new BytesRef(term));
int docID = -1;
while (docID < DocIdSetIterator.NO_MORE_DOCS)
{
int what = Random.Next(3);
if (what == 0)
{
if (Verbose)
{
Console.WriteLine("TEST: docID=" + docID + "; do next()");
}
// nextDoc
int expected = docID + 1;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = postings.NextDoc();
if (Verbose)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4);
for (int pos = 0; pos < freq; pos++)
{
Assert.AreEqual(pos, postings.NextPosition());
if (Random.NextBoolean())
{
var dummy = postings.GetPayload();
if (Random.NextBoolean())
{
dummy = postings.GetPayload(); // get it again
}
}
}
}
}
else
{
// advance
int targetDocID;
if (docID == -1)
{
targetDocID = Random.Next(NUM_DOCS + 1);
}
else
{
targetDocID = docID + TestUtil.NextInt32(Random, 1, NUM_DOCS - docID);
}
if (Verbose)
{
Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")");
}
int expected = targetDocID;
while (true)
{
if (expected == NUM_DOCS)
{
expected = int.MaxValue;
break;
}
else if (isS1.Get(expected) == doS1)
{
break;
}
else
{
expected++;
}
}
docID = postings.Advance(targetDocID);
if (Verbose)
{
Console.WriteLine(" got docID=" + docID);
}
Assert.AreEqual(expected, docID);
if (docID == DocIdSetIterator.NO_MORE_DOCS)
{
break;
}
if (Random.Next(6) == 3)
{
int freq = postings.Freq;
Assert.IsTrue(freq >= 1 && freq <= 4);
for (int pos = 0; pos < freq; pos++)
{
Assert.AreEqual(pos, postings.NextPosition());
if (Random.NextBoolean())
{
var dummy = postings.GetPayload();
if (Random.NextBoolean())
{
dummy = postings.GetPayload(); // get it again
}
}
}
}
}
}
}
r.Dispose();
dir.Dispose();
}
