/// <summary> /// Detect mis-use, where provided parent query in fact sometimes returns child documents. /// </summary> private void ValidateParentDoc() { if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc)) { throw new InvalidOperationException(InvalidQueryMessage + _parentDoc); } }
protected override void Swap(int i, int j) { long tmpDoc = docs.Get(j); docs.Set(j, docs.Get(i)); docs.Set(i, tmpDoc); long tmpVal = values.Get(j); values.Set(j, values.Get(i)); values.Set(i, tmpVal); bool tmpBool = docsWithField.Get(j); if (docsWithField.Get(i)) { docsWithField.Set(j); } else { docsWithField.Clear(j); } if (tmpBool) { docsWithField.Set(i); } else { docsWithField.Clear(i); } }
protected internal override void Swap(int i, int j) { long tmpDoc = Docs.Get(j); Docs.Set(j, Docs.Get(i)); Docs.Set(i, tmpDoc); long tmpVal = Values.Get(j); Values.Set(j, Values.Get(i)); Values.Set(i, tmpVal); bool tmpBool = DocsWithField.Get(j); if (DocsWithField.Get(i)) { DocsWithField.Set(j); } else { DocsWithField.Clear(j); } if (tmpBool) { DocsWithField.Set(i); } else { DocsWithField.Clear(i); } }
/// <summary> /// Detect mis-use, where provided parent query in fact sometimes returns child documents. /// </summary> private void ValidateParentDoc() { if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc)) { throw IllegalStateException.Create(INVALID_QUERY_MESSAGE + _parentDoc); } }
/// <summary> /// Detect mis-use, where provided parent query in fact sometimes returns child documents. /// </summary> private void ValidateParentDoc() { if (_parentDoc != NO_MORE_DOCS && !_parentBits.Get(_parentDoc)) { throw new InvalidOperationException(INVALID_QUERY_MESSAGE + _parentDoc); } }
internal IBits RandomLiveDocs(int maxDoc) { if (Rarely()) { if (Random.nextBoolean()) { return(null); } else { return(new Bits.MatchNoBits(maxDoc)); } } FixedBitSet bits = new FixedBitSet(maxDoc); int bitsSet = TestUtil.NextInt32(Random, 1, maxDoc - 1); for (int i = 0; i < bitsSet; ++i) { while (true) { int index = Random.nextInt(maxDoc); if (!bits.Get(index)) { bits.Set(index); break; } } } return(bits); }
/// <summary> /// Useful from an assert. </summary> internal virtual bool IsConsistent(int maxDoc) { FixedBitSet targets = new FixedBitSet(maxDoc); for (int i = 0; i < maxDoc; ++i) { int target = Map(i); if (target < 0 || target >= maxDoc) { if (Debugging.AssertsEnabled) { Debugging.Assert(false, "out of range: {0} not in [0-{1}[", target, maxDoc); } return(false); } else if (targets.Get(target)) { if (Debugging.AssertsEnabled) { Debugging.Assert(false, "{0} is already taken ({1})", target, i); } return(false); } } return(true); }
protected override void FillDocsAndScores(FixedBitSet matchingDocs, IBits acceptDocs, TermsEnum termsEnum) { BytesRef spare = new BytesRef(); DocsEnum docsEnum = null; for (int i = 0; i < m_outerInstance._terms.Count; i++) { if (termsEnum.SeekExact(m_outerInstance._terms.Get(m_outerInstance._ords[i], spare))) { docsEnum = termsEnum.Docs(acceptDocs, docsEnum, DocsFlags.NONE); float score = m_outerInstance._scores[m_outerInstance._ords[i]]; for (int doc = docsEnum.NextDoc(); doc != NO_MORE_DOCS; doc = docsEnum.NextDoc()) { // I prefer this: /*if (scores[doc] < score) { * scores[doc] = score; * matchingDocs.set(doc); * }*/ // But this behaves the same as MVInnerScorer and only then the tests will pass: if (!matchingDocs.Get(doc)) { scores[doc] = score; matchingDocs.Set(doc); } } } } }
private bool OpenBitSetContains(int[] expectedDocs, FixedBitSet actual, int maxDoc) { if (expectedDocs.Length != actual.Cardinality()) { return(false); } FixedBitSet expected = new FixedBitSet(maxDoc); foreach (int expectedDoc in expectedDocs) { expected.Set(expectedDoc); } int docId; DocIdSetIterator iterator = expected.GetIterator(); while ((docId = iterator.NextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (!actual.Get(docId)) { return(false); } } return(true); }
public int NextDoc() { if (Idx >= Size) { Offset = -1; return(Doc_Renamed = DocIdSetIterator.NO_MORE_DOCS); } Doc_Renamed = (int)Docs.Get(Idx); ++Idx; while (Idx < Size && Docs.Get(Idx) == Doc_Renamed) { ++Idx; } // idx points to the "next" element long prevIdx = Idx - 1; if (!DocsWithField.Get((int)prevIdx)) { Offset = -1; } else { // cannot change 'value' here because nextDoc is called before the // value is used, and it's a waste to clone the BytesRef when we // obtain the value Offset = (int)Offsets.Get(prevIdx); Length = (int)Lengths.Get(prevIdx); } return(Doc_Renamed); }
private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam) { // Use yield return instead of ucsom IEnumerable AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = Lengths.GetIterator(); int size = (int)Lengths.Size(); DataInput bytesIterator = Bytes.DataInput; int maxDoc = maxDocParam; int upto = 0; while (upto < maxDoc) { BytesRef v = null; if (upto < size) { int length = (int)lengthsIterator.Next(); var value = new BytesRef(); value.Grow(length); value.Length = length; bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length); if (DocsWithField.Get(upto)) { v = value; } } upto++; yield return(v); } }
/// <summary> /// Useful from an assert. </summary> internal virtual bool IsConsistent(int maxDoc) { FixedBitSet targets = new FixedBitSet(maxDoc); for (int i = 0; i < maxDoc; ++i) { int target = Map(i); if (target < 0 || target >= maxDoc) { if (Debugging.AssertsEnabled) { Debugging.Assert(false, () => "out of range: " + target + " not in [0-" + maxDoc + "["); } return(false); } else if (targets.Get(target)) { if (Debugging.AssertsEnabled) { Debugging.Assert(false, () => target + " is already taken (" + i + ")"); } return(false); } } return(true); }
public override int NextDoc() { if (idx >= size) { offset = -1; return(doc = DocIdSetIterator.NO_MORE_DOCS); } doc = (int)docs.Get(idx); ++idx; while (idx < size && docs.Get(idx) == doc) { ++idx; } // idx points to the "next" element long prevIdx = idx - 1; if (!docsWithField.Get((int)prevIdx)) { offset = -1; } else { // cannot change 'value' here because nextDoc is called before the // value is used, and it's a waste to clone the BytesRef when we // obtain the value offset = (int)offsets.Get(prevIdx); length = (int)lengths.Get(prevIdx); } return(doc); }
private IEnumerable <long?> GetNumericIterator(int maxDoc) { // LUCENENET specific: using yield return instead of custom iterator type. Much less code. AbstractAppendingInt64Buffer.Iterator iter = pending.GetIterator(); int size = (int)pending.Count; int upto = 0; while (upto < maxDoc) { long?value; if (upto < size) { var v = iter.Next(); if (docsWithField == null || docsWithField.Get(upto)) { value = v; } else { value = null; } } else { value = docsWithField != null ? (long?)null : MISSING; } upto++; // TODO: make reusable Number yield return(value); } }
public void testEquals() { // This test can't handle numBits==0: int numBits = rnd.Next(2000) + 1; FixedBitSet b1 = new FixedBitSet(numBits); FixedBitSet b2 = new FixedBitSet(numBits); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); for (int iter = 0; iter < 10 * rnd.Next(500); iter++) { int idx = rnd.Next(numBits); if (!b1.Get(idx)) { b1.Set(idx); Assert.IsFalse(b1.Equals(b2)); Assert.IsFalse(b2.Equals(b1)); b2.Set(idx); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); } } // try different type of object Assert.IsFalse(b1.Equals(new Object())); }
private ContainsResult MayContainValue(int positiveHash) { Debug.Assert((positiveHash >= 0)); // Bloom sizes are always base 2 and so can be ANDed for a fast modulo var pos = positiveHash & _bloomSize; return(_filter.Get(pos) ? ContainsResult.MAYBE : ContainsResult.NO); }
void doGet(BitArray a, FixedBitSet b) { int max = b.Length(); for (int i = 0; i < max; i++) { if (a.Get(i) != b.Get(i)) { Assert.Fail("mismatch: BitSet=[" + i + "]=" + a.Get(i)); } } }
void doGet(BitArray a, FixedBitSet b) { int max = b.Length(); for (int i = 0; i < max; i++) { if (a.Get(i) != b.Get(i)) { Assert.Fail("mismatch: BitSet=[" + i + "]=" + a.Get(i)); } } }
public virtual void TestBuildDocMap() { int maxDoc = TestUtil.NextInt(Random(), 1, 128); int numDocs = TestUtil.NextInt(Random(), 0, maxDoc); int numDeletedDocs = maxDoc - numDocs; FixedBitSet liveDocs = new FixedBitSet(maxDoc); for (int i = 0; i < numDocs; ++i) { while (true) { int docID = Random().Next(maxDoc); if (!liveDocs.Get(docID)) { liveDocs.Set(docID); break; } } } MergeState.DocMap docMap = MergeState.DocMap.Build(maxDoc, liveDocs); Assert.AreEqual(maxDoc, docMap.MaxDoc); Assert.AreEqual(numDocs, docMap.NumDocs); Assert.AreEqual(numDeletedDocs, docMap.NumDeletedDocs); // assert the mapping is compact for (int i = 0, del = 0; i < maxDoc; ++i) { if (!liveDocs.Get(i)) { Assert.AreEqual(-1, docMap.Get(i)); ++del; } else { Assert.AreEqual(i - del, docMap.Get(i)); } } }
protected internal override sealed bool MatchDoc(int doc) { int ord = fcsi.GetOrd(doc); if (ord == -1) { // missing return(false); } else { return(bits.Get(ord)); } }
protected override void Swap(int i, int j) { long tmpDoc = docs.Get(j); docs.Set(j, docs.Get(i)); docs.Set(i, tmpDoc); long tmpOffset = offsets.Get(j); offsets.Set(j, offsets.Get(i)); offsets.Set(i, tmpOffset); long tmpLength = lengths.Get(j); lengths.Set(j, lengths.Get(i)); lengths.Set(i, tmpLength); bool tmpBool = docsWithField.Get(j); if (docsWithField.Get(i)) { docsWithField.Set(j); } else { docsWithField.Clear(j); } if (tmpBool) { docsWithField.Set(i); } else { docsWithField.Clear(i); } }
protected override void Swap(int i, int j) { long tmpDoc = Docs.Get(j); Docs.Set(j, Docs.Get(i)); Docs.Set(i, tmpDoc); long tmpOffset = Offsets.Get(j); Offsets.Set(j, Offsets.Get(i)); Offsets.Set(i, tmpOffset); long tmpLength = Lengths.Get(j); Lengths.Set(j, Lengths.Get(i)); Lengths.Set(i, tmpLength); bool tmpBool = DocsWithField.Get(j); if (DocsWithField.Get(i)) { DocsWithField.Set(j); } else { DocsWithField.Clear(j); } if (tmpBool) { DocsWithField.Set(i); } else { DocsWithField.Clear(i); } }
/// <summary> /// pp was just advanced. If that caused a repeater collision, resolve by advancing the lesser /// of the two colliding pps. Note that there can only be one collision, as by the initialization /// there were no collisions before pp was advanced. /// </summary> private bool AdvanceRpts(PhrasePositions pp) { if (pp.rptGroup < 0) { return(true); // not a repeater } PhrasePositions[] rg = rptGroups[pp.rptGroup]; FixedBitSet bits = new FixedBitSet(rg.Length); // for re-queuing after collisions are resolved int k0 = pp.rptInd; int k; while ((k = Collide(pp)) >= 0) { pp = Lesser(pp, rg[k]); // always advance the lesser of the (only) two colliding pps if (!AdvancePP(pp)) { return(false); // exhausted } if (k != k0) // careful: mark only those currently in the queue { bits = FixedBitSet.EnsureCapacity(bits, k); bits.Set(k); // mark that pp2 need to be re-queued } } // collisions resolved, now re-queue // empty (partially) the queue until seeing all pps advanced for resolving collisions int n = 0; // TODO would be good if we can avoid calling cardinality() in each iteration! int numBits = bits.Length; // larges bit we set while (bits.Cardinality() > 0) { PhrasePositions pp2 = pq.Pop(); rptStack[n++] = pp2; if (pp2.rptGroup >= 0 && pp2.rptInd < numBits && bits.Get(pp2.rptInd)) // this bit may not have been set { bits.Clear(pp2.rptInd); } } // add back to queue for (int i = n - 1; i >= 0; i--) { pq.Add(rptStack[i]); } return(true); }
private IEnumerable <BytesRef> GetBytesIterator(int maxDocParam) { // Use yield return instead of ucsom IEnumerable var value = new BytesRef(); AppendingDeltaPackedInt64Buffer.Iterator lengthsIterator = lengths.GetIterator(); int size = (int)lengths.Count; DataInput bytesIterator = bytes.GetDataInput(); int maxDoc = maxDocParam; int upto = 0; while (upto < maxDoc) { BytesRef v = null; if (upto < size) { int length = (int)lengthsIterator.Next(); value.Grow(length); value.Length = length; try { bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length); } catch (Exception ioe) when(ioe.IsIOException()) { // Should never happen! throw RuntimeException.Create(ioe); } if (docsWithField.Get(upto)) { v = value; } } upto++; yield return(v); } }
private BigSegmentedArray GetCollapsedCounts() { if (m_collapsedCounts == null) { m_collapsedCounts = new LazyBigInt32Array(m_bucketValues.Count); FacetDataCache dataCache = m_subCollector.DataCache; ITermValueList subList = dataCache.ValArray; BigSegmentedArray subcounts = m_subCollector.Count; FixedBitSet indexSet = new FixedBitSet(subcounts.Length); int c = 0; int i = 0; foreach (string val in m_bucketValues) { if (val.Length > 0) { string[] subVals = m_predefinedBuckets.Get(val); int count = 0; foreach (string subVal in subVals) { int index = subList.IndexOf(subVal); if (index > 0) { int subcount = subcounts.Get(index); count += subcount; if (!indexSet.Get(index)) { indexSet.Set(index); c += dataCache.Freqs[index]; } } } m_collapsedCounts.Add(i, count); } i++; } m_collapsedCounts.Add(0, (m_numdocs - c)); } return(m_collapsedCounts); }
public void testHashCodeEquals() { // This test can't handle numBits==0: int numBits = rnd.Next(2000) + 1; FixedBitSet b1 = new FixedBitSet(numBits); FixedBitSet b2 = new FixedBitSet(numBits); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); for (int iter = 0; iter < 10 * rnd.Next(500); iter++) { int idx = rnd.Next(numBits); if (!b1.Get(idx)) { b1.Set(idx); Assert.IsFalse(b1.Equals(b2)); Assert.AreNotEqual(b1.GetHashCode(), b2.GetHashCode()); b2.Set(idx); Assert.AreEqual(b1, b2); Assert.AreEqual(b1.GetHashCode(), b2.GetHashCode()); } } }
public override int NextDoc() { if (idx >= size) { value = null; return(doc = DocIdSetIterator.NO_MORE_DOCS); } doc = (int)docs.Get(idx); ++idx; while (idx < size && docs.Get(idx) == doc) { ++idx; } if (!docsWithField.Get((int)(idx - 1))) { value = null; } else { // idx points to the "next" element value = Convert.ToInt64(values.Get(idx - 1)); } return(doc); }
public int NextDoc() { if (Idx >= Size) { Value_Renamed = null; return(Doc_Renamed = DocIdSetIterator.NO_MORE_DOCS); } Doc_Renamed = (int)Docs.Get(Idx); ++Idx; while (Idx < Size && Docs.Get(Idx) == Doc_Renamed) { ++Idx; } if (!DocsWithField.Get((int)(Idx - 1))) { Value_Renamed = null; } else { // idx points to the "next" element Value_Renamed = Convert.ToInt64(Values.Get(Idx - 1)); } return(Doc_Renamed); }
public virtual void TestBuildDocMap() { int maxDoc = TestUtil.NextInt(Random(), 1, 128); int numDocs = TestUtil.NextInt(Random(), 0, maxDoc); int numDeletedDocs = maxDoc - numDocs; FixedBitSet liveDocs = new FixedBitSet(maxDoc); for (int i = 0; i < numDocs; ++i) { while (true) { int docID = Random().Next(maxDoc); if (!liveDocs.Get(docID)) { liveDocs.Set(docID); break; } } } MergeState.DocMap docMap = MergeState.DocMap.Build(maxDoc, liveDocs); Assert.AreEqual(maxDoc, docMap.MaxDoc); Assert.AreEqual(numDocs, docMap.NumDocs); Assert.AreEqual(numDeletedDocs, docMap.NumDeletedDocs); // assert the mapping is compact for (int i = 0, del = 0; i < maxDoc; ++i) { if (!liveDocs.Get(i)) { Assert.AreEqual(-1, docMap.Get(i)); ++del; } else { Assert.AreEqual(i - del, docMap.Get(i)); } } }
public void testEquals() { // This test can't handle numBits==0: int numBits = rnd.Next(2000) + 1; FixedBitSet b1 = new FixedBitSet(numBits); FixedBitSet b2 = new FixedBitSet(numBits); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); for (int iter = 0; iter < 10 * rnd.Next(500); iter++) { int idx = rnd.Next(numBits); if (!b1.Get(idx)) { b1.Set(idx); Assert.IsFalse(b1.Equals(b2)); Assert.IsFalse(b2.Equals(b1)); b2.Set(idx); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); } } // try different type of object Assert.IsFalse(b1.Equals(new Object())); }
public bool Get(int index) { return(Bits.Get(index)); }
public void testHashCodeEquals() { // This test can't handle numBits==0: int numBits = rnd.Next(2000) + 1; FixedBitSet b1 = new FixedBitSet(numBits); FixedBitSet b2 = new FixedBitSet(numBits); Assert.IsTrue(b1.Equals(b2)); Assert.IsTrue(b2.Equals(b1)); for (int iter = 0; iter < 10 * rnd.Next(500); iter++) { int idx = rnd.Next(numBits); if (!b1.Get(idx)) { b1.Set(idx); Assert.IsFalse(b1.Equals(b2)); Assert.AreNotEqual(b1.GetHashCode(), b2.GetHashCode()); b2.Set(idx); Assert.AreEqual(b1, b2); Assert.AreEqual(b1.GetHashCode(), b2.GetHashCode()); } } }
public virtual void DoTestLongPostingsNoPositions(IndexOptions options) { // Don't use TestUtil.getTempDir so that we own the // randomness (ie same seed will point to same dir): Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random.NextInt64())); int NUM_DOCS = AtLeast(2000); if (VERBOSE) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); } string s1 = GetRandomTerm(null); string s2 = GetRandomTerm(s1); if (VERBOSE) { Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2); /* * for(int idx=0;idx<s1.Length();idx++) { * System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx))); * } * for(int idx=0;idx<s2.Length();idx++) { * System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx))); * } */ } FixedBitSet isS1 = new FixedBitSet(NUM_DOCS); for (int idx = 0; idx < NUM_DOCS; idx++) { if (Random.NextBoolean()) { isS1.Set(idx); } } IndexReader r; if (true) { IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy()); iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random.NextDouble()); iwc.SetMaxBufferedDocs(-1); RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc); FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.IndexOptions = options; for (int idx = 0; idx < NUM_DOCS; idx++) { Document doc = new Document(); string s = isS1.Get(idx) ? s1 : s2; Field f = NewField("field", s, ft); int count = TestUtil.NextInt32(Random, 1, 4); for (int ct = 0; ct < count; ct++) { doc.Add(f); } riw.AddDocument(doc); } r = riw.GetReader(); riw.Dispose(); } else { #pragma warning disable 162 r = DirectoryReader.Open(dir); #pragma warning restore 162 } /* * if (VERBOSE) { * System.out.println("TEST: terms"); * TermEnum termEnum = r.Terms(); * while(termEnum.Next()) { * System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().Length()); * Assert.IsTrue(termEnum.DocFreq() > 0); * System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.Length()); * System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.Length()); * final String s = termEnum.Term().Text(); * for(int idx=0;idx<s.Length();idx++) { * System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx))); * } * } * } */ Assert.AreEqual(NUM_DOCS, r.NumDocs); Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0); Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0); int num = AtLeast(1000); for (int iter = 0; iter < num; iter++) { string term; bool doS1; if (Random.NextBoolean()) { term = s1; doS1 = true; } else { term = s2; doS1 = false; } if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1 + " term=" + term); } DocsEnum docs; DocsEnum postings; if (options == IndexOptions.DOCS_ONLY) { docs = TestUtil.Docs(Random, r, "field", new BytesRef(term), null, null, DocsFlags.NONE); postings = null; } else { docs = postings = TestUtil.Docs(Random, r, "field", new BytesRef(term), null, null, DocsFlags.FREQS); Debug.Assert(postings != null); } Debug.Assert(docs != null); int docID = -1; while (docID < DocIdSetIterator.NO_MORE_DOCS) { int what = Random.Next(3); if (what == 0) { if (VERBOSE) { Console.WriteLine("TEST: docID=" + docID + "; do next()"); } // nextDoc int expected = docID + 1; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = docs.NextDoc(); if (VERBOSE) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random.Next(6) == 3 && postings != null) { int freq = postings.Freq; Assert.IsTrue(freq >= 1 && freq <= 4); } } else { // advance int targetDocID; if (docID == -1) { targetDocID = Random.Next(NUM_DOCS + 1); } else { targetDocID = docID + TestUtil.NextInt32(Random, 1, NUM_DOCS - docID); } if (VERBOSE) { Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")"); } int expected = targetDocID; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = docs.Advance(targetDocID); if (VERBOSE) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random.Next(6) == 3 && postings != null) { int freq = postings.Freq; Assert.IsTrue(freq >= 1 && freq <= 4, "got invalid freq=" + freq); } } } } r.Dispose(); dir.Dispose(); }
public virtual void DoTestLongPostingsNoPositions(FieldInfo.IndexOptions options) { // Don't use TestUtil.getTempDir so that we own the // randomness (ie same seed will point to same dir): Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random().NextLong())); int NUM_DOCS = AtLeast(2000); if (VERBOSE) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); } string s1 = GetRandomTerm(null); string s2 = GetRandomTerm(s1); if (VERBOSE) { Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2); /* for(int idx=0;idx<s1.Length();idx++) { System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx))); } for(int idx=0;idx<s2.Length();idx++) { System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx))); } */ } FixedBitSet isS1 = new FixedBitSet(NUM_DOCS); for (int idx = 0; idx < NUM_DOCS; idx++) { if (Random().NextBoolean()) { isS1.Set(idx); } } IndexReader r; if (true) { IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE).SetMergePolicy(NewLogMergePolicy()); iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random().NextDouble()); iwc.SetMaxBufferedDocs(-1); RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, iwc); FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.IndexOptions = options; for (int idx = 0; idx < NUM_DOCS; idx++) { Document doc = new Document(); string s = isS1.Get(idx) ? s1 : s2; Field f = NewField("field", s, ft); int count = TestUtil.NextInt(Random(), 1, 4); for (int ct = 0; ct < count; ct++) { doc.Add(f); } riw.AddDocument(doc); } r = riw.Reader; riw.Dispose(); } else { r = DirectoryReader.Open(dir); } /* if (VERBOSE) { System.out.println("TEST: terms"); TermEnum termEnum = r.Terms(); while(termEnum.Next()) { System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().Length()); Assert.IsTrue(termEnum.DocFreq() > 0); System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.Length()); System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.Length()); final String s = termEnum.Term().Text(); for(int idx=0;idx<s.Length();idx++) { System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx))); } } } */ Assert.AreEqual(NUM_DOCS, r.NumDocs); Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0); Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0); int num = AtLeast(1000); for (int iter = 0; iter < num; iter++) { string term; bool doS1; if (Random().NextBoolean()) { term = s1; doS1 = true; } else { term = s2; doS1 = false; } if (VERBOSE) { Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1 + " term=" + term); } DocsEnum docs; DocsEnum postings; if (options == FieldInfo.IndexOptions.DOCS_ONLY) { docs = TestUtil.Docs(Random(), r, "field", new BytesRef(term), null, null, DocsEnum.FLAG_NONE); postings = null; } else { docs = postings = TestUtil.Docs(Random(), r, "field", new BytesRef(term), null, null, DocsEnum.FLAG_FREQS); Debug.Assert(postings != null); } Debug.Assert(docs != null); int docID = -1; while (docID < DocIdSetIterator.NO_MORE_DOCS) { int what = Random().Next(3); if (what == 0) { if (VERBOSE) { Console.WriteLine("TEST: docID=" + docID + "; do next()"); } // nextDoc int expected = docID + 1; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = docs.NextDoc(); if (VERBOSE) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random().Next(6) == 3 && postings != null) { int freq = postings.Freq(); Assert.IsTrue(freq >= 1 && freq <= 4); } } else { // advance int targetDocID; if (docID == -1) { targetDocID = Random().Next(NUM_DOCS + 1); } else { targetDocID = docID + TestUtil.NextInt(Random(), 1, NUM_DOCS - docID); } if (VERBOSE) { Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")"); } int expected = targetDocID; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = docs.Advance(targetDocID); if (VERBOSE) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random().Next(6) == 3 && postings != null) { int freq = postings.Freq(); Assert.IsTrue(freq >= 1 && freq <= 4, "got invalid freq=" + freq); } } } } r.Dispose(); dir.Dispose(); }
public virtual void TestLongPostings_Mem() { // Don't use TestUtil.getTempDir so that we own the // randomness (ie same seed will point to same dir): Directory dir = NewFSDirectory(CreateTempDir("longpostings" + "." + Random.NextInt64())); int NUM_DOCS = AtLeast(2000); if (Verbose) { Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS); } string s1 = GetRandomTerm(null); string s2 = GetRandomTerm(s1); if (Verbose) { Console.WriteLine("\nTEST: s1=" + s1 + " s2=" + s2); /* * for(int idx=0;idx<s1.length();idx++) { * System.out.println(" s1 ch=0x" + Integer.toHexString(s1.charAt(idx))); * } * for(int idx=0;idx<s2.length();idx++) { * System.out.println(" s2 ch=0x" + Integer.toHexString(s2.charAt(idx))); * } */ } FixedBitSet isS1 = new FixedBitSet(NUM_DOCS); for (int idx = 0; idx < NUM_DOCS; idx++) { if (Random.NextBoolean()) { isS1.Set(idx); } } IndexReader r; IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy()); iwc.SetRAMBufferSizeMB(16.0 + 16.0 * Random.NextDouble()); iwc.SetMaxBufferedDocs(-1); RandomIndexWriter riw = new RandomIndexWriter(Random, dir, iwc); for (int idx = 0; idx < NUM_DOCS; idx++) { Document doc = new Document(); string s = isS1.Get(idx) ? s1 : s2; Field f = NewTextField("field", s, Field.Store.NO); int count = TestUtil.NextInt32(Random, 1, 4); for (int ct = 0; ct < count; ct++) { doc.Add(f); } riw.AddDocument(doc); } r = riw.GetReader(); riw.Dispose(); /* * if (VERBOSE) { * System.out.println("TEST: terms"); * TermEnum termEnum = r.Terms(); * while(termEnum.Next()) { * System.out.println(" term=" + termEnum.Term() + " len=" + termEnum.Term().Text().length()); * Assert.IsTrue(termEnum.DocFreq() > 0); * System.out.println(" s1?=" + (termEnum.Term().Text().equals(s1)) + " s1len=" + s1.length()); * System.out.println(" s2?=" + (termEnum.Term().Text().equals(s2)) + " s2len=" + s2.length()); * final String s = termEnum.Term().Text(); * for(int idx=0;idx<s.length();idx++) { * System.out.println(" ch=0x" + Integer.toHexString(s.charAt(idx))); * } * } * } */ Assert.AreEqual(NUM_DOCS, r.NumDocs); Assert.IsTrue(r.DocFreq(new Term("field", s1)) > 0); Assert.IsTrue(r.DocFreq(new Term("field", s2)) > 0); int num = AtLeast(1000); for (int iter = 0; iter < num; iter++) { string term; bool doS1; if (Random.NextBoolean()) { term = s1; doS1 = true; } else { term = s2; doS1 = false; } if (Verbose) { Console.WriteLine("\nTEST: iter=" + iter + " doS1=" + doS1); } DocsAndPositionsEnum postings = MultiFields.GetTermPositionsEnum(r, null, "field", new BytesRef(term)); int docID = -1; while (docID < DocIdSetIterator.NO_MORE_DOCS) { int what = Random.Next(3); if (what == 0) { if (Verbose) { Console.WriteLine("TEST: docID=" + docID + "; do next()"); } // nextDoc int expected = docID + 1; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = postings.NextDoc(); if (Verbose) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random.Next(6) == 3) { int freq = postings.Freq; Assert.IsTrue(freq >= 1 && freq <= 4); for (int pos = 0; pos < freq; pos++) { Assert.AreEqual(pos, postings.NextPosition()); if (Random.NextBoolean()) { var dummy = postings.GetPayload(); if (Random.NextBoolean()) { dummy = postings.GetPayload(); // get it again } } } } } else { // advance int targetDocID; if (docID == -1) { targetDocID = Random.Next(NUM_DOCS + 1); } else { targetDocID = docID + TestUtil.NextInt32(Random, 1, NUM_DOCS - docID); } if (Verbose) { Console.WriteLine("TEST: docID=" + docID + "; do advance(" + targetDocID + ")"); } int expected = targetDocID; while (true) { if (expected == NUM_DOCS) { expected = int.MaxValue; break; } else if (isS1.Get(expected) == doS1) { break; } else { expected++; } } docID = postings.Advance(targetDocID); if (Verbose) { Console.WriteLine(" got docID=" + docID); } Assert.AreEqual(expected, docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } if (Random.Next(6) == 3) { int freq = postings.Freq; Assert.IsTrue(freq >= 1 && freq <= 4); for (int pos = 0; pos < freq; pos++) { Assert.AreEqual(pos, postings.NextPosition()); if (Random.NextBoolean()) { var dummy = postings.GetPayload(); if (Random.NextBoolean()) { dummy = postings.GetPayload(); // get it again } } } } } } } r.Dispose(); dir.Dispose(); }