public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bits = new OpenBitSet((reader.MaxDoc() % 64 == 0 ? reader.MaxDoc() / 64 : reader.MaxDoc() / 64 + 1) * 64); new IndexSearcher(reader).Search(query, new AnonymousClassHitCollector2(bits, this)); return(bits); }
public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bitSet = new OpenBitSet((reader.MaxDoc() % 64 == 0 ? reader.MaxDoc() / 64 : reader.MaxDoc() / 64 + 1) * 64); new AnonymousClassPrefixGenerator2(bitSet, this, prefix).Generate(reader); return(bitSet); }
public override DocIdSet GetDocIdSet(IndexReader reader) { TermEnum enumerator = query.GetEnum(reader); try { // if current term in enum is null, the enum is empty -> shortcut if (enumerator.Term == null) { return(DocIdSet.EMPTY_DOCIDSET); } // else fill into an OpenBitSet OpenBitSet bitSet = new OpenBitSet(reader.MaxDoc); int[] docs = new int[32]; int[] freqs = new int[32]; TermDocs termDocs = reader.TermDocs(); try { int termCount = 0; do { Term term = enumerator.Term; if (term == null) { break; } termCount++; termDocs.Seek(term); while (true) { int count = termDocs.Read(docs, freqs); if (count != 0) { for (int i = 0; i < count; i++) { bitSet.Set(docs[i]); } } else { break; } } } while (enumerator.Next()); query.IncTotalNumberOfTerms(termCount); // {{Aroush-2.9}} is the use of 'temp' as is right? } finally { termDocs.Close(); } return(bitSet); } finally { enumerator.Close(); } }
private void InitBlock(int lower, int upper, bool useBitSet, Lucene.Net.Util.OpenBitSet bits, System.Collections.IEnumerator neededBounds, TestNumericUtils enclosingInstance) { this.lower = lower; this.upper = upper; this.useBitSet = useBitSet; this.bits = bits; this.neededBounds = neededBounds; this.enclosingInstance = enclosingInstance; }
public override IBits ReadLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) { if (Debugging.AssertsEnabled) { Debugging.Assert(info.HasDeletions); } var scratch = new BytesRef(); var scratchUtf16 = new CharsRef(); var fileName = IndexFileNames.FileNameFromGeneration(info.Info.Name, LIVEDOCS_EXTENSION, info.DelGen); ChecksumIndexInput input = null; var success = false; try { input = dir.OpenChecksumInput(fileName, context); SimpleTextUtil.ReadLine(input, scratch); if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, SIZE)); } var size = ParseInt32At(scratch, SIZE.Length, scratchUtf16); var bits = new BitSet(size); SimpleTextUtil.ReadLine(input, scratch); while (!scratch.Equals(END)) { if (Debugging.AssertsEnabled) { Debugging.Assert(StringHelper.StartsWith(scratch, DOC)); } var docid = ParseInt32At(scratch, DOC.Length, scratchUtf16); bits.Set(docid); SimpleTextUtil.ReadLine(input, scratch); } SimpleTextUtil.CheckFooter(input); success = true; return(new SimpleTextBits(bits, size)); } finally { if (success) { IOUtils.Dispose(input); } else { IOUtils.DisposeWhileHandlingException(input); } } }
public virtual void TestRepeatedRollBacks() { int expectedLastRecordId = 100; while (expectedLastRecordId > 10) { expectedLastRecordId -= 10; RollBackLast(expectedLastRecordId); BitSet expecteds = new BitSet(100); expecteds.Set(1, (expectedLastRecordId + 1)); CheckExpecteds(expecteds); } }
public FieldCacheTermsFilterDocIdSet(FieldCacheTermsFilter enclosingInstance, Lucene.Net.Search.StringIndex fcsi) { InitBlock(enclosingInstance); this.fcsi = fcsi; openBitSet = new OpenBitSet(this.fcsi.lookup.Length); for (int i = 0; i < Enclosing_Instance.terms.Length; i++) { int termNumber = this.fcsi.BinarySearchLookup(Enclosing_Instance.terms[i]); if (termNumber > 0) { openBitSet.FastSet(termNumber); } } }
public FieldCacheTermsFilterDocIdSet(FieldCacheTermsFilter enclosingInstance, StringIndex fcsi) { InitBlock(enclosingInstance); this.fcsi = fcsi; openBitSet = new OpenBitSet(this.fcsi.lookup.Length); foreach (string t in Enclosing_Instance.terms) { int termNumber = this.fcsi.BinarySearchLookup(t); if (termNumber > 0) { openBitSet.FastSet(termNumber); } } }
public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet set = new OpenBitSet(); int limit = docBase + reader.MaxDoc; for (; index < docs.Length; index++) { int docId = docs[index]; if (docId > limit) { break; } set.Set(docId - docBase); } docBase = limit; return(set.IsEmpty() ? null : set); }
public override SpanFilterResult BitSpans(IndexReader reader) { OpenBitSet bits = new OpenBitSet(reader.MaxDoc()); Lucene.Net.Search.Spans.Spans spans = query.GetSpans(reader); System.Collections.IList tmp = new System.Collections.ArrayList(20); int currentDoc = - 1; SpanFilterResult.PositionInfo currentInfo = null; while (spans.Next()) { int doc = spans.Doc(); bits.Set(doc); if (currentDoc != doc) { currentInfo = new SpanFilterResult.PositionInfo(doc); tmp.Add(currentInfo); currentDoc = doc; } currentInfo.AddPosition(spans.Start(), spans.End()); } return new SpanFilterResult(bits, tmp); }
/// <summary> Returns a DocIdSet with documents that should be /// permitted in search results. /// </summary> //@Override public override DocIdSet GetDocIdSet(IndexReader reader) { TermEnum enumerator = query.GetEnum(reader); try { // if current term in enum is null, the enum is empty -> shortcut if (enumerator.Term() == null) { return(DocIdSet.EMPTY_DOCIDSET); } // else fill into a OpenBitSet OpenBitSet bitSet = new OpenBitSet(reader.MaxDoc()); new AnonymousClassTermGenerator1(bitSet, this).Generate(query, reader, enumerator); return(bitSet); } finally { enumerator.Close(); } }
public override SpanFilterResult BitSpans(IndexReader reader) { OpenBitSet bits = new OpenBitSet(reader.MaxDoc()); Lucene.Net.Search.Spans.Spans spans = query.GetSpans(reader); System.Collections.IList tmp = new System.Collections.ArrayList(20); int currentDoc = -1; SpanFilterResult.PositionInfo currentInfo = null; while (spans.Next()) { int doc = spans.Doc(); bits.Set(doc); if (currentDoc != doc) { currentInfo = new SpanFilterResult.PositionInfo(doc); tmp.Add(currentInfo); currentDoc = doc; } currentInfo.AddPosition(spans.Start(), spans.End()); } return(new SpanFilterResult(bits, tmp)); }
public override SpanFilterResult BitSpans(IndexReader reader, IState state) { OpenBitSet bits = new OpenBitSet(reader.MaxDoc); Lucene.Net.Search.Spans.Spans spans = internalQuery.GetSpans(reader, state); IList <SpanFilterResult.PositionInfo> tmp = new List <SpanFilterResult.PositionInfo>(20); int currentDoc = -1; SpanFilterResult.PositionInfo currentInfo = null; while (spans.Next(state)) { int doc = spans.Doc(); bits.Set(doc); if (currentDoc != doc) { currentInfo = new SpanFilterResult.PositionInfo(doc); tmp.Add(currentInfo); currentDoc = doc; } currentInfo.AddPosition(spans.Start(), spans.End()); } return(new SpanFilterResult(bits, tmp)); }
private void CheckExpecteds(BitSet expecteds) { IndexReader r = DirectoryReader.Open(dir); //Perhaps not the most efficient approach but meets our //needs here. IBits liveDocs = MultiFields.GetLiveDocs(r); for (int i = 0; i < r.MaxDoc; i++) { if (liveDocs is null || liveDocs.Get(i)) { string sval = r.Document(i).Get(FIELD_RECORD_ID); if (sval != null) { int val = Convert.ToInt32(sval, CultureInfo.InvariantCulture); Assert.IsTrue(expecteds.FastGet(val), "Did not expect document #" + val); expecteds.FastClear(val); } } } r.Dispose(); Assert.AreEqual(0, expecteds.Cardinality, "Should have 0 docs remaining "); }
internal AssertingPostingsConsumer(PostingsConsumer @in, FieldInfo fieldInfo, OpenBitSet visitedDocs) { this.@in = @in; this.fieldInfo = fieldInfo; this.VisitedDocs = visitedDocs; }
private void InitBlock(OpenBitSet bits, QueryWrapperFilter enclosingInstance) { this.bits = bits; this.enclosingInstance = enclosingInstance; }
public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bits = new OpenBitSet((reader.MaxDoc() % 64 == 0 ? reader.MaxDoc() / 64 : reader.MaxDoc() / 64 + 1) * 64); new IndexSearcher(reader).Search(query, new AnonymousClassHitCollector2(bits, this)); return bits; }
private void InitBlock(OpenBitSet bitSet, PrefixFilter enclosingInstance) { this.bitSet = bitSet; this.enclosingInstance = enclosingInstance; }
internal SimpleTextMutableBits(BitSet bits, int size) : base(bits, size) { }
/// <summary> /// Dumps an <see cref="FST{T}"/> to a GraphViz's <c>dot</c> language description /// for visualization. Example of use: /// /// <code> /// using (TextWriter sw = new StreamWriter("out.dot")) /// { /// Util.ToDot(fst, sw, true, true); /// } /// </code> /// /// and then, from command line: /// /// <code> /// dot -Tpng -o out.png out.dot /// </code> /// /// <para/> /// Note: larger FSTs (a few thousand nodes) won't even /// render, don't bother. If the FST is > 2.1 GB in size /// then this method will throw strange exceptions. /// <para/> /// See also <a href="http://www.graphviz.org/">http://www.graphviz.org/</a>. /// </summary> /// <param name="sameRank"> /// If <c>true</c>, the resulting <c>dot</c> file will try /// to order states in layers of breadth-first traversal. This may /// mess up arcs, but makes the output FST's structure a bit clearer. /// </param> /// <param name="labelStates"> /// If <c>true</c> states will have labels equal to their offsets in their /// binary format. Expands the graph considerably. /// </param> public static void ToDot <T>(FST <T> fst, TextWriter @out, bool sameRank, bool labelStates) { const string expandedNodeColor = "blue"; // this is the start arc in the automaton (from the epsilon state to the first state // with outgoing transitions. FST.Arc <T> startArc = fst.GetFirstArc(new FST.Arc <T>()); // A queue of transitions to consider for the next level. IList <FST.Arc <T> > thisLevelQueue = new JCG.List <FST.Arc <T> >(); // A queue of transitions to consider when processing the next level. IList <FST.Arc <T> > nextLevelQueue = new JCG.List <FST.Arc <T> >(); nextLevelQueue.Add(startArc); //System.out.println("toDot: startArc: " + startArc); // A list of states on the same level (for ranking). IList <int?> sameLevelStates = new JCG.List <int?>(); // A bitset of already seen states (target offset). BitSet seen = new BitSet(); seen.Set((int)startArc.Target); // Shape for states. const string stateShape = "circle"; const string finalStateShape = "doublecircle"; // Emit DOT prologue. @out.Write("digraph FST {\n"); @out.Write(" rankdir = LR; splines=true; concentrate=true; ordering=out; ranksep=2.5; \n"); if (!labelStates) { @out.Write(" node [shape=circle, width=.2, height=.2, style=filled]\n"); } EmitDotState(@out, "initial", "point", "white", ""); T NO_OUTPUT = fst.Outputs.NoOutput; var r = fst.GetBytesReader(); // final FST.Arc<T> scratchArc = new FST.Arc<>(); { string stateColor; if (fst.IsExpandedTarget(startArc, r)) { stateColor = expandedNodeColor; } else { stateColor = null; } bool isFinal; T finalOutput; if (startArc.IsFinal) { isFinal = true; finalOutput = startArc.NextFinalOutput.Equals(NO_OUTPUT) ? default(T) : startArc.NextFinalOutput; } else { isFinal = false; finalOutput = default(T); } EmitDotState(@out, Convert.ToString(startArc.Target), isFinal ? finalStateShape : stateShape, stateColor, finalOutput == null ? "" : fst.Outputs.OutputToString(finalOutput)); } @out.Write(" initial -> " + startArc.Target + "\n"); int level = 0; while (nextLevelQueue.Count > 0) { // we could double buffer here, but it doesn't matter probably. //System.out.println("next level=" + level); thisLevelQueue.AddRange(nextLevelQueue); nextLevelQueue.Clear(); level++; @out.Write("\n // Transitions and states at level: " + level + "\n"); while (thisLevelQueue.Count > 0) { FST.Arc <T> arc = thisLevelQueue[thisLevelQueue.Count - 1]; thisLevelQueue.RemoveAt(thisLevelQueue.Count - 1); //System.out.println(" pop: " + arc); if (FST <T> .TargetHasArcs(arc)) { // scan all target arcs //System.out.println(" readFirstTarget..."); long node = arc.Target; fst.ReadFirstRealTargetArc(arc.Target, arc, r); //System.out.println(" firstTarget: " + arc); while (true) { //System.out.println(" cycle arc=" + arc); // Emit the unseen state and add it to the queue for the next level. if (arc.Target >= 0 && !seen.Get((int)arc.Target)) { /* * boolean isFinal = false; * T finalOutput = null; * fst.readFirstTargetArc(arc, scratchArc); * if (scratchArc.isFinal() && fst.targetHasArcs(scratchArc)) { * // target is final * isFinal = true; * finalOutput = scratchArc.output == NO_OUTPUT ? null : scratchArc.output; * System.out.println("dot hit final label=" + (char) scratchArc.label); * } */ string stateColor; if (fst.IsExpandedTarget(arc, r)) { stateColor = expandedNodeColor; } else { stateColor = null; } string finalOutput; if (arc.NextFinalOutput != null && !arc.NextFinalOutput.Equals(NO_OUTPUT)) { finalOutput = fst.Outputs.OutputToString(arc.NextFinalOutput); } else { finalOutput = ""; } EmitDotState(@out, Convert.ToString(arc.Target), stateShape, stateColor, finalOutput); // To see the node address, use this instead: //emitDotState(out, Integer.toString(arc.target), stateShape, stateColor, String.valueOf(arc.target)); seen.Set((int)arc.Target); nextLevelQueue.Add((new FST.Arc <T>()).CopyFrom(arc)); sameLevelStates.Add((int)arc.Target); } string outs; if (!arc.Output.Equals(NO_OUTPUT)) { outs = "/" + fst.Outputs.OutputToString(arc.Output); } else { outs = ""; } if (!FST <T> .TargetHasArcs(arc) && arc.IsFinal && !arc.NextFinalOutput.Equals(NO_OUTPUT)) { // Tricky special case: sometimes, due to // pruning, the builder can [sillily] produce // an FST with an arc into the final end state // (-1) but also with a next final output; in // this case we pull that output up onto this // arc outs = outs + "/[" + fst.Outputs.OutputToString(arc.NextFinalOutput) + "]"; } string arcColor; if (arc.Flag(FST.BIT_TARGET_NEXT)) { arcColor = "red"; } else { arcColor = "black"; } Debug.Assert(arc.Label != FST.END_LABEL); @out.Write(" " + node + " -> " + arc.Target + " [label=\"" + PrintableLabel(arc.Label) + outs + "\"" + (arc.IsFinal ? " style=\"bold\"" : "") + " color=\"" + arcColor + "\"]\n"); // Break the loop if we're on the last arc of this state. if (arc.IsLast) { //System.out.println(" break"); break; } fst.ReadNextRealArc(arc, r); } } } // Emit state ranking information. if (sameRank && sameLevelStates.Count > 1) { @out.Write(" {rank=same; "); foreach (int state in sameLevelStates) { @out.Write(state + "; "); } @out.Write(" }\n"); } sameLevelStates.Clear(); } // Emit terminating state (always there anyway). @out.Write(" -1 [style=filled, color=black, shape=doublecircle, label=\"\"]\n\n"); @out.Write(" {rank=sink; -1 }\n"); @out.Write("}\n"); @out.Flush(); }
public AnonymousClassHitCollector2(OpenBitSet bits, QueryWrapperFilter enclosingInstance) { InitBlock(bits, enclosingInstance); }
internal AnonymousClassPrefixGenerator2(OpenBitSet bitSet, PrefixFilter enclosingInstance, Lucene.Net.Index.Term Param1) : base(Param1) { InitBlock(bitSet, enclosingInstance); }
public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet set = new OpenBitSet(); int limit = docBase + reader.MaxDoc; for (; index < docs.Length; index++) { int docId = docs[index]; if (docId > limit) break; set.Set(docId - docBase); } docBase = limit; return set.IsEmpty() ? null : set; }
/// <summary> Returns a DocIdSet with documents that should be /// permitted in search results. /// </summary> public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bits = new OpenBitSet(reader.MaxDoc()); TermEnum enumerator = (null != lowerTerm && collator == null ? reader.Terms(new Term(fieldName, lowerTerm)) : reader.Terms(new Term(fieldName))); try { if (enumerator.Term() == null) { return bits; } TermDocs termDocs = reader.TermDocs(); try { if (collator != null) { do { Term term = enumerator.Term(); if (term != null && term.Field().Equals(fieldName)) { if ((lowerTerm == null || (includeLower ? collator.Compare(term.Text(), lowerTerm) >= 0 : collator.Compare(term.Text(), lowerTerm) > 0)) && (upperTerm == null || (includeUpper ? collator.Compare(term.Text(), upperTerm) <= 0 : collator.Compare(term.Text(), upperTerm) < 0))) { // term in range, lookup docs termDocs.Seek(enumerator.Term()); while (termDocs.Next()) { bits.Set(termDocs.Doc()); } } } } while (enumerator.Next()); } else // null collator; using Unicode code point ordering { bool checkLower = false; if (!includeLower) // make adjustments to set to exclusive checkLower = true; do { Term term = enumerator.Term(); if (term != null && term.Field().Equals(fieldName)) { if (!checkLower || null == lowerTerm || String.CompareOrdinal(term.Text(), lowerTerm) > 0) { checkLower = false; if (upperTerm != null) { int compare = String.CompareOrdinal(upperTerm, term.Text()); /* if beyond the upper term, or is exclusive and * this is equal to the upper term, break out */ if ((compare < 0) || (!includeUpper && compare == 0)) { break; } } /* we have a good term, find the docs */ termDocs.Seek(enumerator.Term()); while (termDocs.Next()) { bits.Set(termDocs.Doc()); } } } else { break; } } while (enumerator.Next()); } } finally { termDocs.Close(); } } finally { enumerator.Close(); } return bits; }
/// <summary> Returns a BitSet with true for documents which should be /// permitted in search results, and false for those that should /// not. /// </summary> /// <deprecated> Use {@link #GetDocIdSet(IndexReader)} instead. /// </deprecated> //@Override //[Obsolete("Use GetDocIdSet(IndexReader) instead.")] //public override System.Collections.BitArray Bits(IndexReader reader) //{ // TermEnum enumerator = query.GetEnum(reader); // try // { // System.Collections.BitArray bitSet = new System.Collections.BitArray((reader.MaxDoc() % 64 == 0?reader.MaxDoc() / 64:reader.MaxDoc() / 64 + 1) * 64); // new AnonymousClassTermGenerator(bitSet, this).Generate(query, reader, enumerator); // return bitSet; // } // finally // { // enumerator.Close(); // } //} /// <summary> Returns a DocIdSet with documents that should be /// permitted in search results. /// </summary> //@Override public override DocIdSet GetDocIdSet(IndexReader reader) { TermEnum enumerator = query.GetEnum(reader); try { // if current term in enum is null, the enum is empty -> shortcut if (enumerator.Term() == null) return DocIdSet.EMPTY_DOCIDSET; // else fill into a OpenBitSet OpenBitSet bitSet = new OpenBitSet(reader.MaxDoc()); new AnonymousClassTermGenerator1(bitSet, this).Generate(query, reader, enumerator); return bitSet; } finally { enumerator.Close(); } }
internal SimpleTextBits(BitSet bits, int size) { this.bits = bits; this.size = size; }
/// <summary> Returns a DocIdSet with documents that should be /// permitted in search results. /// </summary> public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bits = new OpenBitSet(reader.MaxDoc()); TermEnum enumerator = (null != lowerTerm && collator == null ? reader.Terms(new Term(fieldName, lowerTerm)) : reader.Terms(new Term(fieldName))); try { if (enumerator.Term() == null) { return(bits); } TermDocs termDocs = reader.TermDocs(); try { if (collator != null) { do { Term term = enumerator.Term(); if (term != null && term.Field().Equals(fieldName)) { if ((lowerTerm == null || (includeLower ? collator.Compare(term.Text(), lowerTerm) >= 0 : collator.Compare(term.Text(), lowerTerm) > 0)) && (upperTerm == null || (includeUpper ? collator.Compare(term.Text(), upperTerm) <= 0 : collator.Compare(term.Text(), upperTerm) < 0))) { // term in range, lookup docs termDocs.Seek(enumerator.Term()); while (termDocs.Next()) { bits.Set(termDocs.Doc()); } } } }while (enumerator.Next()); } else // null collator; using Unicode code point ordering { bool checkLower = false; if (!includeLower) // make adjustments to set to exclusive { checkLower = true; } do { Term term = enumerator.Term(); if (term != null && term.Field().Equals(fieldName)) { if (!checkLower || null == lowerTerm || String.CompareOrdinal(term.Text(), lowerTerm) > 0) { checkLower = false; if (upperTerm != null) { int compare = String.CompareOrdinal(upperTerm, term.Text()); /* if beyond the upper term, or is exclusive and * this is equal to the upper term, break out */ if ((compare < 0) || (!includeUpper && compare == 0)) { break; } } /* we have a good term, find the docs */ termDocs.Seek(enumerator.Term()); while (termDocs.Next()) { bits.Set(termDocs.Doc()); } } } else { break; } }while (enumerator.Next()); } } finally { termDocs.Close(); } } finally { enumerator.Close(); } return(bits); }
public override DocIdSet GetDocIdSet(IndexReader reader) { OpenBitSet bitSet = new OpenBitSet((reader.MaxDoc() % 64 == 0 ? reader.MaxDoc() / 64 : reader.MaxDoc() / 64 + 1) * 64); new AnonymousClassPrefixGenerator2(bitSet, this, prefix).Generate(reader); return bitSet; }
public AnonymousClassIntRangeBuilder(int lower, int upper, bool useBitSet, Lucene.Net.Util.OpenBitSet bits, System.Collections.IEnumerator neededBounds, TestNumericUtils enclosingInstance) { InitBlock(lower, upper, useBitSet, bits, neededBounds, enclosingInstance); }