protected internal override string[] CreateValue(IndexReader reader, Entry entryKey, IState state) { System.String field = StringHelper.Intern(entryKey.field); System.String[] retArray = new System.String[reader.MaxDoc]; TermDocs termDocs = reader.TermDocs(state); TermEnum termEnum = reader.Terms(new Term(field), state); try { do { Term term = termEnum.Term; if (term == null || (System.Object)term.Field != (System.Object)field) { break; } System.String termval = term.Text; termDocs.Seek(termEnum, state); while (termDocs.Next(state)) { retArray[termDocs.Doc] = termval; } }while (termEnum.Next(state)); } finally { termDocs.Close(); termEnum.Close(); } return(retArray); }
protected internal override float[] CreateValue(IndexReader reader, Entry entryKey, IState state) { Entry entry = entryKey; System.String field = entry.field; FloatParser parser = (FloatParser)entry.custom; if (parser == null) { try { return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.DEFAULT_FLOAT_PARSER, state)); } catch (System.FormatException) { return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.NUMERIC_UTILS_FLOAT_PARSER, state)); } } float[] retArray = null; TermDocs termDocs = reader.TermDocs(state); TermEnum termEnum = reader.Terms(new Term(field), state); try { do { Term term = termEnum.Term; if (term == null || (System.Object)term.Field != (System.Object)field) { break; } float termval = parser.ParseFloat(term.Text); if (retArray == null) { // late init retArray = new float[reader.MaxDoc]; } termDocs.Seek(termEnum, state); while (termDocs.Next(state)) { retArray[termDocs.Doc] = termval; } }while (termEnum.Next(state)); } catch (StopFillCacheException) { } finally { termDocs.Close(); termEnum.Close(); } if (retArray == null) { // no values retArray = new float[reader.MaxDoc]; } return(retArray); }
protected internal override short[] CreateValue(IndexReader reader, Entry entryKey, IState state) { Entry entry = entryKey; System.String field = entry.field; ShortParser parser = (ShortParser)entry.custom; if (parser == null) { return(wrapper.GetShorts(reader, field, Lucene.Net.Search.FieldCache_Fields.DEFAULT_SHORT_PARSER, state)); } short[] retArray = new short[reader.MaxDoc]; TermDocs termDocs = reader.TermDocs(state); TermEnum termEnum = reader.Terms(new Term(field), state); try { do { Term term = termEnum.Term; if (term == null || (System.Object)term.Field != (System.Object)field) { break; } short termval = parser.ParseShort(term.Text); termDocs.Seek(termEnum, state); while (termDocs.Next(state)) { retArray[termDocs.Doc] = termval; } }while (termEnum.Next(state)); } catch (StopFillCacheException) { } finally { termDocs.Close(); termEnum.Close(); } return(retArray); }
private void PrintSegment(System.IO.StreamWriter out_Renamed, SegmentInfo si) { SegmentReader reader = SegmentReader.Get(si); for (int i = 0; i < reader.NumDocs(); i++) { out_Renamed.WriteLine(reader.Document(i)); } TermEnum tis = reader.Terms(); while (tis.Next()) { out_Renamed.Write(tis.Term()); out_Renamed.WriteLine(" DF=" + tis.DocFreq()); TermPositions positions = reader.TermPositions(tis.Term()); try { while (positions.Next()) { out_Renamed.Write(" doc=" + positions.Doc()); out_Renamed.Write(" TF=" + positions.Freq()); out_Renamed.Write(" pos="); out_Renamed.Write(positions.NextPosition()); for (int j = 1; j < positions.Freq(); j++) { out_Renamed.Write("," + positions.NextPosition()); } out_Renamed.WriteLine(""); } } finally { positions.Close(); } } tis.Close(); reader.Close(); }
private void PrintSegment(System.IO.StreamWriter out_Renamed, SegmentInfo si) { SegmentReader reader = SegmentReader.Get(true, si, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, null); for (int i = 0; i < reader.NumDocs(); i++) { out_Renamed.WriteLine(reader.Document(i, null)); } TermEnum tis = reader.Terms(null); while (tis.Next(null)) { out_Renamed.Write(tis.Term); out_Renamed.WriteLine(" DF=" + tis.DocFreq()); TermPositions positions = reader.TermPositions(tis.Term, null); try { while (positions.Next(null)) { out_Renamed.Write(" doc=" + positions.Doc); out_Renamed.Write(" TF=" + positions.Freq); out_Renamed.Write(" pos="); out_Renamed.Write(positions.NextPosition(null)); for (int j = 1; j < positions.Freq; j++) { out_Renamed.Write("," + positions.NextPosition(null)); } out_Renamed.WriteLine(""); } } finally { positions.Close(); } } tis.Close(); reader.Close(); }
public virtual void TestMultiTermDocs() { RAMDirectory ramDir1 = new RAMDirectory(); AddDoc(ramDir1, "test foo", true); RAMDirectory ramDir2 = new RAMDirectory(); AddDoc(ramDir2, "test blah", true); RAMDirectory ramDir3 = new RAMDirectory(); AddDoc(ramDir3, "test wow", true); IndexReader[] readers1 = new IndexReader[] { IndexReader.Open(ramDir1), IndexReader.Open(ramDir3) }; IndexReader[] readers2 = new IndexReader[] { IndexReader.Open(ramDir1), IndexReader.Open(ramDir2), IndexReader.Open(ramDir3) }; MultiReader mr2 = new MultiReader(readers1); MultiReader mr3 = new MultiReader(readers2); // test mixing up TermDocs and TermEnums from different readers. TermDocs td2 = mr2.TermDocs(); TermEnum te3 = mr3.Terms(new Term("body", "wow")); td2.Seek(te3); int ret = 0; // This should blow up if we forget to check that the TermEnum is from the same // reader as the TermDocs. while (td2.Next()) { ret += td2.Doc(); } td2.Close(); te3.Close(); // really a dummy assert to ensure that we got some docs and to ensure that // nothing is optimized out. Assert.IsTrue(ret > 0); }
public override bool Next() { if (termEnum == null) { return(false); } // another term in this field? if (termEnum.Next() && (System.Object)termEnum.Term().Field() == (System.Object)field) { return(true); // yes, keep going } termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { fieldIterator = SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator(); fieldIterator.MoveNext(); // Skip field to get next one } while (fieldIterator.MoveNext()) { field = ((System.String)fieldIterator.Current); termEnum = ((IndexReader)Enclosing_Instance.fieldToReader[field]).Terms(new Term(field, "")); Term term = termEnum.Term(); if (term != null && (System.Object)term.Field() == (System.Object)field) { return(true); } else { termEnum.Close(); } } return(false); // no more fields }
public override bool Next() { if (termEnum == null) { return(false); } // another term in this field? if (termEnum.Next() && (object)termEnum.Term().Field() == (object)field) { return(true); // yes, keep going } termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { fieldIterator = SupportClass.CollectionsSupport.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator(); } while (fieldIterator.MoveNext()) { field = fieldIterator.Current; termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field)); Term term = termEnum.Term(); if (term != null && (object)term.Field() == (object)field) { return(true); } else { termEnum.Close(); } } return(false); // no more fields }
public override bool Next() { if (termEnum == null) return false; // another term in this field? if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field) return true; // yes, keep going termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { System.Collections.Comparer comparer = System.Collections.Comparer.Default; System.Collections.SortedList newList = new System.Collections.SortedList(); if (Enclosing_Instance.fieldToReader != null) { if (Enclosing_Instance.fieldToReader.Count > 0) { int index = 0; while (comparer.Compare(Enclosing_Instance.fieldToReader.GetKey(index), field) < 0) index++; for (; index < Enclosing_Instance.fieldToReader.Count; index++) { newList.Add(Enclosing_Instance.fieldToReader.GetKey(index), Enclosing_Instance.fieldToReader[Enclosing_Instance.fieldToReader.GetKey(index)]); } } } fieldIterator = newList.Keys.GetEnumerator(); fieldIterator.MoveNext(); System.Object generatedAux = fieldIterator.Current; // Skip field to get next one } while (fieldIterator.MoveNext()) { field = ((System.String) fieldIterator.Current); termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(new Term(field)); Term term = termEnum.Term(); if (term != null && (System.Object) term.Field() == (System.Object) field) return true; else termEnum.Close(); } return false; // no more fields }
public override void Close() { in_Renamed.Close(); }
public override bool Next() { if (termEnum == null) return false; // another term in this field? if (termEnum.Next() && (System.Object) termEnum.Term.Field == (System.Object) field) return true; // yes, keep going termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { var newList = new List<string>(); if (Enclosing_Instance.fieldToReader != null && Enclosing_Instance.fieldToReader.Count > 0) { var comparer = Enclosing_Instance.fieldToReader.Comparer; foreach(var entry in Enclosing_Instance.fieldToReader.Keys.Where(x => comparer.Compare(x, field) >= 0)) newList.Add(entry); } fieldIterator = newList.Skip(1).GetEnumerator(); // Skip field to get next one } while (fieldIterator.MoveNext()) { field = fieldIterator.Current; termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field)); Term term = termEnum.Term; if (term != null && (System.Object) term.Field == (System.Object) field) return true; else termEnum.Close(); } return false; // no more fields }
public override bool Next() { if (termEnum == null) return false; // another term in this field? if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field) return true; // yes, keep going termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { List<string> tmpList = new List<string>(); bool m = false; //JAVA: fieldIterator = fieldToReader.tailMap(field).keySet().iterator(); //JAVA: fieldIterator.next(); // Skip field to get next one foreach (string key in Enclosing_Instance.fieldToReader.Keys) { if (key == field && m==false) m = true; if (m) tmpList.Add(key); } fieldIterator = tmpList.GetEnumerator(); } while (fieldIterator.MoveNext()) { field = fieldIterator.Current; termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field)); Term term = termEnum.Term(); if (term != null && (System.Object) term.Field() == (System.Object) field) return true; else termEnum.Close(); } return false; // no more fields }
public static void VerifyEquals(IndexReader r1, IndexReader r2, System.String idField) { Assert.AreEqual(r1.NumDocs(), r2.NumDocs()); bool hasDeletes = !(r1.MaxDoc() == r2.MaxDoc() && r1.NumDocs() == r1.MaxDoc()); int[] r2r1 = new int[r2.MaxDoc()]; // r2 id to r1 id mapping TermDocs termDocs1 = r1.TermDocs(); TermDocs termDocs2 = r2.TermDocs(); // create mapping from id2 space to id2 based on idField idField = StringHelper.Intern(idField); TermEnum termEnum = r1.Terms(new Term(idField, "")); do { Term term = termEnum.Term(); if (term == null || (System.Object)term.Field() != (System.Object)idField) { break; } termDocs1.Seek(termEnum); if (!termDocs1.Next()) { // This doc is deleted and wasn't replaced termDocs2.Seek(termEnum); Assert.IsFalse(termDocs2.Next()); continue; } int id1 = termDocs1.Doc(); Assert.IsFalse(termDocs1.Next()); termDocs2.Seek(termEnum); Assert.IsTrue(termDocs2.Next()); int id2 = termDocs2.Doc(); Assert.IsFalse(termDocs2.Next()); r2r1[id2] = id1; // verify stored fields are equivalent try { VerifyEquals(r1.Document(id1), r2.Document(id2)); } catch (System.Exception t) { System.Console.Out.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2 + " term=" + term); System.Console.Out.WriteLine(" d1=" + r1.Document(id1)); System.Console.Out.WriteLine(" d2=" + r2.Document(id2)); throw t; } try { // verify term vectors are equivalent VerifyEquals(r1.GetTermFreqVectors(id1), r2.GetTermFreqVectors(id2)); } catch (System.Exception e) { System.Console.Out.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2); TermFreqVector[] tv1 = r1.GetTermFreqVectors(id1); System.Console.Out.WriteLine(" d1=" + tv1); if (tv1 != null) { for (int i = 0; i < tv1.Length; i++) { System.Console.Out.WriteLine(" " + i + ": " + tv1[i]); } } TermFreqVector[] tv2 = r2.GetTermFreqVectors(id2); System.Console.Out.WriteLine(" d2=" + tv2); if (tv2 != null) { for (int i = 0; i < tv2.Length; i++) { System.Console.Out.WriteLine(" " + i + ": " + tv2[i]); } } throw e; } }while (termEnum.Next()); termEnum.Close(); // Verify postings TermEnum termEnum1 = r1.Terms(new Term("", "")); TermEnum termEnum2 = r2.Terms(new Term("", "")); // pack both doc and freq into single element for easy sorting long[] info1 = new long[r1.NumDocs()]; long[] info2 = new long[r2.NumDocs()]; for (; ;) { Term term1, term2; // iterate until we get some docs int len1; for (; ;) { len1 = 0; term1 = termEnum1.Term(); if (term1 == null) { break; } termDocs1.Seek(termEnum1); while (termDocs1.Next()) { int d1 = termDocs1.Doc(); int f1 = termDocs1.Freq(); info1[len1] = (((long)d1) << 32) | f1; len1++; } if (len1 > 0) { break; } if (!termEnum1.Next()) { break; } } // iterate until we get some docs int len2; for (; ;) { len2 = 0; term2 = termEnum2.Term(); if (term2 == null) { break; } termDocs2.Seek(termEnum2); while (termDocs2.Next()) { int d2 = termDocs2.Doc(); int f2 = termDocs2.Freq(); info2[len2] = (((long)r2r1[d2]) << 32) | f2; len2++; } if (len2 > 0) { break; } if (!termEnum2.Next()) { break; } } if (!hasDeletes) { Assert.AreEqual(termEnum1.DocFreq(), termEnum2.DocFreq()); } Assert.AreEqual(len1, len2); if (len1 == 0) { break; // no more terms } Assert.AreEqual(term1, term2); // sort info2 to get it into ascending docid System.Array.Sort(info2, 0, len2 - 0); // now compare for (int i = 0; i < len1; i++) { Assert.AreEqual(info1[i], info2[i]); } termEnum1.Next(); termEnum2.Next(); } }
public override bool Next() { if (termEnum == null) return false; // another term in this field? if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field) return true; // yes, keep going termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { fieldIterator = SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator(); fieldIterator.MoveNext(); // Skip field to get next one } while (fieldIterator.MoveNext()) { field = ((System.String) fieldIterator.Current); termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(new Term(field, "")); Term term = termEnum.Term(); if (term != null && (System.Object) term.Field() == (System.Object) field) return true; else termEnum.Close(); } return false; // no more fields }
public override bool Next() { if (termEnum == null) return false; // another term in this field? if (termEnum.Next() && (object) termEnum.Term().Field() == (object) field) return true; // yes, keep going termEnum.Close(); // close old termEnum // find the next field with terms, if any if (fieldIterator == null) { fieldIterator = SupportClass.CollectionsSupport.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator(); } while (fieldIterator.MoveNext()) { field = fieldIterator.Current; termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field)); Term term = termEnum.Term(); if (term != null && (object) term.Field() == (object) field) return true; else termEnum.Close(); } return false; // no more fields }