protected internal override string[] CreateValue(IndexReader reader, Entry entryKey, IState state)
{
System.String field = StringHelper.Intern(entryKey.field);
System.String[] retArray = new System.String[reader.MaxDoc];
TermDocs termDocs = reader.TermDocs(state);
TermEnum termEnum = reader.Terms(new Term(field), state);
try
{
do
{
Term term = termEnum.Term;
if (term == null || (System.Object)term.Field != (System.Object)field)
{
break;
}
System.String termval = term.Text;
termDocs.Seek(termEnum, state);
while (termDocs.Next(state))
{
retArray[termDocs.Doc] = termval;
}
}while (termEnum.Next(state));
}
finally
{
termDocs.Close();
termEnum.Close();
}
return(retArray);
}
internal SegmentMergeInfo(int b, TermEnum te, IndexReader r)
{
base_Renamed = b;
reader = r;
termEnum = te;
term = te.Term;
}
protected internal override float[] CreateValue(IndexReader reader, Entry entryKey, IState state)
{
Entry entry = entryKey;
System.String field = entry.field;
FloatParser parser = (FloatParser)entry.custom;
if (parser == null)
{
try
{
return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.DEFAULT_FLOAT_PARSER, state));
}
catch (System.FormatException)
{
return(wrapper.GetFloats(reader, field, Lucene.Net.Search.FieldCache_Fields.NUMERIC_UTILS_FLOAT_PARSER, state));
}
}
float[] retArray = null;
TermDocs termDocs = reader.TermDocs(state);
TermEnum termEnum = reader.Terms(new Term(field), state);
try
{
do
{
Term term = termEnum.Term;
if (term == null || (System.Object)term.Field != (System.Object)field)
{
break;
}
float termval = parser.ParseFloat(term.Text);
if (retArray == null)
{
// late init
retArray = new float[reader.MaxDoc];
}
termDocs.Seek(termEnum, state);
while (termDocs.Next(state))
{
retArray[termDocs.Doc] = termval;
}
}while (termEnum.Next(state));
}
catch (StopFillCacheException)
{
}
finally
{
termDocs.Close();
termEnum.Close();
}
if (retArray == null)
{
// no values
retArray = new float[reader.MaxDoc];
}
return(retArray);
}
public virtual void TestPhrasePrefix()
{
RAMDirectory indexStore = new RAMDirectory();
IndexWriter writer = new IndexWriter(indexStore, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
Document doc1 = new Document();
Document doc2 = new Document();
Document doc3 = new Document();
Document doc4 = new Document();
Document doc5 = new Document();
doc1.Add(new Field("body", "blueberry pie", Field.Store.YES, Field.Index.ANALYZED));
doc2.Add(new Field("body", "blueberry strudel", Field.Store.YES, Field.Index.ANALYZED));
doc3.Add(new Field("body", "blueberry pizza", Field.Store.YES, Field.Index.ANALYZED));
doc4.Add(new Field("body", "blueberry chewing gum", Field.Store.YES, Field.Index.ANALYZED));
doc5.Add(new Field("body", "piccadilly circus", Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc1, null);
writer.AddDocument(doc2, null);
writer.AddDocument(doc3, null);
writer.AddDocument(doc4, null);
writer.AddDocument(doc5, null);
writer.Optimize(null);
writer.Close();
IndexSearcher searcher = new IndexSearcher(indexStore, true, null);
//PhrasePrefixQuery query1 = new PhrasePrefixQuery();
MultiPhraseQuery query1 = new MultiPhraseQuery();
//PhrasePrefixQuery query2 = new PhrasePrefixQuery();
MultiPhraseQuery query2 = new MultiPhraseQuery();
query1.Add(new Term("body", "blueberry"));
query2.Add(new Term("body", "strawberry"));
System.Collections.ArrayList termsWithPrefix = new System.Collections.ArrayList();
IndexReader ir = IndexReader.Open((Directory)indexStore, true, null);
// this TermEnum gives "piccadilly", "pie" and "pizza".
System.String prefix = "pi";
TermEnum te = ir.Terms(new Term("body", prefix + "*"), null);
do
{
if (te.Term.Text.StartsWith(prefix))
{
termsWithPrefix.Add(te.Term);
}
}while (te.Next(null));
query1.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
query2.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
ScoreDoc[] result;
result = searcher.Search(query1, null, 1000, null).ScoreDocs;
Assert.AreEqual(2, result.Length);
result = searcher.Search(query2, null, 1000, null).ScoreDocs;
Assert.AreEqual(0, result.Length);
}
public virtual void Seek(TermEnum termEnum)
{
TermInfo ti;
// use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs
if (termEnum is SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == parent.fieldInfos)
// optimized case
ti = ((SegmentTermEnum) termEnum).TermInfo();
// punt case
else
ti = parent.tis.Get(termEnum.Term());
Seek(ti);
}
protected internal override short[] CreateValue(IndexReader reader, Entry entryKey, IState state)
{
Entry entry = entryKey;
System.String field = entry.field;
ShortParser parser = (ShortParser)entry.custom;
if (parser == null)
{
return(wrapper.GetShorts(reader, field, Lucene.Net.Search.FieldCache_Fields.DEFAULT_SHORT_PARSER, state));
}
short[] retArray = new short[reader.MaxDoc];
TermDocs termDocs = reader.TermDocs(state);
TermEnum termEnum = reader.Terms(new Term(field), state);
try
{
do
{
Term term = termEnum.Term;
if (term == null || (System.Object)term.Field != (System.Object)field)
{
break;
}
short termval = parser.ParseShort(term.Text);
termDocs.Seek(termEnum, state);
while (termDocs.Next(state))
{
retArray[termDocs.Doc] = termval;
}
}while (termEnum.Next(state));
}
catch (StopFillCacheException)
{
}
finally
{
termDocs.Close();
termEnum.Close();
}
return(retArray);
}
public virtual void Seek(TermEnum termEnum)
{
TermInfo ti;
Term term;
// use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs
if (termEnum is SegmentTermEnum && ((SegmentTermEnum) termEnum).fieldInfos == parent.core.fieldInfos)
{
// optimized case
var segmentTermEnum = ((SegmentTermEnum) termEnum);
term = segmentTermEnum.Term;
ti = segmentTermEnum.TermInfo();
}
else
{
// punt case
term = termEnum.Term;
ti = parent.core.GetTermsReader().Get(term);
}
Seek(ti, term);
}
public virtual void Seek(TermEnum termEnum)
{
in_Renamed.Seek(termEnum);
}
public ParallelTermEnum(ParallelReader enclosingInstance)
{
InitBlock(enclosingInstance);
try
{
field = Enclosing_Instance.fieldToReader.Keys.First();
}
catch (ArgumentOutOfRangeException)
{
// No fields, so keep field == null, termEnum == null
return;
}
if (field != null)
termEnum = Enclosing_Instance.fieldToReader[field].Terms();
}
public virtual void Seek(TermEnum termEnum)
{
Seek(termEnum.Term);
var multiTermEnum = termEnum as MultiTermEnum;
if (multiTermEnum != null)
{
tenum = multiTermEnum;
if (topReader != tenum.topReader)
tenum = null;
}
}
public virtual void Seek(TermEnum termEnum)
{
Seek(termEnum.Term());
if (termEnum is MultiTermEnum)
{
tenum = (MultiTermEnum) termEnum;
if (topReader != tenum.topReader)
tenum = null;
}
}
public ParallelTermEnum(ParallelReader enclosingInstance)
{
InitBlock(enclosingInstance);
IEnumerator<string> e = Enclosing_Instance.fieldToReader.Keys.GetEnumerator();
if (e.MoveNext())
{
field = e.Current;
if (field != null)
termEnum = Enclosing_Instance.fieldToReader[field].Terms();
}
}
private static void IndexDirectory(IndexWriter writer, TermEnum uidIter, DirectoryInfo dir, Operation operation) {
var entries = Directory.GetFileSystemEntries(dir.FullName);
// Sort the entries. This is important, the uidIter TermEnum is
// iterated in a forward-only fashion, requiring all files to be
// passed in ascending order.
Array.Sort(entries);
foreach (var entry in entries) {
var path = Path.Combine(dir.FullName, entry);
if (Directory.Exists(path)) {
IndexDirectory(writer, uidIter, new DirectoryInfo(path), operation);
} else if (File.Exists(path)) {
IndexFile(writer, uidIter, new FileInfo(path), operation);
}
}
}
/// <summary> Not implemented.</summary>
/// <throws> UnsupportedOperationException </throws>
public virtual void Seek(TermEnum termEnum)
{
throw new System.NotSupportedException();
}
/// <summary> Test the term index.</summary>
private Status.TermIndexStatus TestTermIndex(SegmentInfo info, SegmentReader reader)
{
Status.TermIndexStatus status = new Status.TermIndexStatus();
try
{
if (infoStream != null)
{
infoStream.Write(" test: terms, freq, prox...");
}
TermEnum termEnum = reader.Terms();
TermPositions termPositions = reader.TermPositions();
// Used only to count up # deleted docs for this term
MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
int maxDoc = reader.MaxDoc();
while (termEnum.Next())
{
status.termCount++;
Term term = termEnum.Term();
int docFreq = termEnum.DocFreq();
termPositions.Seek(term);
int lastDoc = -1;
int freq0 = 0;
status.totFreq += docFreq;
while (termPositions.Next())
{
freq0++;
int doc = termPositions.Doc();
int freq = termPositions.Freq();
if (doc <= lastDoc)
{
throw new System.SystemException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
}
if (doc >= maxDoc)
{
throw new System.SystemException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
}
lastDoc = doc;
if (freq <= 0)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
}
int lastPos = -1;
status.totPos += freq;
for (int j = 0; j < freq; j++)
{
int pos = termPositions.NextPosition();
if (pos < -1)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
if (pos < lastPos)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
}
}
// Now count how many deleted docs occurred in
// this term:
int delCount;
if (reader.HasDeletions())
{
myTermDocs.Seek(term);
while (myTermDocs.Next())
{
}
delCount = myTermDocs.delCount;
}
else
{
delCount = 0;
}
if (freq0 + delCount != docFreq)
{
throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
}
}
Msg("OK [" + status.termCount + " terms; " + status.totFreq + " terms/docs pairs; " + status.totPos + " tokens]");
}
catch (System.Exception e)
{
Msg("ERROR [" + System.Convert.ToString(e.Message) + "]");
status.error = e;
if (infoStream != null)
{
infoStream.WriteLine(e.StackTrace);
}
}
return(status);
}
public override bool Next()
{
if (termEnum == null)
return false;
// another term in this field?
if (termEnum.Next() && (object) termEnum.Term().Field() == (object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field with terms, if any
if (fieldIterator == null)
{
fieldIterator = SupportClass.CollectionsSupport.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator();
}
while (fieldIterator.MoveNext())
{
field = fieldIterator.Current;
termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field));
Term term = termEnum.Term();
if (term != null && (object) term.Field() == (object) field)
return true;
else
termEnum.Close();
}
return false; // no more fields
}
public ParallelTermEnum(ParallelReader enclosingInstance)
{
InitBlock(enclosingInstance);
try
{
field = ((System.String) Enclosing_Instance.fieldToReader.GetKey(0));
}
catch (ArgumentOutOfRangeException e)
{
// No fields, so keep field == null, termEnum == null
return;
}
if (field != null)
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms();
}
public override bool Next()
{
if (termEnum == null)
return false;
// another term in this field?
if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field with terms, if any
if (fieldIterator == null)
{
System.Collections.Comparer comparer = System.Collections.Comparer.Default;
System.Collections.SortedList newList = new System.Collections.SortedList();
if (Enclosing_Instance.fieldToReader != null)
{
if (Enclosing_Instance.fieldToReader.Count > 0)
{
int index = 0;
while (comparer.Compare(Enclosing_Instance.fieldToReader.GetKey(index), field) < 0)
index++;
for (; index < Enclosing_Instance.fieldToReader.Count; index++)
{
newList.Add(Enclosing_Instance.fieldToReader.GetKey(index), Enclosing_Instance.fieldToReader[Enclosing_Instance.fieldToReader.GetKey(index)]);
}
}
}
fieldIterator = newList.Keys.GetEnumerator();
fieldIterator.MoveNext();
System.Object generatedAux = fieldIterator.Current; // Skip field to get next one
}
while (fieldIterator.MoveNext())
{
field = ((System.String) fieldIterator.Current);
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(new Term(field));
Term term = termEnum.Term();
if (term != null && (System.Object) term.Field() == (System.Object) field)
return true;
else
termEnum.Close();
}
return false; // no more fields
}
public SeekStatus Seek(String value)
{
termEnum = reader.Terms(new Term(this.fieldName, value));
Term t = Term();
if (t == null)
return SeekStatus.END;
return (t.Text.Equals(value)) ? SeekStatus.FOUND : SeekStatus.NOT_FOUND;
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="td">Underlying TermDocs.</param>
/// <param name="termEnum">Enclosing field enum.</param>
internal TermDocUsingTermsEnumerator(TermDocs td, TermEnum termEnum)
{
this.termDocs = td;
this.termEnum = termEnum;
}
/// <summary>
/// Seel to a term in the underlying TermDocs.
/// </summary>
/// <param name="te">The point to seek to.</param>
internal void Seek(TermEnum te)
{
this.termDocs.Seek(te);
}
public override bool Next()
{
if (field == null)
return false;
bool next = termEnum.Next();
// still within field?
if (next && (System.Object) termEnum.Term().Field() == (System.Object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field, if any
field = ((System.String) SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).GetKey(0));
if (field != null)
{
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms();
return true;
}
return false; // no more fields
}
public TestTermEnum(TermEnum termEnum) : base(termEnum)
{
}
public virtual void TestPhrasePrefix()
{
RAMDirectory indexStore = new RAMDirectory();
IndexWriter writer = new IndexWriter(indexStore, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, null);
Add("blueberry pie", writer);
Add("blueberry strudel", writer);
Add("blueberry pizza", writer);
Add("blueberry chewing gum", writer);
Add("bluebird pizza", writer);
Add("bluebird foobar pizza", writer);
Add("piccadilly circus", writer);
writer.Optimize(null);
writer.Close();
IndexSearcher searcher = new IndexSearcher(indexStore, true, null);
// search for "blueberry pi*":
MultiPhraseQuery query1 = new MultiPhraseQuery();
// search for "strawberry pi*":
MultiPhraseQuery query2 = new MultiPhraseQuery();
query1.Add(new Term("body", "blueberry"));
query2.Add(new Term("body", "strawberry"));
System.Collections.ArrayList termsWithPrefix = new System.Collections.ArrayList();
IndexReader ir = IndexReader.Open((Directory)indexStore, true, null);
// this TermEnum gives "piccadilly", "pie" and "pizza".
System.String prefix = "pi";
TermEnum te = ir.Terms(new Term("body", prefix), null);
do
{
if (te.Term.Text.StartsWith(prefix))
{
termsWithPrefix.Add(te.Term);
}
}while (te.Next(null));
query1.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
Assert.AreEqual("body:\"blueberry (piccadilly pie pizza)\"", query1.ToString());
query2.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
Assert.AreEqual("body:\"strawberry (piccadilly pie pizza)\"", query2.ToString());
ScoreDoc[] result;
result = searcher.Search(query1, null, 1000, null).ScoreDocs;
Assert.AreEqual(2, result.Length);
result = searcher.Search(query2, null, 1000, null).ScoreDocs;
Assert.AreEqual(0, result.Length);
// search for "blue* pizza":
MultiPhraseQuery query3 = new MultiPhraseQuery();
termsWithPrefix.Clear();
prefix = "blue";
te = ir.Terms(new Term("body", prefix), null);
do
{
if (te.Term.Text.StartsWith(prefix))
{
termsWithPrefix.Add(te.Term);
}
}while (te.Next(null));
query3.Add((Term[])termsWithPrefix.ToArray(typeof(Term)));
query3.Add(new Term("body", "pizza"));
result = searcher.Search(query3, null, 1000, null).ScoreDocs;
Assert.AreEqual(2, result.Length); // blueberry pizza, bluebird pizza
Assert.AreEqual("body:\"(blueberry bluebird) pizza\"", query3.ToString());
// test slop:
query3.Slop = 1;
result = searcher.Search(query3, null, 1000, null).ScoreDocs;
Assert.AreEqual(3, result.Length); // blueberry pizza, bluebird pizza, bluebird foobar pizza
MultiPhraseQuery query4 = new MultiPhraseQuery();
// okay, all terms must belong to the same field
Assert.Throws <ArgumentException>(() =>
{
query4.Add(new Term("field1", "foo"));
query4.Add(new Term("field2", "foobar"));
});
searcher.Close();
indexStore.Close();
}
public virtual void Test1()
{
ParallelReader pr = new ParallelReader();
pr.Add(ir1);
pr.Add(ir2);
TermDocs td = pr.TermDocs();
TermEnum te = pr.Terms();
Assert.IsTrue(te.Next());
Assert.AreEqual("field1:brown", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field1:fox", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field1:jumps", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field1:quick", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field1:the", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field2:brown", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field2:fox", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field2:jumps", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field2:quick", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field2:the", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:dog", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:fox", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:jumps", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:lazy", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:over", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsTrue(te.Next());
Assert.AreEqual("field3:the", te.Term.ToString());
td.Seek(te.Term);
Assert.IsTrue(td.Next());
Assert.AreEqual(0, td.Doc);
Assert.IsFalse(td.Next());
Assert.IsFalse(te.Next());
}
public override bool Next()
{
if (termEnum == null)
return false;
// another term in this field?
if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field with terms, if any
if (fieldIterator == null)
{
List<string> tmpList = new List<string>();
bool m = false;
//JAVA: fieldIterator = fieldToReader.tailMap(field).keySet().iterator();
//JAVA: fieldIterator.next(); // Skip field to get next one
foreach (string key in Enclosing_Instance.fieldToReader.Keys)
{
if (key == field && m==false) m = true;
if (m) tmpList.Add(key);
}
fieldIterator = tmpList.GetEnumerator();
}
while (fieldIterator.MoveNext())
{
field = fieldIterator.Current;
termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field));
Term term = termEnum.Term();
if (term != null && (System.Object) term.Field() == (System.Object) field)
return true;
else
termEnum.Close();
}
return false; // no more fields
}
private void MergeTermInfos()
{
int base_Renamed = 0;
int readerCount = readers.Count;
for (int i = 0; i < readerCount; i++)
{
IndexReader reader = (IndexReader)readers[i];
TermEnum termEnum = reader.Terms();
SegmentMergeInfo smi = new SegmentMergeInfo(base_Renamed, termEnum, reader);
int[] docMap = smi.GetDocMap();
if (docMap != null)
{
if (docMaps == null)
{
docMaps = new int[readerCount][];
delCounts = new int[readerCount];
}
docMaps[i] = docMap;
delCounts[i] = smi.reader.MaxDoc() - smi.reader.NumDocs();
}
base_Renamed += reader.NumDocs();
if (smi.Next())
{
queue.Put(smi); // initialize queue
}
else
{
smi.Close();
}
}
SegmentMergeInfo[] match = new SegmentMergeInfo[readers.Count];
while (queue.Size() > 0)
{
int matchSize = 0; // pop matching terms
match[matchSize++] = (SegmentMergeInfo)queue.Pop();
Term term = match[0].term;
SegmentMergeInfo top = (SegmentMergeInfo)queue.Top();
while (top != null && term.CompareTo(top.term) == 0)
{
match[matchSize++] = (SegmentMergeInfo)queue.Pop();
top = (SegmentMergeInfo)queue.Top();
}
int df = MergeTermInfo(match, matchSize); // add new TermInfo
if (checkAbort != null)
{
checkAbort.Work(df / 3.0);
}
while (matchSize > 0)
{
SegmentMergeInfo smi = match[--matchSize];
if (smi.Next())
{
queue.Put(smi);
}
// restore queue
else
{
smi.Close(); // done with a segment
}
}
}
}
public override bool Next()
{
if (termEnum == null)
return false;
// another term in this field?
if (termEnum.Next() && (System.Object) termEnum.Term.Field == (System.Object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field with terms, if any
if (fieldIterator == null)
{
var newList = new List<string>();
if (Enclosing_Instance.fieldToReader != null && Enclosing_Instance.fieldToReader.Count > 0)
{
var comparer = Enclosing_Instance.fieldToReader.Comparer;
foreach(var entry in Enclosing_Instance.fieldToReader.Keys.Where(x => comparer.Compare(x, field) >= 0))
newList.Add(entry);
}
fieldIterator = newList.Skip(1).GetEnumerator(); // Skip field to get next one
}
while (fieldIterator.MoveNext())
{
field = fieldIterator.Current;
termEnum = Enclosing_Instance.fieldToReader[field].Terms(new Term(field));
Term term = termEnum.Term;
if (term != null && (System.Object) term.Field == (System.Object) field)
return true;
else
termEnum.Close();
}
return false; // no more fields
}
public virtual void Seek(TermEnum termEnum)
{
Seek(termEnum.Term());
}
public FilterTermEnum(TermEnum in_Renamed)
{
this.in_Renamed = in_Renamed;
}
public FilterTermEnum(TermEnum in_Renamed)
{
this.in_Renamed = in_Renamed;
}
public TestTermEnum(TermEnum termEnum):base(termEnum)
{
}
public virtual void Seek(TermEnum termEnum)
{
in_Renamed.Seek(termEnum);
}
/// <summary> Not implemented.</summary>
/// <throws> UnsupportedOperationException </throws>
public virtual void Seek(TermEnum termEnum)
{
throw new System.NotSupportedException();
}
private void MergeTermInfos(FormatPostingsFieldsConsumer consumer)
{
int base_Renamed = 0;
int readerCount = readers.Count;
for (int i = 0; i < readerCount; i++)
{
IndexReader reader = readers[i];
TermEnum termEnum = reader.Terms();
SegmentMergeInfo smi = new SegmentMergeInfo(base_Renamed, termEnum, reader);
int[] docMap = smi.GetDocMap();
if (docMap != null)
{
if (docMaps == null)
{
docMaps = new int[readerCount][];
delCounts = new int[readerCount];
}
docMaps[i] = docMap;
delCounts[i] = smi.reader.MaxDoc - smi.reader.NumDocs();
}
base_Renamed += reader.NumDocs();
System.Diagnostics.Debug.Assert(reader.NumDocs() == reader.MaxDoc - smi.delCount);
if (smi.Next())
{
queue.Add(smi);
}
// initialize queue
else
{
smi.Dispose();
}
}
SegmentMergeInfo[] match = new SegmentMergeInfo[readers.Count];
string currentField = null;
FormatPostingsTermsConsumer termsConsumer = null;
while (queue.Size() > 0)
{
int matchSize = 0; // pop matching terms
match[matchSize++] = queue.Pop();
Term term = match[0].term;
SegmentMergeInfo top = queue.Top();
while (top != null && term.CompareTo(top.term) == 0)
{
match[matchSize++] = queue.Pop();
top = queue.Top();
}
if ((object)currentField != (object)term.Field)
{
currentField = term.Field;
if (termsConsumer != null)
{
termsConsumer.Finish();
}
FieldInfo fieldInfo = fieldInfos.FieldInfo(currentField);
termsConsumer = consumer.AddField(fieldInfo);
omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions;
}
int df = AppendPostings(termsConsumer, match, matchSize); // add new TermInfo
checkAbort.Work(df / 3.0);
while (matchSize > 0)
{
SegmentMergeInfo smi = match[--matchSize];
if (smi.Next())
{
queue.Add(smi);
}
// restore queue
else
{
smi.Dispose(); // done with a segment
}
}
}
}
public ParallelTermEnum(ParallelReader enclosingInstance, Term term)
{
InitBlock(enclosingInstance);
field = term.Field();
IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[field]);
if (reader != null)
termEnum = reader.Terms(term);
}
public override bool Next()
{
if (termEnum == null)
return false;
// another term in this field?
if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field)
return true; // yes, keep going
termEnum.Close(); // close old termEnum
// find the next field with terms, if any
if (fieldIterator == null)
{
fieldIterator = SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator();
fieldIterator.MoveNext(); // Skip field to get next one
}
while (fieldIterator.MoveNext())
{
field = ((System.String) fieldIterator.Current);
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(new Term(field, ""));
Term term = termEnum.Term();
if (term != null && (System.Object) term.Field() == (System.Object) field)
return true;
else
termEnum.Close();
}
return false; // no more fields
}
public virtual void Seek(TermEnum termEnum)
{
Seek(termEnum.Term());
}
public ParallelTermEnum(ParallelReader enclosingInstance)
{
InitBlock(enclosingInstance);
field = ((System.String) Enclosing_Instance.fieldToReader.GetKey(0));
if (field != null)
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms();
}
private static void IndexFile(IndexWriter writer, TermEnum uidIter, FileInfo file, Operation operation)
{
if (file.FullName.EndsWith(".html") || file.FullName.EndsWith(".htm") || file.FullName.EndsWith(".txt"))
{
// We've found a file we should index.
if (operation == Operation.IncrementalReindex ||
operation == Operation.RemoveStale)
{
// We should only get here with an open uidIter.
Debug.Assert(uidIter != null, "Expected uidIter != null for operation " + operation);
var uid = HTMLDocument.Uid(file); // construct uid for doc
while (uidIter.Term != null && uidIter.Term.Field == "uid" && String.CompareOrdinal(uidIter.Term.Text, uid) < 0)
{
if (operation == Operation.RemoveStale)
{
Console.Out.WriteLine("deleting " + HTMLDocument.Uid2url(uidIter.Term.Text));
writer.DeleteDocuments(uidIter.Term);
}
uidIter.Next();
}
// The uidIter TermEnum should now be pointing at either
// 1) a null term, meaning there are no more uids to check.
// 2) a term matching the current file.
// 3) a term not matching us.
if (uidIter.Term != null && uidIter.Term.Field == "uid" && String.CompareOrdinal(uidIter.Term.Text, uid) == 0)
{
// uidIter points to the current document, we should move one
// step ahead to keep state consistant, and carry on.
uidIter.Next();
}
else if (operation == Operation.IncrementalReindex)
{
// uidIter does not point to the current document, and we're
// currently indexing documents.
var doc = HTMLDocument.Document(file);
Console.Out.WriteLine("adding " + doc.Get("path"));
writer.AddDocument(doc);
}
}
else
{
// We're doing a complete reindexing. We aren't using uidIter,
// but for completeness we assert that it's null (as expected).
Debug.Assert(uidIter == null, "Expected uidIter == null for operation == " + operation);
var doc = HTMLDocument.Document(file);
Console.Out.WriteLine("adding " + doc.Get("path"));
writer.AddDocument(doc);
}
}
}
public ParallelTermEnum(ParallelReader enclosingInstance, Term term)
{
InitBlock(enclosingInstance);
field = term.Field();
termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(term);
}
public void Seek(TermEnum termEnum)
{
throw new NotSupportedException();
}
public TermsEnumCompatibility(IndexReader reader, String fieldName)
{
this.reader = reader;
this.fieldName = string.Intern(fieldName);
this.termEnum = reader.Terms(new Term(this.fieldName));
}
