/// <summary>Adds field info for a Document. </summary>
public void Add(Document doc)
{
lock (this)
{
System.Collections.IList fields = doc.GetFields();
System.Collections.IEnumerator fieldIterator = fields.GetEnumerator();
while (fieldIterator.MoveNext())
{
Fieldable field = (Fieldable)fieldIterator.Current;
Add(field.Name(), field.IsIndexed(), field.IsTermVectorStored(), field.IsStorePositionWithTermVector(), field.IsStoreOffsetWithTermVector(), field.GetOmitNorms(), false, field.GetOmitTf());
}
}
}
internal override bool Start(Fieldable[] fields, int count)
{
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for (int i = 0; i < count; i++)
{
Fieldable field = fields[i];
if (field.IsIndexed() && field.IsTermVectorStored())
{
doVectors = true;
doVectorPositions |= field.IsStorePositionWithTermVector();
doVectorOffsets |= field.IsStoreOffsetWithTermVector();
}
}
if (doVectors)
{
if (perThread.doc == null)
{
perThread.doc = termsWriter.GetPerDoc();
perThread.doc.docID = docState.docID;
System.Diagnostics.Debug.Assert(perThread.doc.numVectorFields == 0);
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.Length());
System.Diagnostics.Debug.Assert(0 == perThread.doc.perDocTvf.GetFilePointer());
}
System.Diagnostics.Debug.Assert(perThread.doc.docID == docState.docID);
if (termsHashPerField.numPostings != 0)
{
// Only necessary if previous doc hit a
// non-aborting exception while writing vectors in
// this field:
termsHashPerField.Reset();
perThread.termsHashPerThread.Reset(false);
}
}
// TODO: only if needed for performance
//perThread.postingsCount = 0;
return(doVectors);
}
public override DocumentsWriter.DocWriter ProcessDocument()
{
consumer.StartDocument();
fieldsWriter.StartDocument();
Document doc = docState.doc;
System.Diagnostics.Debug.Assert(docFieldProcessor.docWriter.writer.TestPoint("DocumentsWriter.ThreadState.init start"));
fieldCount = 0;
int thisFieldGen = fieldGen++;
System.Collections.IList docFields = doc.GetFields();
int numDocFields = docFields.Count;
// Absorb any new fields first seen in this document.
// Also absorb any changes to fields we had already
// seen before (eg suddenly turning on norms or
// vectors, etc.):
for (int i = 0; i < numDocFields; i++)
{
Fieldable field = (Fieldable)docFields[i];
System.String fieldName = field.Name();
// Make sure we have a PerField allocated
int hashPos = fieldName.GetHashCode() & hashMask;
DocFieldProcessorPerField fp = fieldHash[hashPos];
while (fp != null && !fp.fieldInfo.name.Equals(fieldName))
{
fp = fp.next;
}
if (fp == null)
{
// TODO FI: we need to genericize the "flags" that a
// field holds, and, how these flags are merged; it
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
FieldInfo fi = fieldInfos.Add(fieldName, field.IsIndexed(), field.IsTermVectorStored(), field.IsStorePositionWithTermVector(), field.IsStoreOffsetWithTermVector(), field.GetOmitNorms(), false, field.GetOmitTf());
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
fieldHash[hashPos] = fp;
totalFieldCount++;
if (totalFieldCount >= fieldHash.Length / 2)
{
Rehash();
}
}
else
{
fp.fieldInfo.Update(field.IsIndexed(), field.IsTermVectorStored(), field.IsStorePositionWithTermVector(), field.IsStoreOffsetWithTermVector(), field.GetOmitNorms(), false, field.GetOmitTf());
}
if (thisFieldGen != fp.lastGen)
{
// First time we're seeing this field for this doc
fp.fieldCount = 0;
if (fieldCount == fields.Length)
{
int newSize = fields.Length * 2;
DocFieldProcessorPerField[] newArray = new DocFieldProcessorPerField[newSize];
Array.Copy(fields, 0, newArray, 0, fieldCount);
fields = newArray;
}
fields[fieldCount++] = fp;
fp.lastGen = thisFieldGen;
}
if (fp.fieldCount == fp.fields.Length)
{
Fieldable[] newArray = new Fieldable[fp.fields.Length * 2];
Array.Copy(fp.fields, 0, newArray, 0, fp.fieldCount);
fp.fields = newArray;
}
fp.fields[fp.fieldCount++] = field;
if (field.IsStored())
{
fieldsWriter.AddField(field, fp.fieldInfo);
}
}
// If we are writing vectors then we must visit
// fields in sorted order so they are written in
// sorted order. TODO: we actually only need to
// sort the subset of fields that have vectors
// enabled; we could save [small amount of] CPU
// here.
QuickSort(fields, 0, fieldCount - 1);
for (int i = 0; i < fieldCount; i++)
{
fields[i].consumer.ProcessFields(fields[i].fields, fields[i].fieldCount);
}
if (docState.maxTermPrefix != null && docState.infoStream != null)
{
docState.infoStream.WriteLine("WARNING: document contains at least one immense term (longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
docState.maxTermPrefix = null;
}
DocumentsWriter.DocWriter one = fieldsWriter.FinishDocument();
DocumentsWriter.DocWriter two = consumer.FinishDocument();
if (one == null)
{
return(two);
}
else if (two == null)
{
return(one);
}
else
{
PerDoc both = GetPerDoc();
both.docID = docState.docID;
System.Diagnostics.Debug.Assert(one.docID == docState.docID);
System.Diagnostics.Debug.Assert(two.docID == docState.docID);
both.one = one;
both.two = two;
return(both);
}
}
public override void ProcessFields(Fieldable[] fields, int count)
{
fieldState.Reset(docState.doc.GetBoost());
int maxFieldLength = docState.maxFieldLength;
bool doInvert = consumer.Start(fields, count);
for (int i = 0; i < count; i++)
{
Fieldable field = fields[i];
// TODO FI: this should be "genericized" to querying
// consumer if it wants to see this particular field
// tokenized.
if (field.IsIndexed() && doInvert)
{
bool anyToken;
if (fieldState.length > 0)
{
fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name);
}
if (!field.IsTokenized())
{
// un-tokenized field
System.String stringValue = field.StringValue();
int valueLength = stringValue.Length;
perThread.singleTokenTokenStream.Reinit(stringValue, 0, valueLength);
fieldState.attributeSource = perThread.singleTokenTokenStream;
consumer.Start(field);
bool success = false;
try
{
consumer.Add();
success = true;
}
finally
{
if (!success)
{
docState.docWriter.SetAborting();
}
}
fieldState.offset += valueLength;
fieldState.length++;
fieldState.position++;
anyToken = valueLength > 0;
}
else
{
// tokenized field
TokenStream stream;
TokenStream streamValue = field.TokenStreamValue();
if (streamValue != null)
{
stream = streamValue;
}
else
{
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
System.IO.TextReader reader; // find or make Reader
System.IO.TextReader readerValue = field.ReaderValue();
if (readerValue != null)
{
reader = readerValue;
}
else
{
System.String stringValue = field.StringValue();
if (stringValue == null)
{
throw new System.ArgumentException("field must have either TokenStream, String or Reader value");
}
perThread.stringReader.Init(stringValue);
reader = perThread.stringReader;
}
// Tokenize field and add to postingTable
stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader);
}
// reset the TokenStream to the first token
stream.Reset();
int startLength = fieldState.length;
// deprecated
bool allowMinus1Position = docState.allowMinus1Position;
try
{
int offsetEnd = fieldState.offset - 1;
bool hasMoreTokens = stream.IncrementToken();
fieldState.attributeSource = stream;
OffsetAttribute offsetAttribute = (OffsetAttribute)fieldState.attributeSource.AddAttribute(typeof(OffsetAttribute));
PositionIncrementAttribute posIncrAttribute = (PositionIncrementAttribute)fieldState.attributeSource.AddAttribute(typeof(PositionIncrementAttribute));
consumer.Start(field);
for (; ;)
{
// If we hit an exception in stream.next below
// (which is fairly common, eg if analyzer
// chokes on a given document), then it's
// non-aborting and (above) this one document
// will be marked as deleted, but still
// consume a docID
if (!hasMoreTokens)
{
break;
}
int posIncr = posIncrAttribute.GetPositionIncrement();
fieldState.position += posIncr;
if (allowMinus1Position || fieldState.position > 0)
{
fieldState.position--;
}
if (posIncr == 0)
{
fieldState.numOverlap++;
}
bool success = false;
try
{
// If we hit an exception in here, we abort
// all buffered documents since the last
// flush, on the likelihood that the
// internal state of the consumer is now
// corrupt and should not be flushed to a
// new segment:
consumer.Add();
success = true;
}
finally
{
if (!success)
{
docState.docWriter.SetAborting();
}
}
fieldState.position++;
offsetEnd = fieldState.offset + offsetAttribute.EndOffset();
if (++fieldState.length >= maxFieldLength)
{
if (docState.infoStream != null)
{
docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens");
}
break;
}
hasMoreTokens = stream.IncrementToken();
}
// trigger streams to perform end-of-stream operations
stream.End();
fieldState.offset += offsetAttribute.EndOffset();
anyToken = fieldState.length > startLength;
}
finally
{
stream.Close();
}
}
if (anyToken)
{
fieldState.offset += docState.analyzer.GetOffsetGap(field);
}
fieldState.boost *= field.GetBoost();
}
// LUCENE-2387: don't hang onto the field, so GC can
// reclaim
fields[i] = null;
}
consumer.Finish();
endConsumer.Finish();
}
