public virtual void TestIncrementingPositions()
{
Analyzer analyzer = new WhitespaceAnalyzer();
TokenStream ts = analyzer.TokenStream("Field", new System.IO.StringReader("one two three four five"));
while (true)
{
Token token = ts.Next();
if (token == null)
{
break;
}
Assert.AreEqual(1, token.GetPositionIncrement(), token.TermText());
}
}
/// <summary>Returns the next input Token, after being stemmed </summary>
public override Token Next()
{
Token token = input.Next();
if (token == null)
{
return(null);
}
stemmer.SetCurrent(token.TermText());
try
{
stemMethod.Invoke(stemmer, (System.Object[])EMPTY_ARGS);
}
catch (System.Exception e)
{
throw new System.SystemException(e.ToString());
}
Token newToken = new Token(stemmer.GetCurrent(), token.StartOffset(), token.EndOffset(), token.Type());
newToken.SetPositionIncrement(token.GetPositionIncrement());
return(newToken);
}
internal override void processFields(Fieldable[] fields,
int count)
{
fieldState.reset(docState.doc.GetBoost());
int maxFieldLength = docState.maxFieldLength;
bool doInvert = consumer.start(fields, count);
for (int i = 0; i < count; i++)
{
Fieldable field = fields[i];
// TODO FI: this should be "genericized" to querying
// consumer if it wants to see this particular field
// tokenized.
if (field.IsIndexed() && doInvert)
{
if (fieldState.length > 0)
{
fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name);
}
if (!field.IsTokenized())
{ // un-tokenized field
string stringValue = field.StringValue();
int valueLength = stringValue.Length;
Token token = perThread.localToken.Reinit(stringValue, fieldState.offset, fieldState.offset + valueLength);
bool success = false;
try
{
consumer.add(token);
success = true;
}
finally
{
if (!success)
{
docState.docWriter.SetAborting();
}
}
fieldState.offset += valueLength;
fieldState.length++;
fieldState.position++;
}
else
{ // tokenized field
TokenStream stream;
TokenStream streamValue = field.TokenStreamValue();
if (streamValue != null)
{
stream = streamValue;
}
else
{
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
System.IO.TextReader reader; // find or make Reader
System.IO.TextReader readerValue = field.ReaderValue();
if (readerValue != null)
{
reader = readerValue;
}
else
{
string stringValue = field.StringValue();
if (stringValue == null)
{
throw new System.ArgumentException("field must have either TokenStream, string or Reader value");
}
perThread.stringReader.Init(stringValue);
reader = perThread.stringReader;
}
// Tokenize field and add to postingTable
stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader);
}
// reset the TokenStream to the first token
stream.Reset();
try
{
int offsetEnd = fieldState.offset - 1;
Token localToken = perThread.localToken;
for (; ;)
{
// If we hit an exception in stream.next below
// (which is fairly common, eg if analyzer
// chokes on a given document), then it's
// non-aborting and (above) this one document
// will be marked as deleted, but still
// consume a docID
Token token = stream.Next(localToken);
if (token == null)
{
break;
}
fieldState.position += (token.GetPositionIncrement() - 1);
bool success = false;
try
{
// If we hit an exception in here, we abort
// all buffered documents since the last
// flush, on the likelihood that the
// internal state of the consumer is now
// corrupt and should not be flushed to a
// new segment:
consumer.add(token);
success = true;
}
finally
{
if (!success)
{
docState.docWriter.SetAborting();
}
}
fieldState.position++;
offsetEnd = fieldState.offset + token.EndOffset();
if (++fieldState.length >= maxFieldLength)
{
if (docState.infoStream != null)
{
docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens");
}
break;
}
}
fieldState.offset = offsetEnd + 1;
}
finally
{
stream.Close();
}
}
fieldState.boost *= field.GetBoost();
}
}
consumer.finish();
endConsumer.finish();
}
// Tokenizes the fields of a document into Postings.
private void InvertDocument(Document doc)
{
foreach (Field field in doc.Fields())
{
System.String fieldName = field.Name();
int fieldNumber = fieldInfos.FieldNumber(fieldName);
int length = fieldLengths[fieldNumber]; // length of field
int position = fieldPositions[fieldNumber]; // position in field
if (length > 0)
{
position += analyzer.GetPositionIncrementGap(fieldName);
}
int offset = fieldOffsets[fieldNumber]; // offset field
if (field.IsIndexed())
{
if (!field.IsTokenized())
{
// un-tokenized field
System.String stringValue = field.StringValue();
if (field.IsStoreOffsetWithTermVector())
{
AddPosition(fieldName, stringValue, position++, new TermVectorOffsetInfo(offset, offset + stringValue.Length));
}
else
{
AddPosition(fieldName, stringValue, position++, null);
}
offset += stringValue.Length;
length++;
}
else
{
System.IO.TextReader reader; // find or make Reader
if (field.ReaderValue() != null)
{
reader = field.ReaderValue();
}
else if (field.StringValue() != null)
{
reader = new System.IO.StringReader(field.StringValue());
}
else
{
throw new System.ArgumentException("field must have either String or Reader value");
}
// Tokenize field and add to postingTable
TokenStream stream = analyzer.TokenStream(fieldName, reader);
try
{
Token lastToken = null;
for (Token t = stream.Next(); t != null; t = stream.Next())
{
position += (t.GetPositionIncrement() - 1);
if (field.IsStoreOffsetWithTermVector())
{
AddPosition(fieldName, t.TermText(), position++, new TermVectorOffsetInfo(offset + t.StartOffset(), offset + t.EndOffset()));
}
else
{
AddPosition(fieldName, t.TermText(), position++, null);
}
lastToken = t;
if (++length > maxFieldLength)
{
if (infoStream != null)
{
infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached, ignoring following tokens");
}
break;
}
}
if (lastToken != null)
{
offset += lastToken.EndOffset() + 1;
}
}
finally
{
stream.Close();
}
}
fieldLengths[fieldNumber] = length; // save field length
fieldPositions[fieldNumber] = position; // save field position
fieldBoosts[fieldNumber] *= field.GetBoost();
fieldOffsets[fieldNumber] = offset;
}
}
}
