public override sealed bool IncrementToken()
{
ClearAttributes();
skippedPositions = 0;
while (true)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerInterface.YYEOF)
{
return(false);
}
if (scanner.YyLength <= maxTokenLength)
{
posIncrAtt.PositionIncrement = skippedPositions + 1;
scanner.GetText(termAtt);
int start = scanner.YyChar;
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.Length));
typeAtt.Type = TOKEN_TYPES[tokenType];
return(true);
}
else
// When we skip a too-long term, we still increment the
// position increment
{
skippedPositions++;
}
}
}
/*
* (non-Javadoc)
*
* @see org.apache.lucene.analysis.TokenStream#next()
*/
public override bool IncrementToken()
{
ClearAttributes();
skippedPositions = 0;
while (true)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerInterface_Fields.YYEOF)
{
return(false);
}
if (scanner.YyLength <= maxTokenLength)
{
posIncrAtt.PositionIncrement = skippedPositions + 1;
scanner.GetText(termAtt);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int start = scanner.YyChar();
int start = scanner.YyChar;
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.Length));
// This 'if' should be removed in the next release. For now, it converts
// invalid acronyms to HOST. When removed, only the 'else' part should
// remain.
#pragma warning disable 612, 618
if (tokenType == StandardTokenizer.ACRONYM_DEP)
{
typeAtt.Type = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.HOST];
#pragma warning restore 612, 618
termAtt.Length = termAtt.Length - 1; // remove extra '.'
}
else
{
typeAtt.Type = StandardTokenizer.TOKEN_TYPES[tokenType];
}
return(true);
}
else
// When we skip a too-long term, we still increment the
// position increment
{
skippedPositions++;
}
}
}
/*
* (non-Javadoc)
*
* @see org.apache.lucene.analysis.TokenStream#next()
*/
public override bool IncrementToken()
{
ClearAttributes();
skippedPositions = 0;
while (true)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerInterface_Fields.YYEOF)
{
return(false);
}
if (scanner.YyLength <= maxTokenLength)
{
posIncrAtt.PositionIncrement = skippedPositions + 1;
scanner.GetText(termAtt);
int start = scanner.YyChar;
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.Length));
if (tokenType == ClassicTokenizer.ACRONYM_DEP)
{
typeAtt.Type = ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.HOST];
termAtt.Length = termAtt.Length - 1; // remove extra '.'
}
else
{
typeAtt.Type = ClassicTokenizer.TOKEN_TYPES[tokenType];
}
return(true);
}
else
// When we skip a too-long term, we still increment the
// position increment
{
skippedPositions++;
}
}
}