public override bool IncrementToken()
{
if (!done)
{
ClearAttributes();
done = true;
int upto = 0;
char[] buffer = termAtt.TermBuffer();
while (true)
{
int length = input.Read(buffer, upto, buffer.Length - upto);
if (length == 0)
{
break;
}
upto += length;
if (upto == buffer.Length)
{
buffer = termAtt.ResizeTermBuffer(1 + buffer.Length);
}
}
termAtt.SetTermLength(upto);
finalOffset = CorrectOffset(upto);
offsetAtt.SetOffset(CorrectOffset(0), finalOffset);
return(true);
}
return(false);
}
/*
* (non-Javadoc)
*
* @see Lucene.Net.Analysis.TokenStream#next()
*/
public override bool IncrementToken()
{
ClearAttributes();
int posIncr = 1;
while (true)
{
int tokenType = scanner.GetNextToken();
if (tokenType == StandardTokenizerImpl.YYEOF)
{
return(false);
}
if (scanner.Yylength() <= maxTokenLength)
{
posIncrAtt.SetPositionIncrement(posIncr);
scanner.GetText(termAtt);
int start = scanner.Yychar();
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.TermLength()));
// This 'if' should be removed in the next release. For now, it converts
// invalid acronyms to HOST. When removed, only the 'else' part should
// remain.
if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
{
if (replaceInvalidAcronym)
{
typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]);
termAtt.SetTermLength(termAtt.TermLength() - 1); // remove extra '.'
}
else
{
typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]);
}
}
else
{
typeAtt.SetType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]);
}
return(true);
}
// When we skip a too-long term, we still increment the
// position increment
else
{
posIncr++;
}
}
}
public override bool IncrementToken()
{
if (i == TOKENS.Length)
{
return(false);
}
ClearAttributes();
termAtt.SetTermBuffer(TOKENS[i]);
offsetAtt.SetOffset(i, i);
posIncrAtt.SetPositionIncrement(INCREMENTS[i]);
i++;
return(true);
}
public override bool IncrementToken()
{
if (index == Enclosing_Instance.tokens.Length)
{
return(false);
}
else
{
ClearAttributes();
termAtt.SetTermBuffer(Enclosing_Instance.tokens[index++]);
offsetAtt.SetOffset(0, 0);
return(true);
}
}
public override bool IncrementToken()
{
if (Lucene.Net.QueryParsers.TestMultiAnalyzer.multiToken > 0)
{
termAtt.SetTermBuffer("multi" + (Lucene.Net.QueryParsers.TestMultiAnalyzer.multiToken + 1));
offsetAtt.SetOffset(prevStartOffset, prevEndOffset);
typeAtt.SetType(prevType);
posIncrAtt.SetPositionIncrement(0);
Lucene.Net.QueryParsers.TestMultiAnalyzer.multiToken--;
return(true);
}
else
{
bool next = input.IncrementToken();
if (next == false)
{
return(false);
}
prevType = typeAtt.Type();
prevStartOffset = offsetAtt.StartOffset();
prevEndOffset = offsetAtt.EndOffset();
System.String text = termAtt.Term();
if (text.Equals("triplemulti"))
{
Lucene.Net.QueryParsers.TestMultiAnalyzer.multiToken = 2;
return(true);
}
else if (text.Equals("multi"))
{
Lucene.Net.QueryParsers.TestMultiAnalyzer.multiToken = 1;
return(true);
}
else
{
return(true);
}
}
}
public override bool IncrementToken()
{
if (tokenUpto >= Enclosing_Instance.tokens.Length)
{
return(false);
}
else
{
TestToken testToken = Enclosing_Instance.tokens[tokenUpto++];
ClearAttributes();
termAtt.SetTermBuffer(testToken.text);
offsetAtt.SetOffset(testToken.startOffset, testToken.endOffset);
if (tokenUpto > 1)
{
posIncrAtt.SetPositionIncrement(testToken.pos - Enclosing_Instance.tokens[tokenUpto - 2].pos);
}
else
{
posIncrAtt.SetPositionIncrement(testToken.pos + 1);
}
return(true);
}
}
public override bool IncrementToken()
{
ClearAttributes();
int length = 0;
int start = bufferIndex;
char[] buffer = termAtt.TermBuffer();
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
dataLen = input.Read((System.Char[])ioBuffer, 0, ioBuffer.Length);
if (dataLen <= 0)
{
dataLen = 0; // so next offset += dataLen won't decrement offset
if (length > 0)
{
break;
}
else
{
return(false);
}
}
bufferIndex = 0;
}
char c = ioBuffer[bufferIndex++];
if (IsTokenChar(c))
{
// if it's a token char
if (length == 0)
{
// start of token
start = offset + bufferIndex - 1;
}
else if (length == buffer.Length)
{
buffer = termAtt.ResizeTermBuffer(1 + length);
}
buffer[length++] = Normalize(c); // buffer it, normalized
if (length == MAX_WORD_LEN)
{
// buffer overflow!
break;
}
}
else if (length > 0)
{
// at non-Letter w/ chars
break; // return 'em
}
}
termAtt.SetTermLength(length);
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + length));
return(true);
}
public void Reinit(System.String stringValue, int startOffset, int endOffset)
{
termAttribute.SetTermBuffer(stringValue);
offsetAttribute.SetOffset(startOffset, endOffset);
}
