public override bool IncrementToken()
{
if (orginal != null)
{
termAtt.SetTermBuffer(orginal, 0, orginal.Length);
orginal = null;
return(true);
}
if (!input.IncrementToken())
{
return(false);
}
orginal = new char[termAtt.TermLength()];
Array.Copy(termAtt.TermBuffer(), 0, orginal, 0, termAtt.TermLength());
if (stemmer.Stem(termAtt.TermBuffer(), 0, termAtt.TermLength()))
{
termAtt.SetTermBuffer(stemmer.ResultBuffer, 0, stemmer.ResultLength);
if (CharArrayIsEqual(orginal, termAtt.TermBuffer()))
{
orginal = null;
}
}
return(true);
}
/** Returns the next word in the stream.
* @throws IOException If a problem occurs
* @return The word
*/
public override bool IncrementToken()
{
while (true)
{
if (receivedText.Length == 0)
{
if (input.IncrementToken())
{
receivedText.Append(termAtt.TermBuffer());
receivedText.Length = termAtt.TermLength();
}
}
if (receivedText.Length == 0)
{
return(false);
}
while (true)
{
string emittedText = GetNextPart();
if (emittedText.Length > 0 && !seen.Contains(emittedText))
{
termAtt.SetTermBuffer(emittedText.ToCharArray(), 0, emittedText.Length);
offsetAtt.SetOffset(0, emittedText.Length);
seen.Add(emittedText);
return(true);
}
if (emittedText.Length <= 0)
{
break;
//return false;
}
}
}
/*
* while (true)
* {
* //New token ?
* if (receivedText.Length == 0)
* {
* receivedToken = input.Next();
* newToken = true;
* if (receivedToken == null) return false;
* receivedText.Append(receivedToken.TermText());
* }
* String emittedText = GetNextPart();
* if (emittedText.Length > 0)
* {
* termAtt.SetTermBuffer(emittedText.ToString().ToCharArray(), receivedToken.StartOffset(), receivedToken.EndOffset());
* offsetAtt.SetOffset(0, emittedText.Length);
*
* if (newToken) posIncrAtt.SetPositionIncrement(receivedToken.GetPositionIncrement());
* else posIncrAtt.SetPositionIncrement(0);
*
* return true;
* }
* }
*/
}