public override bool IncrementToken() { if (orginal != null) { termAtt.SetTermBuffer(orginal, 0, orginal.Length); orginal = null; return(true); } if (!input.IncrementToken()) { return(false); } orginal = new char[termAtt.TermLength()]; Array.Copy(termAtt.TermBuffer(), 0, orginal, 0, termAtt.TermLength()); if (stemmer.Stem(termAtt.TermBuffer(), 0, termAtt.TermLength())) { termAtt.SetTermBuffer(stemmer.ResultBuffer, 0, stemmer.ResultLength); if (CharArrayIsEqual(orginal, termAtt.TermBuffer())) { orginal = null; } } return(true); }
/** Returns the next word in the stream. * @throws IOException If a problem occurs * @return The word */ public override bool IncrementToken() { while (true) { if (receivedText.Length == 0) { if (input.IncrementToken()) { receivedText.Append(termAtt.TermBuffer()); receivedText.Length = termAtt.TermLength(); } } if (receivedText.Length == 0) { return(false); } while (true) { string emittedText = GetNextPart(); if (emittedText.Length > 0 && !seen.Contains(emittedText)) { termAtt.SetTermBuffer(emittedText.ToCharArray(), 0, emittedText.Length); offsetAtt.SetOffset(0, emittedText.Length); seen.Add(emittedText); return(true); } if (emittedText.Length <= 0) { break; //return false; } } } /* * while (true) * { * //New token ? * if (receivedText.Length == 0) * { * receivedToken = input.Next(); * newToken = true; * if (receivedToken == null) return false; * receivedText.Append(receivedToken.TermText()); * } * String emittedText = GetNextPart(); * if (emittedText.Length > 0) * { * termAtt.SetTermBuffer(emittedText.ToString().ToCharArray(), receivedToken.StartOffset(), receivedToken.EndOffset()); * offsetAtt.SetOffset(0, emittedText.Length); * * if (newToken) posIncrAtt.SetPositionIncrement(receivedToken.GetPositionIncrement()); * else posIncrAtt.SetPositionIncrement(0); * * return true; * } * } */ }