public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
{
TokenStream result = new LetterDigitTokenizer(reader);
//result = new LowerCaseFilter(result);
return(result);
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new LetterDigitTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
{
tokenizer.Reset(reader);
}
return(tokenizer);
}