public TokenStream TokenStream(string fieldName, TextReader reader)
{
var tokenizer = new JieBaTokenizer(reader, _mode);
var tokenStream = (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_48, tokenizer);
tokenStream.AddAttribute <ICharTermAttribute>();
tokenStream.AddAttribute <IOffsetAttribute>();
return(tokenStream);
}
protected override TokenStreamComponents CreateComponents(string filedName, TextReader reader)
{
var tokenizer = new JieBaTokenizer(reader, _mode);
var tokenStream = (TokenStream) new LowerCaseFilter(LuceneVersion.LUCENE_48, tokenizer);
tokenStream.AddAttribute <ICharTermAttribute>();
tokenStream.AddAttribute <IOffsetAttribute>();
return(new TokenStreamComponents(tokenizer, tokenStream));
}