protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
var seg = new JiebaSegmenter();
var tokenizer = new JiebaTokenizer(seg, reader);
TokenStream tokenStream = new LowerCaseFilter(LuceneVersion.LUCENE_48, tokenizer);
tokenStream = new WordLengthFilter(_minWordLength, _maxWordLength, tokenStream);
tokenStream = new StopFilter(LuceneVersion.LUCENE_48, tokenStream, new CharArraySet(LuceneVersion.LUCENE_48, StopWordSet, true));
return(new JiebaTokenStreamComponents(tokenizer, tokenStream));
}
public JiebaTokenStreamComponents(JiebaTokenizer tokenizer, TokenStream result)
: base(tokenizer, result)
{
this.jiebaTokenizer = tokenizer;
}
