protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
stream = new JapanesePartOfSpeechStopFilter(m_matchVersion, stream, stoptags);
stream = new CJKWidthFilter(stream);
stream = new StopFilter(m_matchVersion, stream, m_stopwords);
stream = new JapaneseKatakanaStemFilter(stream);
stream = new LowerCaseFilter(m_matchVersion, stream);
return(new TokenStreamComponents(tokenizer, stream));
}
public override TokenStream Create(TokenStream stream)
{
// if stoptags is null, it means the file is empty
if (stopTags != null)
{
#pragma warning disable 612, 618
TokenStream filter = new JapanesePartOfSpeechStopFilter(m_luceneMatchVersion, enablePositionIncrements, stream, stopTags);
#pragma warning restore 612, 618
return(filter);
}
else
{
return(stream);
}
}
