protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode); TokenStream stream = new JapaneseBaseFormFilter(tokenizer); stream = new JapanesePartOfSpeechStopFilter(m_matchVersion, stream, stoptags); stream = new CJKWidthFilter(stream); stream = new StopFilter(m_matchVersion, stream, m_stopwords); stream = new JapaneseKatakanaStemFilter(stream); stream = new LowerCaseFilter(m_matchVersion, stream); return(new TokenStreamComponents(tokenizer, stream)); }
public override TokenStream Create(TokenStream stream) { // if stoptags is null, it means the file is empty if (stopTags != null) { #pragma warning disable 612, 618 TokenStream filter = new JapanesePartOfSpeechStopFilter(m_luceneMatchVersion, enablePositionIncrements, stream, stopTags); #pragma warning restore 612, 618 return(filter); } else { return(stream); } }