protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer tokenizer;
TokenStream result;
if (matchVersion.OnOrAfter(LuceneVersion.LUCENE_48))
{
tokenizer = new HMMChineseTokenizer(reader);
result = tokenizer;
}
else
{
#pragma warning disable 612, 618
tokenizer = new SentenceTokenizer(reader);
result = new WordTokenFilter(tokenizer);
#pragma warning restore 612, 618
}
// result = new LowerCaseFilter(result);
// LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
// The porter stemming is too strict, this is not a bug, this is a feature:)
result = new PorterStemFilter(result);
if (stopWords.Any())
{
result = new StopFilter(matchVersion, result, stopWords);
}
return(new TokenStreamComponents(tokenizer, result));
}
/// <summary>
/// Creates a <see cref="Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <see cref="TextReader"/>.
/// </summary>
/// <returns>
/// A <see cref="Analyzer.TokenStreamComponents"/> built from an <see cref="StandardTokenizer"/>
/// filtered with <see cref="StandardFilter"/>, <see cref="LowerCaseFilter"/>, <see cref="StopFilter"/>,
/// <see cref="SetKeywordMarkerFilter"/> if a stem excusion set is provided and <see cref="StempelFilter"/>.
/// </returns>
protected override TokenStreamComponents CreateComponents(string fieldName,
TextReader reader)
{
Tokenizer source = new StandardTokenizer(m_matchVersion, reader);
TokenStream result = new StandardFilter(m_matchVersion, source);
result = new LowerCaseFilter(m_matchVersion, result);
result = new StopFilter(m_matchVersion, result, m_stopwords);
if (stemExclusionSet.Any())
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new StempelFilter(result, new StempelStemmer(stemTable));
return(new TokenStreamComponents(source, result));
}
/// <summary>
/// Creates a
/// <see cref="TokenStreamComponents"/>
/// which tokenizes all the text in the provided <see cref="TextReader"/>.
/// </summary>
/// <returns> A
/// <see cref="TokenStreamComponents"/>
/// built from an <see cref="StandardTokenizer"/> filtered with
/// <see cref="StandardFilter"/>, <see cref="TurkishLowerCaseFilter"/>,
/// <see cref="StopFilter"/>, <see cref="SetKeywordMarkerFilter"/> if a stem
/// exclusion set is provided and <see cref="SnowballFilter"/>. </returns>
protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer source = new StandardTokenizer(m_matchVersion, reader);
TokenStream result = new StandardFilter(m_matchVersion, source);
if (m_matchVersion.OnOrAfter(LuceneVersion.LUCENE_48))
{
result = new ApostropheFilter(result);
}
result = new TurkishLowerCaseFilter(result);
result = new StopFilter(m_matchVersion, result, m_stopwords);
if (stemExclusionSet.Any())
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new SnowballFilter(result, new TurkishStemmer());
return(new TokenStreamComponents(source, result));
}
/// <summary>
/// Creates a
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// which tokenizes all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> A
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from an <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="EnglishPossessiveFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>
/// , <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided and <seealso cref="PorterStemFilter"/>. </returns>
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
// prior to this we get the classic behavior, standardfilter does it for us.
if (matchVersion.OnOrAfter(LuceneVersion.LUCENE_31))
{
result = new EnglishPossessiveFilter(matchVersion, result);
}
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (stemExclusionSet.Any())
{
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
}
result = new PorterStemFilter(result);
return(new TokenStreamComponents(source, result));
}