/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="ElisionFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>,
/// <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided, and <seealso cref="FrenchLightStemFilter"/> </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
if (matchVersion.onOrAfter(Version.LUCENE_36))
{
result = new FrenchLightStemFilter(result);
}
else
{
result = new SnowballFilter(result, new org.tartarus.snowball.ext.FrenchStemmer());
}
return(new TokenStreamComponents(source, result));
}
else
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, stopwords);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
result = new FrenchStemFilter(result);
// Convert to lowercase after stemming!
return(new TokenStreamComponents(source, new LowerCaseFilter(matchVersion, result)));
}
}
/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="ElisionFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>,
/// <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided, and <seealso cref="FrenchLightStemFilter"/> </returns>
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
if (matchVersion.onOrAfter(Version.LUCENE_31))
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
if (matchVersion.onOrAfter(Version.LUCENE_36))
{
result = new FrenchLightStemFilter(result);
}
else
{
result = new SnowballFilter(result, new org.tartarus.snowball.ext.FrenchStemmer());
}
return new TokenStreamComponents(source, result);
}
else
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer source = new org.apache.lucene.analysis.standard.StandardTokenizer(matchVersion, reader);
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, stopwords);
if (!excltable.Empty)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
result = new FrenchStemFilter(result);
// Convert to lowercase after stemming!
return new TokenStreamComponents(source, new LowerCaseFilter(matchVersion, result));
}
}
