/*
* Creates a {@link TokenStream} which tokenizes all the text in the provided
* {@link Reader}.
*
* @return A {@link TokenStream} built from a {@link StandardTokenizer}
* filtered with {@link StandardFilter}, {@link StopFilter},
* {@link FrenchStemFilter} and {@link LowerCaseFilter}
*/
public override sealed TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
result, stoptable);
result = new FrenchStemFilter(result, excltable);
// Convert to lowercase after stemming!
result = new LowerCaseFilter(result);
return(result);
}
public override TokenStream TokenStream(string fieldName, TextReader reader) {
TokenStream result = new StandardTokenizer(this._luceneVersion, reader);
result = new StandardFilter(result);
result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(this._luceneVersion),
result,
CharArraySet.UnmodifiableSet(new CharArraySet((IEnumerable<string>)FRENCH_STOP_WORDS, false))
);
result = new FrenchStemFilter(result, CharArraySet.EMPTY_SET);
// Convert to lowercase after stemming!
result = new LowerCaseFilter(result);
result = new ASCIIFoldingFilter(result);
return result;
}
/// <summary>
/// Creates
/// <see cref="TokenStreamComponents"/>
/// used to tokenize all the text in the provided <see cref="TextReader"/>.
/// </summary>
/// <returns> <see cref="TokenStreamComponents"/>
/// built from a <see cref="StandardTokenizer"/> filtered with
/// <see cref="StandardFilter"/>, <see cref="ElisionFilter"/>,
/// <see cref="LowerCaseFilter"/>, <see cref="StopFilter"/>,
/// <see cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided, and <see cref="FrenchLightStemFilter"/> </returns>
///
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
#pragma warning disable 612, 618
if (m_matchVersion.OnOrAfter(LuceneVersion.LUCENE_31))
#pragma warning restore 612, 618
{
Tokenizer source = new StandardTokenizer(m_matchVersion, reader);
TokenStream result = new StandardFilter(m_matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(m_matchVersion, result);
result = new StopFilter(m_matchVersion, result, m_stopwords);
if (excltable.Count > 0)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
#pragma warning disable 612, 618
if (m_matchVersion.OnOrAfter(LuceneVersion.LUCENE_36))
#pragma warning restore 612, 618
{
result = new FrenchLightStemFilter(result);
}
else
{
result = new SnowballFilter(result, new Tartarus.Snowball.Ext.FrenchStemmer());
}
return(new TokenStreamComponents(source, result));
}
else
{
Tokenizer source = new StandardTokenizer(m_matchVersion, reader);
TokenStream result = new StandardFilter(m_matchVersion, source);
result = new StopFilter(m_matchVersion, result, m_stopwords);
if (excltable.Count > 0)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
#pragma warning disable 612, 618
result = new FrenchStemFilter(result);
#pragma warning restore 612, 618
// Convert to lowercase after stemming!
return(new TokenStreamComponents(source, new LowerCaseFilter(m_matchVersion, result)));
}
}
/// <summary>
/// Creates a TokenStream which tokenizes all the text in the provided Reader.
/// </summary>
/// <returns>
/// A TokenStream build from a StandardTokenizer filtered with
/// StandardFilter, StopFilter, FrenchStemFilter and LowerCaseFilter
/// </returns>
public override TokenStream TokenStream(String fieldName, TextReader reader)
{
if (fieldName == null)
{
throw new ArgumentException("fieldName must not be null");
}
if (reader == null)
{
throw new ArgumentException("readermust not be null");
}
TokenStream result = new StandardTokenizer(reader);
result = new StandardFilter(result);
result = new StopFilter(result, stoptable);
result = new FrenchStemFilter(result, excltable);
// Convert to lowercase after stemming!
result = new LowerCaseFilter(result);
return(result);
}
/// <summary>
/// Creates
/// <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// used to tokenize all the text in the provided <seealso cref="Reader"/>.
/// </summary>
/// <returns> <seealso cref="org.apache.lucene.analysis.Analyzer.TokenStreamComponents"/>
/// built from a <seealso cref="StandardTokenizer"/> filtered with
/// <seealso cref="StandardFilter"/>, <seealso cref="ElisionFilter"/>,
/// <seealso cref="LowerCaseFilter"/>, <seealso cref="StopFilter"/>,
/// <seealso cref="SetKeywordMarkerFilter"/> if a stem exclusion set is
/// provided, and <seealso cref="FrenchLightStemFilter"/> </returns>
///
public override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
#pragma warning disable 612, 618
if (matchVersion.OnOrAfter(LuceneVersion.LUCENE_31))
#pragma warning restore 612, 618
{
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if (excltable.Count > 0)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
#pragma warning disable 612, 618
if (matchVersion.OnOrAfter(LuceneVersion.LUCENE_36))
#pragma warning restore 612, 618
{
result = new FrenchLightStemFilter(result);
}
else
{
result = new SnowballFilter(result, new Tartarus.Snowball.Ext.FrenchStemmer());
}
return new TokenStreamComponents(source, result);
}
else
{
Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, stopwords);
if (excltable.Count > 0)
{
result = new SetKeywordMarkerFilter(result, excltable);
}
#pragma warning disable 612, 618
result = new FrenchStemFilter(result);
#pragma warning restore 612, 618
// Convert to lowercase after stemming!
return new TokenStreamComponents(source, new LowerCaseFilter(matchVersion, result));
}
}
/*
* Creates a {@link TokenStream} which tokenizes all the text in the provided
* {@link Reader}.
*
* @return A {@link TokenStream} built from a {@link StandardTokenizer}
* filtered with {@link StandardFilter}, {@link StopFilter},
* {@link FrenchStemFilter} and {@link LowerCaseFilter}
*/
public override sealed TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
result, stoptable);
result = new FrenchStemFilter(result, excltable);
// Convert to lowercase after stemming!
result = new LowerCaseFilter(result);
return result;
}
/// <summary>
/// Creates a TokenStream which tokenizes all the text in the provided Reader.
/// </summary>
/// <returns>
/// A TokenStream build from a StandardTokenizer filtered with
/// StandardFilter, StopFilter, FrenchStemFilter and LowerCaseFilter
/// </returns>
public override TokenStream TokenStream( String fieldName, TextReader reader )
{
if (fieldName==null) throw new ArgumentException("fieldName must not be null");
if (reader==null) throw new ArgumentException("readermust not be null");
TokenStream result = new StandardTokenizer( reader );
result = new StandardFilter( result );
result = new StopFilter( result, stoptable );
result = new FrenchStemFilter( result, excltable );
// Convert to lowercase after stemming!
result = new LowerCaseFilter( result );
return result;
}
