/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="fieldName"></param>
/// <param name="reader"></param>
/// <returns></returns>
public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
//create the tokenizer
TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader);
//add in filters
result = new StandardFilter(result); // first normalize the StandardTokenizer
result = new LowerCaseFilter(result); // makes sure everything is lower case
result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); // use the default list of Stop Words, provided by the StopAnalyzer class.
result = new SynonymFilter(result, SynonymEngine); // injects the synonyms.
//return the built token stream.
return(result);
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="fieldName"></param>
/// <param name="reader"></param>
/// <returns></returns>
public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
//create the tokenizer
TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader);
//add in filters
result = new StandardFilter(result); // first normalize the StandardTokenizer
result = new LowerCaseFilter(result);// makes sure everything is lower case
result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); // use the default list of Stop Words, provided by the StopAnalyzer class.
result = new SynonymFilter(result, SynonymEngine); // injects the synonyms.
//return the built token stream.
return result;
}
/// <summary>
///
/// </summary>
/// <remarks></remarks>
/// <seealso cref=""/>
/// <param name="fieldName"></param>
/// <param name="reader"></param>
/// <returns></returns>
public override TokenStream TokenStream(String fieldName, System.IO.TextReader reader)
{
TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
result = new StopFilter(true, result, stoptable);
//result = new GermanStemFilter(result, excltable);
//result = new PorterStemFilter(result);
result = new SynonymFilter(result, SynonymEngine); // injects the synonyms.
return result;
}
