/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="fieldName"></param> /// <param name="reader"></param> /// <returns></returns> public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader) { //create the tokenizer TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader); //add in filters result = new StandardFilter(result); // first normalize the StandardTokenizer result = new LowerCaseFilter(result); // makes sure everything is lower case result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); // use the default list of Stop Words, provided by the StopAnalyzer class. result = new SynonymFilter(result, SynonymEngine); // injects the synonyms. //return the built token stream. return(result); }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="fieldName"></param> /// <param name="reader"></param> /// <returns></returns> public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader) { //create the tokenizer TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader); //add in filters result = new StandardFilter(result); // first normalize the StandardTokenizer result = new LowerCaseFilter(result);// makes sure everything is lower case result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); // use the default list of Stop Words, provided by the StopAnalyzer class. result = new SynonymFilter(result, SynonymEngine); // injects the synonyms. //return the built token stream. return result; }
/// <summary> /// /// </summary> /// <remarks></remarks> /// <seealso cref=""/> /// <param name="fieldName"></param> /// <param name="reader"></param> /// <returns></returns> public override TokenStream TokenStream(String fieldName, System.IO.TextReader reader) { TokenStream result = new StandardTokenizer(Lucene.Net.Util.Version.LUCENE_30, reader); result = new StandardFilter(result); result = new LowerCaseFilter(result); result = new StopFilter(true, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); result = new StopFilter(true, result, stoptable); //result = new GermanStemFilter(result, excltable); //result = new PorterStemFilter(result); result = new SynonymFilter(result, SynonymEngine); // injects the synonyms. return result; }