public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
TokenStream result = new HebrewTokenizer(reader, PrefixTree);
// Niqqud normalization
result = new NiqqudFilter(result);
// TODO: should we ignoreCase in StopFilter?
result = new StopFilter(enableStopPositionIncrements, result, STOP_WORDS_SET);
// TODO: Apply LowerCaseFilter to NonHebrew tokens only
result = new LowerCaseFilter(result);
if (suffixByTokenType != null && suffixByTokenType.Count > 0)
{
result = new AddSuffixFilter(result, suffixByTokenType);
}
return(result);
}
public override TokenStream TokenStream(string fieldName, System.IO.TextReader reader)
{
TokenStream result = new HebrewTokenizer(reader, PrefixTree);
// Niqqud normalization
result = new NiqqudFilter(result);
// TODO: should we ignoreCase in StopFilter?
result = new StopFilter(enableStopPositionIncrements, result, STOP_WORDS_SET);
// TODO: Apply LowerCaseFilter to NonHebrew tokens only
result = new LowerCaseFilter(result);
if (suffixByTokenType != null && suffixByTokenType.Count > 0)
result = new AddSuffixFilter(result, suffixByTokenType);
return result;
}