public override TokenStream TokenStream(String fieldName, TextReader reader)
{
TokenStream stream = new StandardTokenizer(LuceneVersion.LUCENE_29, reader);
stream = new LowerCaseFilter(stream);
stream = new HunspellStemFilter(stream, _dictionary);
return(stream);
}
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream stream = base.TokenStream(fieldName, reader);
using (var affixStream = GenerateStreamFromString(Encoding.UTF8.GetString(Properties.Resources.sk_SK_aff)))
using (var dictionaryStream = GenerateStreamFromString(Properties.Resources.sk_SK_dic))
{
var dict = new HunspellDictionary(affixStream, dictionaryStream);
stream = new HunspellStemFilter(stream, dict);
}
return(stream);
}
/// <summary>
/// Simple test for KeywordAttribute </summary>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testKeywordAttribute() throws java.io.IOException
public virtual void testKeywordAttribute()
{
MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"));
tokenizer.EnableChecks = true;
HunspellStemFilter filter = new HunspellStemFilter(tokenizer, dictionary);
assertTokenStreamContents(filter, new string[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
// assert with keyword marker
tokenizer = new MockTokenizer(new StringReader("lucene is awesome"));
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), dictionary);
assertTokenStreamContents(filter, new string[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
}
/// <summary>
/// simple test for longestOnly option </summary>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testLongestOnly() throws java.io.IOException
public virtual void testLongestOnly()
{
MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"));
tokenizer.EnableChecks = true;
HunspellStemFilter filter = new HunspellStemFilter(tokenizer, dictionary, true, true);
assertTokenStreamContents(filter, new string[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
}
public override TokenStream TokenStream(String fieldName, TextReader reader) {
TokenStream stream = new StandardTokenizer(LuceneVersion.LUCENE_29, reader);
stream = new LowerCaseFilter(stream);
stream = new HunspellStemFilter(stream, _dictionary);
return stream;
}
