protected internal override TokenStreamComponents CreateComponents(string fieldName)
{
StandardTokenizer src = new StandardTokenizer();
src.MaxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
TokenStream tok = new StandardFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
tok = new ASCIIFoldingFilter(tok);
return(new TokenStreamComponents(src, tok));
}
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
filters = new NGramTokenFilter(TEST_VERSION_CURRENT, filters, 2, 2);
return new TokenStreamComponents(tokenizer, filters);
}
protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
{
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
filters = new EdgeNGramTokenFilter(Version.LUCENE_43, filters, EdgeNGramTokenFilter.Side.FRONT, 2, 15);
return new TokenStreamComponents(tokenizer, filters);
}
