public override Tokenizer Create(AttributeSource.AttributeFactory factory, TextReader input) { UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(m_luceneMatchVersion, factory, input); tokenizer.MaxTokenLength = maxTokenLength; return(tokenizer); }
public TokenStreamComponentsAnonymousInnerClassHelper(UAX29URLEmailAnalyzer outerInstance, UAX29URLEmailTokenizer src, TokenStream tok, TextReader reader) : base(src, tok) { this.outerInstance = outerInstance; this.reader = reader; this.src = src; }
protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader) { UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(m_matchVersion, reader); src.MaxTokenLength = maxTokenLength; TokenStream tok = new StandardFilter(m_matchVersion, src); tok = new LowerCaseFilter(m_matchVersion, tok); tok = new StopFilter(m_matchVersion, tok, m_stopwords); return(new TokenStreamComponentsAnonymousInnerClassHelper(this, src, tok, reader)); }