Example #1
0
        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
        {
            TokenStream result = new LetterDigitTokenizer(reader);

            //result = new LowerCaseFilter(result);
            return(result);
        }
Example #2
0
        public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
        {
            Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();

            if (tokenizer == null)
            {
                tokenizer = new LetterDigitTokenizer(reader);
                SetPreviousTokenStream(tokenizer);
            }
            else
            {
                tokenizer.Reset(reader);
            }
            return(tokenizer);
        }