/// <summary>
/// </summary>
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream t = null;
t = new LetterOrDigitTokenizer(reader);
t = new LowerCaseFilter(t);
t = new ASCIIFoldingFilter(t);
t = new SingleCharTokenizer(t);
return t;
}
/// <summary>
/// </summary>
public override TokenStream TokenStream(string fieldName, TextReader reader)
{
TokenStream t = null;
t = new LetterOrDigitTokenizer(reader);
t = new LowerCaseFilter(t);
t = new ASCIIFoldingFilter(t);
t = new SingleCharTokenizer(t);
return(t);
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
Tokenizer tokenizer = (Tokenizer)GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new SingleCharTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
{
tokenizer.Reset(reader);
}
return(tokenizer);
}
public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
{
Tokenizer tokenizer = (Tokenizer) GetPreviousTokenStream();
if (tokenizer == null)
{
tokenizer = new SingleCharTokenizer(reader);
SetPreviousTokenStream(tokenizer);
}
else
tokenizer.Reset(reader);
return tokenizer;
}
public _241(string source)
{
this.tokenizer = new SingleCharTokenizer(source);
lookahead = tokenizer.nextToken();
}
public _241(string source)
{
this.tokenizer = new SingleCharTokenizer(source);
lookahead = tokenizer.nextToken();
}
public _242(string source)
{
this.tokenizer = new SingleCharTokenizer(source);
nextChar = tokenizer.nextToken();
}
public _243(string source)
{
this.tokenizer = new SingleCharTokenizer(source);
next = tokenizer.nextToken();
second = tokenizer.nextToken();
}
public _242(string source)
{
this.tokenizer = new SingleCharTokenizer(source);
nextChar = tokenizer.nextToken();
}
