public Tokenizer(TextReader input, bool ignoreCase)
{
this.ignoreCase = false;
this.regexpMatchers = new ArrayList();
this.stringMatcher = new StringTokenMatcher(ignoreCase);
this.input = new LookAheadReader(input);
this.ignoreCase = ignoreCase;
}
public void Reset(TextReader input)
{
this.input.Close();
this.input = new LookAheadReader(input);
this.previousToken = null;
this.stringMatcher.Reset();
var num = this.regexpMatchers.Count - 1;
for (var i = 0; i <= num; i++)
{
((RegExpTokenMatcher)this.regexpMatchers[i]).Reset(this.input);
}
}
public object MatchFrom(LookAheadReader input, int pos, bool caseInsensitive)
{
object result = null;
var c = input.Peek(pos);
var flag = this.tree != null && c >= 0;
if (flag)
{
var state = this.tree.Find(Convert.ToChar(c), caseInsensitive);
var flag2 = state != null;
if (flag2)
{
result = RuntimeHelpers.GetObjectValue(state.MatchFrom(input, pos + 1, caseInsensitive));
}
}
return(Interaction.IIf(result == null, RuntimeHelpers.GetObjectValue(this.value), RuntimeHelpers.GetObjectValue(result)));
}
public void Reset(LookAheadReader input)
{
this.matcher.Reset(input);
}
public RegExpTokenMatcher(TokenPattern pattern, bool ignoreCase, LookAheadReader input)
{
this.pattern = pattern;
this.regExp = new RegExp(pattern.Pattern, ignoreCase);
this.matcher = this.regExp.Matcher(input);
}
public bool Match(LookAheadReader input)
{
this.Reset();
this.tokenPattern = (TokenPattern)this.start.MatchFrom(input, 0, this.ignoreCase);
return(this.tokenPattern != null);
}