public Tokenizer(TextReader input, bool ignoreCase)
{
this._stringDfaMatcher = new StringDFAMatcher(ignoreCase);
this._nfaMatcher = new NFAMatcher(ignoreCase);
this._regExpMatcher = new RegExpMatcher(ignoreCase);
this._buffer = new ReaderBuffer(input);
}
public void Reset(TextReader input)
{
//this.buffer.Dispose();
this._buffer = new ReaderBuffer(input);
this._previousToken = null;
this._lastMatch.Clear();
}
public override void Match(ReaderBuffer buffer, TokenMatch match)
{
TokenPattern res = _automaton.Match(buffer, IgnoreCase);
if (res != null)
{
match.Update(res.Pattern.Length, res);
}
}
public override void Match(ReaderBuffer buffer, TokenMatch match)
{
for (int i = 0; i < _regExps.Length; i++)
{
int length = _regExps[i].Match(buffer);
if (length > 0)
{
match.Update(length, Patterns[i]);
}
}
}
public override int Match(ReaderBuffer buffer)
{
if (_matcher == null)
{
_matcher = _regExp.Matcher(buffer);
}
else
{
_matcher.Reset(buffer);
}
return(_matcher.MatchFromBeginning() ? _matcher.Length() : 0);
}
public TokenPattern Match(ReaderBuffer buffer, bool caseInsensitive)
{
TokenPattern result = null;
DFAState state;
int pos = 0;
var c = buffer.Peek(0);
if (c < 0)
{
return(null);
}
if (caseInsensitive)
{
c = Char.ToLower((char)c);
}
if (c < 128)
{
state = _ascii[c];
if (state == null)
{
return(null);
}
else if (state.Value != null)
{
result = state.Value;
}
pos++;
}
else
{
state = _nonAscii;
}
while ((c = buffer.Peek(pos)) >= 0)
{
state = state.Tree.Find((char)c, caseInsensitive);
if (state == null)
{
break;
}
else if (state.Value != null)
{
result = state.Value;
}
pos++;
}
return(result);
}
public int Match(ReaderBuffer buffer, TokenMatch match)
{
int length = 0;
int pos = 1;
NFAState state;
// The first step of the match loop has been unrolled and
// optimized for performance below.
this._queue.Clear();
var peekChar = buffer.Peek(0);
if (0 <= peekChar && peekChar < 128)
{
state = this._initialChar[peekChar];
if (state != null)
{
this._queue.AddLast(state);
}
}
if (peekChar >= 0)
{
this._initial.MatchTransitions((char)peekChar, this._queue, true);
}
this._queue.MarkEnd();
peekChar = buffer.Peek(1);
// The remaining match loop processes all subsequent states
while (!this._queue.Empty)
{
if (this._queue.Marked)
{
pos++;
peekChar = buffer.Peek(pos);
this._queue.MarkEnd();
}
state = this._queue.RemoveFirst();
if (state.Value != null)
{
match.Update(pos, state.Value);
}
if (peekChar >= 0)
{
state.MatchTransitions((char)peekChar, this._queue, false);
}
}
return(length);
}
public override int Match(ReaderBuffer buffer)
{
Match m;
// Ugly hack since .NET doesn't have a flag for when the
// end of the input string was encountered...
buffer.Peek(1024 * 16);
// Also, there is no API to limit the search to the specified
// position, so we double-check the index afterwards instead.
m = _reg.Match(buffer.ToString(), buffer.Position);
if (m.Success && m.Index == buffer.Position)
{
return(m.Length);
}
else
{
return(0);
}
}
public abstract int Match(ReaderBuffer buffer);
public override void Match(ReaderBuffer buffer, TokenMatch match)
{
_automaton.Match(buffer, match);
}
public abstract void Match(ReaderBuffer buffer, TokenMatch match);