public TokenPattern Match(ReaderBuffer buffer, bool caseInsensitive)
{
TokenPattern result = null;
DFAState state;
int pos = 0;
var c = buffer.Peek(0);
if (c < 0)
{
return(null);
}
if (caseInsensitive)
{
c = Char.ToLower((char)c);
}
if (c < 128)
{
state = _ascii[c];
if (state == null)
{
return(null);
}
else if (state.Value != null)
{
result = state.Value;
}
pos++;
}
else
{
state = _nonAscii;
}
while ((c = buffer.Peek(pos)) >= 0)
{
state = state.Tree.Find((char)c, caseInsensitive);
if (state == null)
{
break;
}
else if (state.Value != null)
{
result = state.Value;
}
pos++;
}
return(result);
}
public int Match(ReaderBuffer buffer, TokenMatch match)
{
int length = 0;
int pos = 1;
NFAState state;
// The first step of the match loop has been unrolled and
// optimized for performance below.
this._queue.Clear();
var peekChar = buffer.Peek(0);
if (0 <= peekChar && peekChar < 128)
{
state = this._initialChar[peekChar];
if (state != null)
{
this._queue.AddLast(state);
}
}
if (peekChar >= 0)
{
this._initial.MatchTransitions((char)peekChar, this._queue, true);
}
this._queue.MarkEnd();
peekChar = buffer.Peek(1);
// The remaining match loop processes all subsequent states
while (!this._queue.Empty)
{
if (this._queue.Marked)
{
pos++;
peekChar = buffer.Peek(pos);
this._queue.MarkEnd();
}
state = this._queue.RemoveFirst();
if (state.Value != null)
{
match.Update(pos, state.Value);
}
if (peekChar >= 0)
{
state.MatchTransitions((char)peekChar, this._queue, false);
}
}
return(length);
}
public override int Match(ReaderBuffer buffer)
{
Match m;
// Ugly hack since .NET doesn't have a flag for when the
// end of the input string was encountered...
buffer.Peek(1024 * 16);
// Also, there is no API to limit the search to the specified
// position, so we double-check the index afterwards instead.
m = _reg.Match(buffer.ToString(), buffer.Position);
if (m.Success && m.Index == buffer.Position)
{
return(m.Length);
}
else
{
return(0);
}
}
private Token NextToken()
{
try {
_lastMatch.Clear();
_stringDfaMatcher.Match(_buffer, _lastMatch);
_nfaMatcher.Match(_buffer, _lastMatch);
_regExpMatcher.Match(_buffer, _lastMatch);
int line;
int column;
if (_lastMatch.Length > 0)
{
line = _buffer.LineNumber;
column = _buffer.ColumnNumber;
var str = _buffer.Read(_lastMatch.Length);
return(NewToken(_lastMatch.Pattern, str, line, column));
}
else if (_buffer.Peek(0) < 0)
{
return(null);
}
else
{
line = _buffer.LineNumber;
column = _buffer.ColumnNumber;
throw new ParseException(
ParseException.ErrorType.UNEXPECTED_CHAR,
_buffer.Read(1),
line,
column);
}
} catch (IOException e) {
throw new ParseException(ParseException.ErrorType.IO,
e.Message,
-1,
-1);
}
}