} // Tokenizer constructor
// Read the next token from the stream. Skip whitespace and comments.
// If there are no further tokens in the stream, return null.
protected override Token ReadNextToken()
{
// Wrap the entire function in a loop so we can come back around
// after matching a comment or whitespace.
while (true)
{
if (readToEnd)
return null;
if (buffer.atEnd)
{
readToEnd = true;
return null;
}
SrcLoc loc;
loc.lineNum = buffer.lineNum;
loc.colNum = buffer.colNum;
loc.absPosition = buffer.pos;
loc.len = 1; // this will typically be overridden later
int tokenLen, tokenInfo;
if (matcher.Match(buffer, out tokenLen, out tokenInfo))
{
switch ((TokDispatch)tokenInfo)
{
case TokDispatch.ident:
return ParseIdentifier(loc);
case TokDispatch.digit:
return ParseNumber(loc);
case TokDispatch.stringLit:
return ParseStringLiteral(loc);
case TokDispatch.op:
{
Operator token = new Operator();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return token;
}
case TokDispatch.keyword:
{
Keyword token = new Keyword();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return token;
}
case TokDispatch.reserved:
{
ReservedWord token = new ReservedWord();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return token;
}
case TokDispatch.comment:
{
SkipComment(loc);
// Now fall through to the end of the main loop so
// we will parse the next token.
break;
}
case TokDispatch.ws:
{
// This is a whitespace character, skip it. For
// efficiency, we suck up all subsequent whitespace
// at the same time (up to one buffer's worth).
SkipWhitespace();
// Now fall through to the end of the main loop so
// we will parse the next token.
break;
}
default:
Trace.Assert(false);
break;
} // switch (tokenInfo)
}
else
{
string msg = "unexpected input character '" +
new String(buffer.PeekOrZero(), 1) +
"'";
throw new ParseError(msg, loc);
}
} // while (true)
} // ReadNextToken
} // Tokenizer constructor
// Read the next token from the stream. Skip whitespace and comments.
// If there are no further tokens in the stream, return null.
protected override Token ReadNextToken()
{
// Wrap the entire function in a loop so we can come back around
// after matching a comment or whitespace.
while (true)
{
if (readToEnd)
{
return(null);
}
if (buffer.atEnd)
{
readToEnd = true;
return(null);
}
SrcLoc loc;
loc.lineNum = buffer.lineNum;
loc.colNum = buffer.colNum;
loc.absPosition = buffer.pos;
loc.len = 1; // this will typically be overridden later
int tokenLen, tokenInfo;
if (matcher.Match(buffer, out tokenLen, out tokenInfo))
{
switch ((TokDispatch)tokenInfo)
{
case TokDispatch.ident:
return(ParseIdentifier(loc));
case TokDispatch.digit:
return(ParseNumber(loc));
case TokDispatch.stringLit:
return(ParseStringLiteral(loc));
case TokDispatch.op:
{
Operator token = new Operator();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return(token);
}
case TokDispatch.keyword:
{
Keyword token = new Keyword();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return(token);
}
case TokDispatch.reserved:
{
ReservedWord token = new ReservedWord();
token.rawText = buffer.ConsumeString(tokenLen);
loc.len = tokenLen;
token.loc = loc;
return(token);
}
case TokDispatch.comment:
{
SkipComment(loc);
// Now fall through to the end of the main loop so
// we will parse the next token.
break;
}
case TokDispatch.ws:
{
// This is a whitespace character, skip it. For
// efficiency, we suck up all subsequent whitespace
// at the same time (up to one buffer's worth).
SkipWhitespace();
// Now fall through to the end of the main loop so
// we will parse the next token.
break;
}
default:
Trace.Assert(false);
break;
} // switch (tokenInfo)
}
else
{
string msg = "unexpected input character '" +
new String(buffer.PeekOrZero(), 1) +
"'";
throw new ParseError(msg, loc);
}
} // while (true)
} // ReadNextToken