private TokenAst ReadToken()
{
if (_bufferedTokens.Count > 0)
{
TokenAst tkn = _bufferedTokens[0];
_bufferedTokens.RemoveAt(0);
return(tkn);
}
SkipWhiteSpaces();
SetTokenStartLocation();
if (_source.EOF())
{
return(TokenAst.Create(Grammar.Eof, _context, _source.TokenStart, string.Empty, Grammar.Eof.Name));
}
TerminalList terms = SelectTerminals(_source.CurrentChar);
TokenAst result = MatchTerminals(terms);
if (result != null && !result.IsError())
{
_source.Position = _source.TokenStart.Position + result.Length;
return(result);
}
if (result == null)
{
result = Grammar.CreateSyntaxErrorToken(_context, _source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);
}
return(result);
}
public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
{
if (!source.MatchSymbol(_startSymbol, false))
{
return(null);
}
source.Position += _startSymbol.Length;
while (!source.EOF())
{
int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);
if (firstCharPos < 0)
{
source.Position = source.Text.Length;
if (_isLineComment)
{
return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
}
else
{
return(Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block"));
}
}
source.Position = firstCharPos;
if (source.MatchSymbol(_endSymbol, false))
{
source.Position += _endSymbol.Length;
return(TokenAst.Create(this, context, source.TokenStart, source.GetLexeme()));
}
source.Position++;
}
throw new NotSupportedException();
}
