public IEnumerable <Token> Lex(string sourceCode)
{
ICharacterStream stream = new CharacterStream(sourceCode);
var result = new List <Token>();
var lineCounter = 1;
_buffer = "";
try
{
while (stream.HasNext())
{
var current = stream.Next();
ReadOnlyCollection <Token> tokens = TryFindMarker(stream);
if (!tokens.Any())
{
tokens = TryNewLine(stream);
if (tokens.Any(token => token.Type == TokenType.NewLine))
{
++lineCounter;
}
if (!tokens.Any())
{
tokens = TryFindSpecial(stream);
if (!tokens.Any())
{
_buffer += current;
}
}
}
result.AddRange(tokens);
}
}
catch (Exception exc)
{
throw new TranslationException($"Lexer error: {exc.Message}", lineCounter, exc);
}
result.AddRange(PopBuffer());
return(result);
}