public IEnumerable <Token> Lex(string sourceCode)
{
ICharacterStream stream = new CharacterStream(sourceCode);
var result = new List <Token>();
var lineCounter = 1;
_buffer = "";
try
{
while (stream.HasNext())
{
var current = stream.Next();
ReadOnlyCollection <Token> tokens = TryFindMarker(stream);
if (!tokens.Any())
{
tokens = TryNewLine(stream);
if (tokens.Any(token => token.Type == TokenType.NewLine))
{
++lineCounter;
}
if (!tokens.Any())
{
tokens = TryFindSpecial(stream);
if (!tokens.Any())
{
_buffer += current;
}
}
}
result.AddRange(tokens);
}
}
catch (Exception exc)
{
throw new TranslationException($"Lexer error: {exc.Message}", lineCounter, exc);
}
result.AddRange(PopBuffer());
return(result);
}
public Token NextToken()
{
if (peekedWords.Count > 0)
{
return(peekedWords.Dequeue());
}
Token res = new Token();
char peekChar = stream.Peek();
while (stream.PeekEquals("//"))
{
stream.DumpUntil('\n');
stream.Next();
peekChar = stream.Peek();
}
while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t')
{
if (peekChar == '\n')
{
TabIndex = 0;
StartOfLine = true;
}
else if (peekChar == '\t' && StartOfLine)
{
TabIndex++;
}
stream.Next();
peekChar = stream.Peek();
}
res.row = stream.Row;
res.column = stream.Column;
if (StartOfLine)
{
res.type = TokenTypes.Punctuation_NewLine;
res.value = null;
StartOfLine = false;
}
else if (peekChar == 0)
{
res.type = TokenTypes.EOF;
res.value = null;
}
else if (IsPunctuation(peekChar))
{
res.type = punctuationDictionary[stream.Next()];
res.value = null;
}
else if (char.IsDigit(peekChar))
{
res.type = TokenTypes.Number;
res.value = ReadNumber();
}
else if (peekChar == '"')
{
res.type = TokenTypes.String;
res.value = ReadString();
}
else if (IsOperatorStart(peekChar))
{
res.type = operatorDictionary[ReadOperator()];
res.value = null;
}
else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters)))
{
res.type = TokenTypes.Keyword;
res.value = stream.NextOnly(IsKeywordCharacters);
}
else if (IsIdentifierStart(peekChar))
{
res.type = TokenTypes.Identifier;
res.value = stream.NextOnly(IsIdentifier);
}
else
{
throw new Exception("Unknown thing..");
}
return(res);
}
