public Token NextToken()
{
if (peekedWords.Count > 0)
{
return(peekedWords.Dequeue());
}
Token res = new Token();
char peekChar = stream.Peek();
while (stream.PeekEquals("//"))
{
stream.DumpUntil('\n');
stream.Next();
peekChar = stream.Peek();
}
while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t')
{
if (peekChar == '\n')
{
TabIndex = 0;
StartOfLine = true;
}
else if (peekChar == '\t' && StartOfLine)
{
TabIndex++;
}
stream.Next();
peekChar = stream.Peek();
}
res.row = stream.Row;
res.column = stream.Column;
if (StartOfLine)
{
res.type = TokenTypes.Punctuation_NewLine;
res.value = null;
StartOfLine = false;
}
else if (peekChar == 0)
{
res.type = TokenTypes.EOF;
res.value = null;
}
else if (IsPunctuation(peekChar))
{
res.type = punctuationDictionary[stream.Next()];
res.value = null;
}
else if (char.IsDigit(peekChar))
{
res.type = TokenTypes.Number;
res.value = ReadNumber();
}
else if (peekChar == '"')
{
res.type = TokenTypes.String;
res.value = ReadString();
}
else if (IsOperatorStart(peekChar))
{
res.type = operatorDictionary[ReadOperator()];
res.value = null;
}
else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters)))
{
res.type = TokenTypes.Keyword;
res.value = stream.NextOnly(IsKeywordCharacters);
}
else if (IsIdentifierStart(peekChar))
{
res.type = TokenTypes.Identifier;
res.value = stream.NextOnly(IsIdentifier);
}
else
{
throw new Exception("Unknown thing..");
}
return(res);
}
