public Token GetNextToken()
{
if (curCharIndex >= inputText.Length)
{
return new Token()
{
TokenType = TokenType.EndOfFile
};
}
char curChar = inputText[curCharIndex];
if (char.IsWhiteSpace(curChar))
{
if (curChar == '\n') curLineNumber++;
curCharIndex++;
return new Token()
{
Lexeme = curChar.ToString(),
TokenType = TokenType.WhiteSpace
};
}
State curState = startState;
State lastFinal = null;
StringBuilder lexeme = new StringBuilder();
while (curCharIndex < inputText.Length)
{
curChar = inputText[curCharIndex];
//if (char.IsWhiteSpace(curChar))
//{
// if (curState == startState)
// {
// curCharIndex++;
// return new Token()
// {
// Lexeme = curChar.ToString(),
// TokenType = TokenType.WhiteSpace
// };
// }
// else break;
//}
if (!stateTransitionTable.ContainsKey(curState) || !stateTransitionTable[curState].ContainsKey(curChar))
break;
if (curChar == '\n') curLineNumber++;
curState = stateTransitionTable[curState][curChar];
if (curState.IsFinal)
lastFinal = curState;
lexeme.Append(curChar);
curCharIndex++;
}
if (lastFinal == null || lastFinal.MatchedSymbol == "number" && char.IsLetter(curChar))
{
return new TokenError()
{
Lexeme = curChar.ToString(),
TokenType = TokenType.Error,
ErrorLineNumber = curLineNumber
};
}
Token token = new Token()
{
Lexeme = lexeme.ToString(),
TokenType = symbolToTokenDictionary[lastFinal.MatchedSymbol]
};
ChangeTokenTypeIfReserved(token);
return token;
}
private void ChangeTokenTypeIfReserved(Token token)
{
if (reservedWordsTokens.ContainsKey(token.Lexeme))
token.TokenType = reservedWordsTokens[token.Lexeme];
}
