public Token GetNextToken()
{
var token = new Token()
{
StartColumn = _currentColumn + 1, StartLine = _currentLine
};
TokenType lastErrorToken = TokenType.Error;
if (_currentInputIndex >= _inputCode.Length)
{
token.TokenType = TokenType.EOF;
return(token);
}
TokenType type = TokenType.Intermediate;
int stateID = 0;
int previousCol = 0;
int previousRow = 0;
bool startPosSet = false;
_dfa.Begin();
do
{
char currentChar = _currentInputIndex < _inputCode.Length ? _inputCode.ElementAt(_currentInputIndex) : '~'; // Insert EOF tag if past end of input
if (!_dfa.IsCharacterInAlphabet(currentChar))
{
_currentInputIndex++;
continue;
}
if (!startPosSet && !char.IsWhiteSpace(currentChar))
{
token.StartColumn = _currentColumn + 1;
token.StartLine = _currentLine;
startPosSet = true;
}
previousCol = _currentColumn;
previousRow = _currentLine;
if (currentChar == '\n')
{
_currentColumn = 0;
_currentLine++;
}
else
{
_currentColumn++;
}
if (stateID == 1 && currentChar == '~')
{
type = TokenType.EOF;
continue;
}
lastErrorToken = _dfa.GetErrorType(stateID);
stateID = _dfa.GetNextStateID(currentChar);
type = _dfa.GetStateType(stateID);
token.Lexeme += currentChar;
_currentInputIndex++;
}while (type == TokenType.Intermediate);
if (_dfa.IsBacktrackingState(stateID))
{
_currentInputIndex--;
_currentColumn = previousCol;
_currentLine = previousRow;
token.Lexeme = token.Lexeme.Remove(token.Lexeme.Length - 1, 1);
// undo col row position change
}
token.Lexeme = token.Lexeme.Trim().Replace("~", "");
//token.StartLine = _currentLine;
//token.StartColumn = _currentColumn;
token.TokenType = type;
if (token.TokenType == TokenType.Identifier)
{
var keywordMapping = _dfa.GetKeywordMapping();
TokenType keywordToken = TokenType.Intermediate;
if (keywordMapping.TryGetValue(token.Lexeme, out keywordToken))
{
token.TokenType = keywordToken;
}
}
if (token.TokenType == TokenType.Error)
{
token.TokenType = lastErrorToken;
}
return(token);
}
public bool IsCommentToken(TokenType tokenType)
{
return(tokenType == TokenType.Comment ||
tokenType == TokenType.BlockComment);
}