Пример #1
0
        public Token GetNextToken()
        {
            var token = new Token()
            {
                StartColumn = _currentColumn + 1, StartLine = _currentLine
            };
            TokenType lastErrorToken = TokenType.Error;

            if (_currentInputIndex >= _inputCode.Length)
            {
                token.TokenType = TokenType.EOF;
                return(token);
            }

            TokenType type        = TokenType.Intermediate;
            int       stateID     = 0;
            int       previousCol = 0;
            int       previousRow = 0;
            bool      startPosSet = false;

            _dfa.Begin();
            do
            {
                char currentChar = _currentInputIndex < _inputCode.Length ? _inputCode.ElementAt(_currentInputIndex) : '~'; // Insert EOF tag if past end of input
                if (!_dfa.IsCharacterInAlphabet(currentChar))
                {
                    _currentInputIndex++;
                    continue;
                }

                if (!startPosSet && !char.IsWhiteSpace(currentChar))
                {
                    token.StartColumn = _currentColumn + 1;
                    token.StartLine   = _currentLine;
                    startPosSet       = true;
                }

                previousCol = _currentColumn;
                previousRow = _currentLine;
                if (currentChar == '\n')
                {
                    _currentColumn = 0;
                    _currentLine++;
                }
                else
                {
                    _currentColumn++;
                }

                if (stateID == 1 && currentChar == '~')
                {
                    type = TokenType.EOF;
                    continue;
                }

                lastErrorToken = _dfa.GetErrorType(stateID);
                stateID        = _dfa.GetNextStateID(currentChar);
                type           = _dfa.GetStateType(stateID);

                token.Lexeme += currentChar;
                _currentInputIndex++;
            }while (type == TokenType.Intermediate);

            if (_dfa.IsBacktrackingState(stateID))
            {
                _currentInputIndex--;
                _currentColumn = previousCol;
                _currentLine   = previousRow;
                token.Lexeme   = token.Lexeme.Remove(token.Lexeme.Length - 1, 1);
                // undo col row position change
            }

            token.Lexeme = token.Lexeme.Trim().Replace("~", "");
            //token.StartLine = _currentLine;
            //token.StartColumn = _currentColumn;
            token.TokenType = type;

            if (token.TokenType == TokenType.Identifier)
            {
                var       keywordMapping = _dfa.GetKeywordMapping();
                TokenType keywordToken   = TokenType.Intermediate;
                if (keywordMapping.TryGetValue(token.Lexeme, out keywordToken))
                {
                    token.TokenType = keywordToken;
                }
            }

            if (token.TokenType == TokenType.Error)
            {
                token.TokenType = lastErrorToken;
            }

            return(token);
        }
Пример #2
0
 public bool IsCommentToken(TokenType tokenType)
 {
     return(tokenType == TokenType.Comment ||
            tokenType == TokenType.BlockComment);
 }