Exemplo n.º 1
0
        private Token GetToken()
        {
            Token token = new Token();

            while (token.Type == TokenType.None)
            {
                switch (_reader.Current)
                {
                case ' ':
                    while (_reader.Current == ' ' && !_reader.IsEOF)
                    {
                        _reader.ConsumeChar();
                    }

                    continue;

                case '\t':
                    break;

                case '#':                    // comments
                    if (_reader.Next == '[') // block comment
                    {
                        while (_reader.Current != ']' || _reader.Next != '#' && !_reader.IsEOF)
                        {
                            _reader.ConsumeChar();
                        }

                        _reader.ConsumeChar();     // consume last #
                    }
                    else
                    {
                        _reader.ConsumeLine();
                    }

                    break;

                case '\r':
                    break;

                case '\n':
                    if (_reader.IsEOF)
                    {
                        return(GetToken(TokenType.EOF));
                    }
                    break;

                case '$':
                    switch (_reader.Next)
                    {
                    case '\'':
                        //return new Token(TokenType.String, _line, _col, text: GetInterpolatedString('\''));
                        GetInterpolatedString('\'');
                        return(_interpolatedTokens.Dequeue());

                    case '"':
                        GetInterpolatedString('"');
                        return(_interpolatedTokens.Dequeue());

                    default:
                        throw new LexerException($"Unexpected token {_reader.Current} ({(int) _reader.Current})");
                    }

                case '\'':
                    return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('\'')));

                case '"':
                    return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('"')));

                case TokenValues.Colon:
                    if (_reader.Next == TokenValues.Colon)
                    {
                        token = GetToken(TokenType.DoubleColon);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Colon);
                    }

                    break;

                case TokenValues.QuestionMark:
                    token = GetToken(TokenType.QuestionMark);
                    break;

                case TokenValues.At:
                    token = GetToken(TokenType.At);
                    break;

                case TokenValues.Tilde:
                    token = GetToken(TokenType.Tilde);
                    break;

                case TokenValues.Semicolon:
                    token = GetToken(TokenType.Semicolon);
                    break;

                case TokenValues.LeftParenthesis:
                    token = GetToken(TokenType.LeftParenthesis);
                    break;

                case TokenValues.RightParenthesis:
                    token = GetToken(TokenType.RightParenthesis);
                    break;

                case TokenValues.LeftBracket:
                    token = GetToken(TokenType.LeftBracket);
                    break;

                case TokenValues.RightBracket:
                    token = GetToken(TokenType.RightBracket);
                    break;

                case TokenValues.Not:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.NotEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Not);
                    }

                    break;

                case TokenValues.EqualsAssign:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.EqualsEquals);
                        _reader.ConsumeChar();
                    }
                    else if (_reader.Next == '>')
                    {
                        token = GetToken(TokenType.LambdaArrow);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Equals);
                    }

                    break;

                case TokenValues.LeftBrace:
                    token = GetToken(TokenType.LeftBrace);
                    break;

                case TokenValues.RightBrace:
                    token = GetToken(TokenType.RightBrace);
                    break;

                case TokenValues.Comma:
                    token = GetToken(TokenType.Comma);
                    break;

                case TokenValues.Plus:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.PlusEquals);
                        _reader.ConsumeChar();
                    }
                    else if (_reader.Next == TokenValues.Plus)
                    {
                        token = GetToken(TokenType.PlusPlus);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Plus);
                    }

                    break;

                case TokenValues.Minus:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.MinusEquals);
                        _reader.ConsumeChar();
                    }
                    else if (_reader.Next == TokenValues.GreaterThan)
                    {
                        token = GetToken(TokenType.Arrow);
                        _reader.ConsumeChar();
                    }
                    else if (_reader.Next == TokenValues.Minus)
                    {
                        token = GetToken(TokenType.MinusMinus);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Minus);
                    }

                    break;

                case TokenValues.Slash:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.DivideEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Slash);
                    }

                    break;

                case TokenValues.Dot:
                    if (_reader.Next == TokenValues.Dot)
                    {
                        token = GetToken(TokenType.DoubleDot);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Dot);
                    }

                    break;

                case TokenValues.Asterisk:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.TimesEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Asterisk);
                    }

                    break;

                case TokenValues.Percent:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.ModuloEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Percent);
                    }

                    break;

                case '&':
                    if (_reader.Next == '&')
                    {
                        token = GetToken(TokenType.And);
                        _reader.ConsumeChar();
                    }

                    break;

                case '|':
                    if (_reader.Next == '|')
                    {
                        token = GetToken(TokenType.Or);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.Pipe);
                    }

                    break;

                case TokenValues.LessThan:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.LessThanEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.LessThan);
                    }

                    break;

                case TokenValues.GreaterThan:
                    if (_reader.Next == '=')
                    {
                        token = GetToken(TokenType.GreaterThanEquals);
                        _reader.ConsumeChar();
                    }
                    else
                    {
                        token = GetToken(TokenType.GreaterThan);
                    }

                    break;

                default:
                    if (IsIdentifierStart(_reader.Current))
                    {
                        var identifier = _commonLexer.GetIdentifier();
                        if (Keywords.TryGetValue(identifier, out var kw))
                        {
                            return(new Token(kw, _reader.Line, _reader.Col));
                        }
                        else
                        {
                            return(new Token(TokenType.Identifier, _reader.Line, _reader.Col, text: identifier));
                        }
                    }
                    else if (IsNumberStart(_reader.Current))
                    {
                        return(GetNumberToken());
                        // parse number
                    }

                    throw new LexerException($"Unexpected token {_reader.Current} ({(int) _reader.Current})");
                }

                _reader.ConsumeChar();
            }

            return(token);
        }
Exemplo n.º 2
0
        private Token NextToken()
        {
            var token = new Token();

            while (token.Type == TokenType.None)
            {
                switch (_reader.Current)
                {
                case ' ':
                    while (_reader.Current == ' ' && !_reader.IsEOF)
                    {
                        _reader.ConsumeChar();
                    }

                    continue;

                case '\t':
                    break;

                case '#':     // comments
                    if (_reader.Next == '+')
                    {
                        while (_reader.Current != '-' || _reader.Next != '#' && !_reader.IsEOF)
                        {
                            _reader.ConsumeChar();
                        }

                        _reader.ConsumeChar();     // consume last #
                    }
                    else
                    {
                        _reader.ConsumeLine();
                    }

                    break;

                case '\r':
                    break;

                case '\n':
                    if (_reader.IsEOF)
                    {
                        return(GetToken(TokenType.EOF));
                    }
                    break;

                case ';':
                    token = GetToken(TokenType.Semicolon);
                    break;

                case '\'':
                    return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('\'')));

                case '"':
                    return(new Token(TokenType.String, _reader.Line, _reader.Col, text: _commonLexer.GetString('"')));

                default:
                    if (IsIdentifierStart(_reader.Current))
                    {
                        var identifier = _commonLexer.GetIdentifier();
                        if (identifier == ImportKeyword)
                        {
                            return(new Token(TokenType.Import, _reader.Line, _reader.Col));
                        }
                        return(new Token(TokenType.Identifier, _reader.Line, _reader.Col, text: identifier));
                    }
                    _reader.ConsumeChar();
                    return(new Token());
                }

                _reader.ConsumeChar();
            }

            return(token);
        }