示例#1
0
        protected override bool LexNext(State hiddenState)
        {
            CppLexerState state            = (CppLexerState)hiddenState;
            bool          allowWhitespaces = !state.IsInsidePreprocessor;

            if (allowWhitespaces)
            {
                Buffer.SkipAllWhitespaces();
            }
            if (Buffer.IsEOF)
            {
                return(false);
            }
            int  line   = Buffer.TextPosition.Line;
            char first  = Buffer.Peek();
            char second = Buffer.Peek(1);
            char third  = Buffer.Peek(2);

            Buffer.StartLexeme();
            LexResult lexRes = new LexResult(CppTokenKind.Unknown, true);

            switch (first)
            {
            case '&':
            {
                if (second == '&')
                {
                    lexRes.Kind = CppTokenKind.LogicalAndOp;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.AndAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.AndOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '|':
            {
                if (second == '|')
                {
                    lexRes.Kind = CppTokenKind.LogicalOrOp;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.OrAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.OrOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '=':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.LogicalEqualsOp;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.EqOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '!':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.LogicalNotEqualsOp;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.ExclationMark;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '<':
            {
                if (second == '<')
                {
                    if (third == '=')
                    {
                        lexRes.Kind = CppTokenKind.LeftShiftAssign;
                        Buffer.AdvanceColumns(3);
                    }
                    else
                    {
                        lexRes.Kind = CppTokenKind.LeftShiftOp;
                        Buffer.AdvanceColumns(2);
                    }
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.LessOrEqualOp;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.LessThanOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '>':
            {
                if (second == '>')
                {
                    if (third == '=')
                    {
                        lexRes.Kind = CppTokenKind.RightShiftAssign;
                        Buffer.AdvanceColumns(3);
                    }
                    else
                    {
                        lexRes.Kind = CppTokenKind.RightShiftOp;
                        Buffer.AdvanceColumns(2);
                    }
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.GreaterOrEqualOp;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.GreaterThanOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '+':
            {
                if (second == '+')
                {
                    lexRes.Kind = CppTokenKind.IncOp;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.AddAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.AddOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '-':
            {
                if (second == '-')
                {
                    lexRes.Kind = CppTokenKind.DecOp;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.SubAssign;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '>')
                {
                    lexRes.Kind = CppTokenKind.PtrOp;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.SubOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '/':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.DivAssign;
                    Buffer.AdvanceColumns(2);
                }
                else if (second == '/')
                {
                    lexRes = LexSingleLineComment(Buffer, true);
                    if (!lexRes.IsComplete)
                    {
                        AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '\n' or '\r' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
                    }
                }
                else if (second == '*')
                {
                    lexRes = LexMultiLineComment(Buffer, true);
                    if (!lexRes.IsComplete)
                    {
                        AddError(Buffer.LexemeStart, $"Unterminated single-line comment, expect '*/' but found '{Buffer.Peek()}'", lexRes.Kind.ToString());
                    }
                }
                else
                {
                    Buffer.AdvanceColumn();
                    lexRes.Kind = CppTokenKind.DivOp;
                }
            }
            break;

            case '*':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.MulAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.MulOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '%':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.ModAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.ModOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '.':
            {
                if (second == '.' && third == '.')
                {
                    lexRes.Kind = CppTokenKind.Ellipsis;
                    Buffer.AdvanceColumns(3);
                }
                else if (SyntaxUtils.IsNumeric(second))
                {
                    lexRes = LexNumber();
                }
                else
                {
                    lexRes.Kind = CppTokenKind.Dot;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '^':
            {
                if (second == '=')
                {
                    lexRes.Kind = CppTokenKind.XorAssign;
                    Buffer.AdvanceColumns(2);
                }
                else
                {
                    lexRes.Kind = CppTokenKind.XorOp;
                    Buffer.AdvanceColumn();
                }
            }
            break;

            case '#':
                return(LexPreprocessor(state));

            case '"':
                lexRes = LexString("string");
                break;

            case '\'':
                lexRes = LexString("char");
                break;

            case '~':
                lexRes.Kind = CppTokenKind.Tilde;
                Buffer.AdvanceColumn();
                break;

            case '\\':
                lexRes.Kind = CppTokenKind.Backslash;
                Buffer.AdvanceColumn();
                break;

            case ',':
                lexRes.Kind = CppTokenKind.Comma;
                Buffer.AdvanceColumn();
                break;

            case ';':
                lexRes.Kind = CppTokenKind.Semicolon;
                Buffer.AdvanceColumn();
                break;

            case ':':
                lexRes.Kind = CppTokenKind.Colon;
                Buffer.AdvanceColumn();
                break;

            case '?':
                lexRes.Kind = CppTokenKind.QuestionMark;
                Buffer.AdvanceColumn();
                break;

            case '{':
                lexRes.Kind = CppTokenKind.LeftBrace;
                Buffer.AdvanceColumn();
                break;

            case '}':
                lexRes.Kind = CppTokenKind.RightBrace;
                Buffer.AdvanceColumn();
                break;

            case '[':
                lexRes.Kind = CppTokenKind.LeftBracket;
                Buffer.AdvanceColumn();
                break;

            case ']':
                lexRes.Kind = CppTokenKind.RightBracket;
                Buffer.AdvanceColumn();
                break;

            case '(':
                lexRes.Kind = CppTokenKind.LeftParen;
                Buffer.AdvanceColumn();
                break;

            case ')':
                lexRes.Kind = CppTokenKind.RightParen;
                Buffer.AdvanceColumn();
                break;

            default:
            {
                if (SyntaxUtils.IsLineBreak(first) && allowWhitespaces)
                {
                    lexRes.Kind = CppTokenKind.EndOfLine;
                    int nb = SyntaxUtils.GetLineBreakChars(first, second);
                    Buffer.AdvanceLine(nb);
                }
                else if (first == '\t' && allowWhitespaces)
                {
                    lexRes.Kind = CppTokenKind.Spacings;
                    while (!Buffer.IsEOF)
                    {
                        if (Buffer.Peek() != '\t')
                        {
                            break;
                        }
                        Buffer.AdvanceTab();
                    }
                }
                else if (SyntaxUtils.IsSpacing(first) && allowWhitespaces)
                {
                    lexRes.Kind = CppTokenKind.Spacings;
                    Buffer.AdvanceColumnsWhile(SyntaxUtils.IsSpacing);
                }
                else if (SyntaxUtils.IsIdentStart(first))
                {
                    Debug.Assert(!state.IsInsidePreprocessor);
                    lexRes = LexIdent(false);
                }
                else if (SyntaxUtils.IsNumeric(first))
                {
                    lexRes = LexNumber();
                }
                else
                {
                    AddError(Buffer.TextPosition, $"Unexpected character '{first}'", "Character");
                    return(false);
                }
            }
            break;
            }
            return(PushToken(CppTokenPool.Make(lexRes.Kind, Buffer.LexemeRange, lexRes.IsComplete)));
        }
示例#2
0
        private bool LexPreprocessor(CppLexerState state)
        {
            Debug.Assert(Buffer.Peek() == '#');

            state.StartPreprocessor();

            // Preprocessor start
            Buffer.StartLexeme();
            Buffer.AdvanceColumn();
            PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorStart, Buffer.LexemeRange, true));

            do
            {
                Buffer.SkipSpacings(TextStream.SkipType.All);
                Buffer.StartLexeme();
                char first  = Buffer.Peek();
                char second = Buffer.Peek(1);
                char third  = Buffer.Peek(2);
                if (first == '\\')
                {
                    if (SyntaxUtils.IsLineBreak(second))
                    {
                        Buffer.AdvanceColumn();
                        int lb = SyntaxUtils.GetLineBreakChars(second, third);
                        Buffer.AdvanceLine(lb);
                        continue;
                    }
                    else
                    {
                        AddError(Buffer.TextPosition, $"Unterminated preprocessor next-line, expect linebreak after '\' but got '{second}'", "Preprocessor");
                        return(false);
                    }
                }
                else if (first == '#')
                {
                    Buffer.AdvanceColumn();
                    PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorOperator, Buffer.LexemeRange, true));
                }
                else if (SyntaxUtils.IsLineBreak(first))
                {
                    int lb = SyntaxUtils.GetLineBreakChars(first, second);
                    Buffer.AdvanceLine(lb);
                    PushToken(CppTokenPool.Make(CppTokenKind.EndOfLine, Buffer.LexemeRange, true));
                    break;
                }
                else if (SyntaxUtils.IsIdentStart(first))
                {
                    LexResult identResult = LexIdent(true);
                    CppToken  identToken  = CppTokenPool.Make(identResult.Kind, Buffer.LexemeRange, identResult.IsComplete);
                    PushToken(identToken);
                    Buffer.SkipSpacings(TextStream.SkipType.All);
                    Buffer.StartLexeme();
                    if (identToken.Kind == CppTokenKind.PreprocessorKeyword)
                    {
                        switch (identToken.Value)
                        {
                        case "define":
                        {
                            if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
                            {
                                AddError(Buffer.TextPosition, $"Expect identifier for define, but got '{Buffer.Peek()}'", "Preprocessor");
                                return(false);
                            }
                            LexResult defineValueResult = LexIdent(false);
                            CppToken  defineValueToken  = CppTokenPool.Make(CppTokenKind.PreprocessorDefineSource, Buffer.LexemeRange, defineValueResult.IsComplete);
                            PushToken(defineValueToken);
                        }
                        break;

                        case "defined":
                        {
                            if (Buffer.Peek() == '(')
                            {
                                Buffer.AdvanceColumn();
                                if (!SyntaxUtils.IsIdentStart(Buffer.Peek()))
                                {
                                    AddError(Buffer.TextPosition, $"Expect identifier for defined, but got '{Buffer.Peek()}'", "Preprocessor");
                                    return(false);
                                }
                                LexResult definedValueResult = LexIdent(false);
                                CppToken  definedValueToken  = CppTokenPool.Make(CppTokenKind.PreprocessorDefineTarget, Buffer.LexemeRange, definedValueResult.IsComplete);
                                PushToken(definedValueToken);
                                Buffer.SkipSpacings(TextStream.SkipType.All);
                                if (Buffer.Peek() != ')')
                                {
                                    AddError(Buffer.TextPosition, $"Unterminated defined token, expect ')' but got '{Buffer.Peek()}'", "Preprocessor");
                                    return(false);
                                }
                            }
                        }
                        break;

                        case "include":
                        {
                            char n = Buffer.Peek();
                            if (n == '<' || n == '"')
                            {
                                bool isComplete = false;
                                Buffer.AdvanceColumn();
                                char quote = (n == '<') ? '>' : n;
                                while (!Buffer.IsEOF)
                                {
                                    if (Buffer.Peek() == quote)
                                    {
                                        isComplete = true;
                                        Buffer.AdvanceColumn();
                                        break;
                                    }
                                    Buffer.AdvanceColumn();
                                }
                                CppToken includeToken = CppTokenPool.Make(CppTokenKind.PreprocessorInclude, Buffer.LexemeRange, isComplete);
                                PushToken(includeToken);
                            }
                            else
                            {
                                return(false);
                            }
                        }
                        break;

                        case "pragma":
                        {
                            // @NOTE(final): Just skip until end-of-line
                            while (!Buffer.IsEOF)
                            {
                                char c = Buffer.Peek();
                                if (SyntaxUtils.IsLineBreak(c))
                                {
                                    break;
                                }
                                if (c == '\t')
                                {
                                    Buffer.AdvanceTab();
                                }
                                else
                                {
                                    Buffer.AdvanceColumn();
                                }
                            }
                        }
                        break;
                        }
                    }
                }
                else
                {
                    if (!LexNext(state))
                    {
                        break;
                    }
                }
            } while (!Buffer.IsEOF);

            state.EndPreprocessor();

            PushToken(CppTokenPool.Make(CppTokenKind.PreprocessorEnd, new TextRange(Buffer.TextPosition, 0), true));

            return(true);
        }