Ejemplo n.º 1
0
        public static TokenRecognizer ExactMatchRecognizer(TokenType tokenType, string target)
        {
            // example: keyword int
            //      i        n        t
            // (1) ---> (2) ---> (3) ---> ((4))
            // 3 transitions, 4 states

            var recognizer = new TokenRecognizer(tokenType, new State(1));

            for (int i = 1; i <= target.Length; i++)
            {
                var from = new State(i, StateType.Nonfinite);
                var to   = new State(i + 1, i == target.Length ? StateType.Finite : StateType.Nonfinite);

                var transition = new Transition(from, to, CharTests.Is(target[i - 1]));

                recognizer.AddTransition(transition);
            }

            return(recognizer);
        }
Ejemplo n.º 2
0
        static MyLanguage()
        {
            #region IdentifiersRecognizer [digit | letter| _]{1,}
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Finite);

                // set low priority to avoid conflicts with integer constants
                IdentifiersRecognizer = new TokenRecognizer(TokenType.Identifier, s1, -1);

                IdentifiersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsLetter.Or(CharTests.IsDigit).Or('_')));
                IdentifiersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsLetter.Or(CharTests.IsDigit).Or('_')));
            }
            #endregion

            #region DelimetersRecognizer [delim]{1,}
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Finite);

                DelimetersRecognizer = new TokenRecognizer(TokenType.Delimiter, s1);

                DelimetersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDelimeter));
                DelimetersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDelimeter));
            }
            #endregion

            #region StringsRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Finite); // accept unclosed literals
                State s3 = new State(3, StateType.Finite);

                StringsRecognizer = new TokenRecognizer(TokenType.StringConstant, s1);

                StringsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsStringMarker));
                StringsRecognizer.AddTransition(new Transition(s2, s2, CharTests.Not(CharTests.IsStringMarker.Or('\r'))));
                StringsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsStringMarker));
            }
            #endregion

            #region CharsRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Finite); // accept unclosed char literals
                State s3 = new State(3, StateType.Finite);

                CharsRecognizer = new TokenRecognizer(TokenType.CharConstant, s1);

                CharsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsCharMarker));
                CharsRecognizer.AddTransition(new Transition(s2, s2, CharTests.Not(CharTests.IsCharMarker.Or('\r'))));
                CharsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsCharMarker));
            }
            #endregion

            #region IntegersRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Finite);

                IntegersRecognizer = new TokenRecognizer(TokenType.IntegerConstant, s1);

                IntegersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDigit));
                IntegersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDigit));
            }
            #endregion

            #region FloatsRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Nonfinite);
                State s3 = new State(3, StateType.Nonfinite);
                State s4 = new State(4, StateType.Finite);

                FloatsRecognizer = new TokenRecognizer(TokenType.FloatConstant, s1);

                FloatsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDigit));
                FloatsRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDigit));
                FloatsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsFloatNumberDelimiter));
                FloatsRecognizer.AddTransition(new Transition(s3, s4, CharTests.IsDigit));
                FloatsRecognizer.AddTransition(new Transition(s4, s4, CharTests.IsDigit));
            }
            #endregion

            #region CommentsRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Nonfinite);
                State s3 = new State(3, StateType.Finite);

                CommentsRecognizer = new TokenRecognizer(TokenType.Commentary, s1);

                CommentsRecognizer.AddTransition(new Transition(s1, s2, CharTests.Is('/')));
                CommentsRecognizer.AddTransition(new Transition(s2, s3, CharTests.Is('/')));
                CommentsRecognizer.AddTransition(new Transition(s3, s3, CharTests.Is('\n').Or('\r').Inverse()));
            }
            #endregion

            #region MultiLineCommentsRecognizer
            {
                State s1 = new State(1, StateType.Nonfinite);
                State s2 = new State(2, StateType.Nonfinite);
                State s3 = new State(3, StateType.Nonfinite);
                State s4 = new State(4, StateType.Nonfinite);
                State s5 = new State(5, StateType.Finite);

                MultiLineCommentsRecognizer = new TokenRecognizer(TokenType.Commentary, s1);

                MultiLineCommentsRecognizer.AddTransition(new Transition(s1, s2, CharTests.Is('/')));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s2, s3, CharTests.Is('*')));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s3, CharTests.Is('*').Inverse()));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s4, CharTests.Is('*')));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s3, CharTests.Is('/').Or('*').Inverse()));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s4, CharTests.Is('*')));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s5, CharTests.Is('/')));

                // unclosed multiline comment
                MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s5, CharTests.IsEndOfText));
                MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s5, CharTests.IsEndOfText));
            }
            #endregion

            #region OperatorRecognizers
            OperatorRecognizers = TokenRecognizer.GenerateExactMatchRecognizers(TokenType.Operator, new[]
            {
                "+",
                "-",
                "*",
                "/",
                "=",
                "==",
                "!=",
                ">",
                "<",
                "<=",
                ">=",
                "(",
                ")",
                "{",
                "}",
                ";",
                ","
            });
            #endregion

            #region Keywords
            Keywords = new List <string>(new[]
            {
                "int",
                "float",
                "void",
                "if",
                "else",
                "for"
            });
            #endregion

            #region Functions
            Functions = new List <string>(new[]
            {
                "sin",
                "cos",
                "tan",
                "exp",
                "read",
                "write",
                "ln",
                "abs"
            });
            #endregion
        }