Esempio n. 1
0
        public void FillingBufferSeveralTimesTest()
        {
            const int BUFFER_SIZE = 6; // note that sample file contains 17 bytes

            lexer = new Lexer(tokenizer, BUFFER_SIZE);
            lexer.SetDataSource(fileStream);

            TokensIteratorTest();
        }
Esempio n. 2
0
        public static LRParser GetSLRParser(AugmentedGrammar grammar, Lexer lexer)
        {
            var parsingTableBuilder = new SLRParsingTableBuilder(lexer.TokensNumber);
            parsingTableBuilder.SetGrammar(grammar);
            parsingTableBuilder.ConstructParsingTable();
            var parser = new LRParser
                             {
                                 ParsingTable = parsingTableBuilder.GetTable(),
                                 Lexer = lexer,
                                 Grammar = grammar
                             };

            return parser;
        }
Esempio n. 3
0
        public void InitTests()
        {
            fileName = Path.GetTempFileName();
            File.WriteAllText(fileName, SOURCE);

            tokenizer = new RegExpTokenizer();
            tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction());
            Array.ForEach(lexemes, (s) => tokenizer.UseTerminal(RegExp.Literal(s)));
            tokenizer.BuildTransitions();

            lexer = new Lexer(tokenizer);
            fileStream = File.OpenRead(fileName);
            lexer.SetDataSource(fileStream);
        }
Esempio n. 4
0
        public void Test()
        {
            lexer = ScopusFacade.GetLexer();
            SetUpLexer(); // This should be called before BuildGrammar, since grammar use terminals defined here
            var motekGrammar = BuildGrammar();
            var parser = ScopusFacade.GetSLRParser(motekGrammar, lexer);

            var input = @"
            domain::other  = 7;
            variable = domain::other + 5;
            another = sin(variable)*other;
            ";
            var inputStream = new MemoryStream(encoding.GetBytes(input));
            lexer.SetDataSource(inputStream);
            parser.ParseInput();
        }
Esempio n. 5
0
        public void Test()
        {
            lexer = ScopusFacade.GetLexer();
            SetUpLexer(); // This should be called before BuildGrammar, since grammar use terminals defined here
            var mISAGrammar = BuildGrammar();
            var parser = ScopusFacade.GetSLRParser(mISAGrammar, lexer);

            // TODO: PARSE DIRECTIVE, I.E. ADD TOKENS FOR inclde, text ...
            var input = @"
            ! ldlohi16.s
            !
            ! This program tests the ldlo.16 and ldhi.16 opcode

            .include ""mISA_test_common_s.h""
              .text

            ! NO MACRO SUPPORT YET!!!
            !TEST_START test_ldhilo_16
            and.c.32 r63, r63, 0            !

            add.32 r56, r63, 0              ! test counter
            add.32 r57, r63, data_start     ! load init addr base

            case1:
            add.32/0 r56, r56, 1;           ! increment test counter

            add.32 r58, r63, 0x00002211     ! load LO expected result
            add.32 r59, r63, 0x00000000     ! load HI expected result
            add.32 r1, r63, 0xffffffff     ! clear target
            add.32 r11,  r63, 0xffffffff     ! clear target

            ldlo.16    r1, nat:[r57]        ! + <nat:> 8 aligned
            bubbles 2
            CMP_AND_FAIL_REG .32 r1, r58, fail
            .if CO20005 == 0
            ldhi.16    r11, nat:[r57+8]       ! + <nat:>
            .else
            ldhi.16    r11, nat:[r57+64]       ! + <nat:>
            .endif
            bubbles 2
            CMP_AND_FAIL_REG .32 r11, r59, fail
            or.64      r1, r1, r11
            !        cmit                           ! - nat: is not speculative

            case2:
            add.32/0 r56, r56, 1;           ! increment test counter
            sub.32/0 r57,  r57,  1;         ! decrement test address

            add.32 r58, r63, 0x00000008     ! load LO expected result
            add.32 r59, r63, 0x00001100     ! load HI expected result
            add.32 r2, r63, 0xffffffff     ! clear target
            add.32 r12,  r63, 0xffffffff     ! clear target

            ldlo.16    r2, nat:[r57]        ! + <nat:>   8 misaligned (1 less)
            bubbles 2
            CMP_AND_FAIL_REG .32 r2, r58, fail
            .if CO20005 == 0
            ldhi.16    r12, nat:[r57+8]       ! + <nat:>
            .else
            ldhi.16    r12, nat:[r57+64]       ! + <nat:>
            .endif
            bubbles 2
            CMP_AND_FAIL_REG .32 r12, r59, fail
            or.64      r2, r2, r12
            !        cmit                           ! - nat: is not speculative

            case3:
            add.32/0 r56, r56, 1;           ! increment test counter
            add.32/0 r57,  r57,  2;         ! increment test address

            add.32 r58, r63, 0x00003322     ! load LO expected result
            add.32 r59, r63, 0x00000000     ! load HI expected result
            add.32 r3, r63, 0xffffffff     ! clear target
            add.32 r13,  r63, 0xffffffff     ! clear target
            ";
            var inputStream = new MemoryStream(encoding.GetBytes(input));
            lexer.SetDataSource(inputStream);
            parser.ParseInput();
        }
Esempio n. 6
0
        public void ArithmeticStatementFullyAutonomousParseTest()
        {
            const string INPUT = @"2*(3+4)";

            string fileName = Path.GetTempFileName();
            File.WriteAllText(fileName, INPUT);
            RegExpTokenizer tokenizer = new RegExpTokenizer();
            tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction());
            ILexer lexer = new Lexer(tokenizer);
            Stream fileStream = File.OpenRead(fileName);
            lexer.SetDataSource(fileStream);

            var stack = new Stack<int>();
            var parser = new LRParser();

            var E = new NonTerminal("E");
            var T = new NonTerminal("T");
            var F = new NonTerminal("F");

            var id = tokenizer.UseTerminal(RegExp.AtLeastOneOf(RegExp.Choice(
                RegExp.Literal('0'), RegExp.Literal('1'), RegExp.Literal('2'),
                RegExp.Literal('3'), RegExp.Literal('4'), RegExp.Literal('5'),
                RegExp.Literal('6'), RegExp.Literal('7'), RegExp.Literal('8'),
                RegExp.Literal('9'))));
            var plus = tokenizer.UseTerminal(RegExp.Literal('+'));
            var mult = tokenizer.UseTerminal(RegExp.Literal('*'));
            var leftBrace = tokenizer.UseTerminal(RegExp.Literal('('));
            var rightBrace = tokenizer.UseTerminal(RegExp.Literal(')'));

            tokenizer.BuildTransitions();

            var grammar = new AugmentedGrammar()
                              {
                                  E --> E & plus & T ^ (v => stack.Push(stack.Pop() + stack.Pop())),
                                  E --> T,
                                  T --> T & mult & F ^ (v => stack.Push(stack.Pop() * stack.Pop())),
                                  T --> F,
                                  F --> leftBrace & E & rightBrace,
                                  F --> id ^ (v => stack.Push(v[id].AsInt()))
                              };

            Console.WriteLine("Grammar is being tested: \n{0}", grammar);
            Console.WriteLine("Input is being parsed: {0}\n", INPUT);
            Console.WriteLine("Parsing process:\n");
            parser.Grammar = grammar;
            parser.Lexer = lexer;

            var ptBuilder = new SLRParsingTableBuilder();
            ptBuilder.SetGrammar(grammar);
            ptBuilder.ConstructParsingTable();
            parser.ParsingTable = ptBuilder.GetTable();

            parser.InputAccepted += (sender, eventArgs) => Console.WriteLine("Accepted!");
            parser.ParseInput();

            Assert.That(stack.Pop(), Is.EqualTo(14));
            Assert.That(stack.Count, Is.EqualTo(0));

            fileStream.Close();
            File.Delete(fileName);
        }
Esempio n. 7
0
        public void ArithmeticStatementParseTest()
        {
            const string INPUT = @"2*(3+4)";//@"2*(3+4)$"
            ActionTableEntry[,] actionTable;
            int[,] gotoTable;
            string fileName;
            ILexer lexer;
            Stream fileStream;
            RegExpTokenizer tokenizer;

            fileName = Path.GetTempFileName();
            File.WriteAllText(fileName, INPUT);
            tokenizer = new RegExpTokenizer();
            tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction());
            lexer = new Lexer(tokenizer);
            fileStream = File.OpenRead(fileName);
            lexer.SetDataSource(fileStream);

            actionTable = new ActionTableEntry[12, 6];
            #region Populating Action Table

            actionTable[0, 1].Action = ParserAction.Shift;
            actionTable[0, 1].Destination = 5;
            actionTable[0, 4].Action = ParserAction.Shift;
            actionTable[0, 4].Destination = 4;
            actionTable[1, 0].Action = ParserAction.Accept; // <-- !!!!
            actionTable[1, 2].Action = ParserAction.Shift;
            actionTable[1, 2].Destination = 6;
            actionTable[2, 0].Action = ParserAction.Reduce;
            actionTable[2, 0].Destination = 2;
            actionTable[2, 2].Action = ParserAction.Reduce;
            actionTable[2, 2].Destination = 2;
            actionTable[2, 3].Action = ParserAction.Shift;
            actionTable[2, 3].Destination = 7;
            actionTable[2, 5].Action = ParserAction.Reduce;
            actionTable[2, 5].Destination = 2;
            actionTable[3, 0].Action = ParserAction.Reduce;
            actionTable[3, 0].Destination = 4;
            actionTable[3, 2].Action = ParserAction.Reduce;
            actionTable[3, 2].Destination = 4;
            actionTable[3, 3].Action = ParserAction.Reduce;
            actionTable[3, 3].Destination = 4;
            actionTable[3, 5].Action = ParserAction.Reduce;
            actionTable[3, 5].Destination = 4;
            actionTable[4, 1].Action = ParserAction.Shift;
            actionTable[4, 1].Destination = 5;
            actionTable[4, 4].Action = ParserAction.Shift;
            actionTable[4, 4].Destination = 4;
            actionTable[5, 0].Action = ParserAction.Reduce;
            actionTable[5, 0].Destination = 6;
            actionTable[5, 2].Action = ParserAction.Reduce;
            actionTable[5, 2].Destination = 6;
            actionTable[5, 3].Action = ParserAction.Reduce;
            actionTable[5, 3].Destination = 6;
            actionTable[5, 5].Action = ParserAction.Reduce;
            actionTable[5, 5].Destination = 6;
            actionTable[6, 1].Action = ParserAction.Shift;
            actionTable[6, 1].Destination = 5;
            actionTable[6, 4].Action = ParserAction.Shift;
            actionTable[6, 4].Destination = 4;
            actionTable[7, 1].Action = ParserAction.Shift;
            actionTable[7, 1].Destination = 5;
            actionTable[7, 4].Action = ParserAction.Shift;
            actionTable[7, 4].Destination = 4;
            actionTable[8, 2].Action = ParserAction.Shift;
            actionTable[8, 2].Destination = 6;
            actionTable[8, 5].Action = ParserAction.Shift;
            actionTable[8, 5].Destination = 11;
            actionTable[9, 0].Action = ParserAction.Reduce;
            actionTable[9, 0].Destination = 1;
            actionTable[9, 2].Action = ParserAction.Reduce;
            actionTable[9, 2].Destination = 1;
            actionTable[9, 3].Action = ParserAction.Shift;
            actionTable[9, 3].Destination = 7;
            actionTable[9, 5].Action = ParserAction.Reduce;
            actionTable[9, 5].Destination = 1;
            actionTable[10, 0].Action = ParserAction.Reduce;
            actionTable[10, 0].Destination = 3;
            actionTable[10, 2].Action = ParserAction.Reduce;
            actionTable[10, 2].Destination = 3;
            actionTable[10, 3].Action = ParserAction.Reduce;
            actionTable[10, 3].Destination = 3;
            actionTable[10, 5].Action = ParserAction.Reduce;
            actionTable[10, 5].Destination = 3;
            actionTable[11, 0].Action = ParserAction.Reduce;
            actionTable[11, 0].Destination = 5;
            actionTable[11, 2].Action = ParserAction.Reduce;
            actionTable[11, 2].Destination = 5;
            actionTable[11, 3].Action = ParserAction.Reduce;
            actionTable[11, 3].Destination = 5;
            actionTable[11, 5].Action = ParserAction.Reduce;
            actionTable[11, 5].Destination = 5;
            //actionTable[0, 0].Action = ParserAction.Shift;
            //actionTable[0, 0].Destination = 5;
            //actionTable[0, 3].Action = ParserAction.Shift;
            //actionTable[0, 3].Destination = 4;
            //actionTable[1, 1].Action = ParserAction.Shift;
            //actionTable[1, 1].Destination = 6;
            //actionTable[1, 5].Action = ParserAction.Accept; // <-- !!!!
            //actionTable[2, 1].Action = ParserAction.Reduce;
            //actionTable[2, 1].Destination = 2;
            //actionTable[2, 2].Action = ParserAction.Shift;
            //actionTable[2, 2].Destination = 7;
            //actionTable[2, 4].Action = ParserAction.Reduce;
            //actionTable[2, 4].Destination = 2;
            //actionTable[2, 5].Action = ParserAction.Reduce;
            //actionTable[2, 5].Destination = 2;
            //actionTable[3, 1].Action = ParserAction.Reduce;
            //actionTable[3, 1].Destination = 4;
            //actionTable[3, 2].Action = ParserAction.Reduce;
            //actionTable[3, 2].Destination = 4;
            //actionTable[3, 4].Action = ParserAction.Reduce;
            //actionTable[3, 4].Destination = 4;
            //actionTable[3, 5].Action = ParserAction.Reduce;
            //actionTable[3, 5].Destination = 4;
            //actionTable[4, 0].Action = ParserAction.Shift;
            //actionTable[4, 0].Destination = 5;
            //actionTable[4, 3].Action = ParserAction.Shift;
            //actionTable[4, 3].Destination = 4;
            //actionTable[5, 1].Action = ParserAction.Reduce;
            //actionTable[5, 1].Destination = 6;
            //actionTable[5, 2].Action = ParserAction.Reduce;
            //actionTable[5, 2].Destination = 6;
            //actionTable[5, 4].Action = ParserAction.Reduce;
            //actionTable[5, 4].Destination = 6;
            //actionTable[5, 5].Action = ParserAction.Reduce;
            //actionTable[5, 5].Destination = 6;
            //actionTable[6, 0].Action = ParserAction.Shift;
            //actionTable[6, 0].Destination = 5;
            //actionTable[6, 3].Action = ParserAction.Shift;
            //actionTable[6, 3].Destination = 4;
            //actionTable[7, 0].Action = ParserAction.Shift;
            //actionTable[7, 0].Destination = 5;
            //actionTable[7, 3].Action = ParserAction.Shift;
            //actionTable[7, 3].Destination = 4;
            //actionTable[8, 1].Action = ParserAction.Shift;
            //actionTable[8, 1].Destination = 6;
            //actionTable[8, 4].Action = ParserAction.Shift;
            //actionTable[8, 4].Destination = 11;
            //actionTable[9, 1].Action = ParserAction.Reduce;
            //actionTable[9, 1].Destination = 1;
            //actionTable[9, 2].Action = ParserAction.Shift;
            //actionTable[9, 2].Destination = 7;
            //actionTable[9, 4].Action = ParserAction.Reduce;
            //actionTable[9, 4].Destination = 1;
            //actionTable[9, 5].Action = ParserAction.Reduce;
            //actionTable[9, 5].Destination = 1;
            //actionTable[10, 1].Action = ParserAction.Reduce;
            //actionTable[10, 1].Destination = 3;
            //actionTable[10, 2].Action = ParserAction.Reduce;
            //actionTable[10, 2].Destination = 3;
            //actionTable[10, 4].Action = ParserAction.Reduce;
            //actionTable[10, 4].Destination = 3;
            //actionTable[10, 5].Action = ParserAction.Reduce;
            //actionTable[10, 5].Destination = 3;
            //actionTable[11, 1].Action = ParserAction.Reduce;
            //actionTable[11, 1].Destination = 5;
            //actionTable[11, 2].Action = ParserAction.Reduce;
            //actionTable[11, 2].Destination = 5;
            //actionTable[11, 4].Action = ParserAction.Reduce;
            //actionTable[11, 4].Destination = 5;
            //actionTable[11, 5].Action = ParserAction.Reduce;
            //actionTable[11, 5].Destination = 5;

            #endregion

            gotoTable = new int[12, 3];
            #region Populating Goto Table

            gotoTable[0, 0] = 1;
            gotoTable[0, 1] = 2;
            gotoTable[0, 2] = 3;
            gotoTable[4, 0] = 8;
            gotoTable[4, 1] = 2;
            gotoTable[4, 2] = 3;
            gotoTable[6, 1] = 9;
            gotoTable[6, 2] = 3;
            gotoTable[7, 2] = 10;

            #endregion

            var stack = new Stack<int>();
            var parser = new LRParser();

            var E = new NonTerminal("E");
            var T = new NonTerminal("T");
            var F = new NonTerminal("F");

            var id = tokenizer.UseTerminal(RegExp.AtLeastOneOf(RegExp.Choice(
                RegExp.Literal('0'), RegExp.Literal('1'), RegExp.Literal('2'),
                RegExp.Literal('3'), RegExp.Literal('4'), RegExp.Literal('5'),
                RegExp.Literal('6'), RegExp.Literal('7'), RegExp.Literal('8'),
                RegExp.Literal('9'))));
            var plus = tokenizer.UseTerminal(RegExp.Literal('+'));
            var mult = tokenizer.UseTerminal(RegExp.Literal('*'));
            var leftBrace = tokenizer.UseTerminal(RegExp.Literal('('));
            var rightBrace = tokenizer.UseTerminal(RegExp.Literal(')'));

            tokenizer.BuildTransitions();

            var grammar = new AugmentedGrammar()
                              {
                                  E --> E & plus & T ^ (v => stack.Push(stack.Pop() + stack.Pop())),
                                  E --> T,
                                  T --> T & mult & F ^ (v => stack.Push(stack.Pop() * stack.Pop())),
                                  T --> F,
                                  F --> leftBrace & E & rightBrace,
                                  F --> id ^ (v => stack.Push(v[id].AsInt()))
                              };

            Console.WriteLine("Grammar is being tested: \n{0}", grammar);
            Console.WriteLine("Input is being parsed: {0}\n", INPUT);
            Console.WriteLine("Parsing process:\n");
            parser.Grammar = grammar;
            parser.Lexer = lexer;
            parser.ParsingTable = new LRParsingTable(actionTable, gotoTable);
            parser.InputAccepted += (sender, eventArgs) => Console.WriteLine("Accepted!");
            parser.ParseInput();

            Assert.That(stack.Pop(), Is.EqualTo(14));
            Assert.That(stack.Count, Is.EqualTo(0));

            fileStream.Close();
            File.Delete(fileName);
        }