public void FillingBufferSeveralTimesTest() { const int BUFFER_SIZE = 6; // note that sample file contains 17 bytes lexer = new Lexer(tokenizer, BUFFER_SIZE); lexer.SetDataSource(fileStream); TokensIteratorTest(); }
public static LRParser GetSLRParser(AugmentedGrammar grammar, Lexer lexer) { var parsingTableBuilder = new SLRParsingTableBuilder(lexer.TokensNumber); parsingTableBuilder.SetGrammar(grammar); parsingTableBuilder.ConstructParsingTable(); var parser = new LRParser { ParsingTable = parsingTableBuilder.GetTable(), Lexer = lexer, Grammar = grammar }; return parser; }
public void InitTests() { fileName = Path.GetTempFileName(); File.WriteAllText(fileName, SOURCE); tokenizer = new RegExpTokenizer(); tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction()); Array.ForEach(lexemes, (s) => tokenizer.UseTerminal(RegExp.Literal(s))); tokenizer.BuildTransitions(); lexer = new Lexer(tokenizer); fileStream = File.OpenRead(fileName); lexer.SetDataSource(fileStream); }
public void Test() { lexer = ScopusFacade.GetLexer(); SetUpLexer(); // This should be called before BuildGrammar, since grammar use terminals defined here var motekGrammar = BuildGrammar(); var parser = ScopusFacade.GetSLRParser(motekGrammar, lexer); var input = @" domain::other = 7; variable = domain::other + 5; another = sin(variable)*other; "; var inputStream = new MemoryStream(encoding.GetBytes(input)); lexer.SetDataSource(inputStream); parser.ParseInput(); }
public void Test() { lexer = ScopusFacade.GetLexer(); SetUpLexer(); // This should be called before BuildGrammar, since grammar use terminals defined here var mISAGrammar = BuildGrammar(); var parser = ScopusFacade.GetSLRParser(mISAGrammar, lexer); // TODO: PARSE DIRECTIVE, I.E. ADD TOKENS FOR inclde, text ... var input = @" ! ldlohi16.s ! ! This program tests the ldlo.16 and ldhi.16 opcode .include ""mISA_test_common_s.h"" .text ! NO MACRO SUPPORT YET!!! !TEST_START test_ldhilo_16 and.c.32 r63, r63, 0 ! add.32 r56, r63, 0 ! test counter add.32 r57, r63, data_start ! load init addr base case1: add.32/0 r56, r56, 1; ! increment test counter add.32 r58, r63, 0x00002211 ! load LO expected result add.32 r59, r63, 0x00000000 ! load HI expected result add.32 r1, r63, 0xffffffff ! clear target add.32 r11, r63, 0xffffffff ! clear target ldlo.16 r1, nat:[r57] ! + <nat:> 8 aligned bubbles 2 CMP_AND_FAIL_REG .32 r1, r58, fail .if CO20005 == 0 ldhi.16 r11, nat:[r57+8] ! + <nat:> .else ldhi.16 r11, nat:[r57+64] ! + <nat:> .endif bubbles 2 CMP_AND_FAIL_REG .32 r11, r59, fail or.64 r1, r1, r11 ! cmit ! - nat: is not speculative case2: add.32/0 r56, r56, 1; ! increment test counter sub.32/0 r57, r57, 1; ! decrement test address add.32 r58, r63, 0x00000008 ! load LO expected result add.32 r59, r63, 0x00001100 ! load HI expected result add.32 r2, r63, 0xffffffff ! clear target add.32 r12, r63, 0xffffffff ! clear target ldlo.16 r2, nat:[r57] ! + <nat:> 8 misaligned (1 less) bubbles 2 CMP_AND_FAIL_REG .32 r2, r58, fail .if CO20005 == 0 ldhi.16 r12, nat:[r57+8] ! + <nat:> .else ldhi.16 r12, nat:[r57+64] ! + <nat:> .endif bubbles 2 CMP_AND_FAIL_REG .32 r12, r59, fail or.64 r2, r2, r12 ! cmit ! - nat: is not speculative case3: add.32/0 r56, r56, 1; ! increment test counter add.32/0 r57, r57, 2; ! increment test address add.32 r58, r63, 0x00003322 ! load LO expected result add.32 r59, r63, 0x00000000 ! load HI expected result add.32 r3, r63, 0xffffffff ! clear target add.32 r13, r63, 0xffffffff ! clear target "; var inputStream = new MemoryStream(encoding.GetBytes(input)); lexer.SetDataSource(inputStream); parser.ParseInput(); }
public void ArithmeticStatementFullyAutonomousParseTest() { const string INPUT = @"2*(3+4)"; string fileName = Path.GetTempFileName(); File.WriteAllText(fileName, INPUT); RegExpTokenizer tokenizer = new RegExpTokenizer(); tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction()); ILexer lexer = new Lexer(tokenizer); Stream fileStream = File.OpenRead(fileName); lexer.SetDataSource(fileStream); var stack = new Stack<int>(); var parser = new LRParser(); var E = new NonTerminal("E"); var T = new NonTerminal("T"); var F = new NonTerminal("F"); var id = tokenizer.UseTerminal(RegExp.AtLeastOneOf(RegExp.Choice( RegExp.Literal('0'), RegExp.Literal('1'), RegExp.Literal('2'), RegExp.Literal('3'), RegExp.Literal('4'), RegExp.Literal('5'), RegExp.Literal('6'), RegExp.Literal('7'), RegExp.Literal('8'), RegExp.Literal('9')))); var plus = tokenizer.UseTerminal(RegExp.Literal('+')); var mult = tokenizer.UseTerminal(RegExp.Literal('*')); var leftBrace = tokenizer.UseTerminal(RegExp.Literal('(')); var rightBrace = tokenizer.UseTerminal(RegExp.Literal(')')); tokenizer.BuildTransitions(); var grammar = new AugmentedGrammar() { E --> E & plus & T ^ (v => stack.Push(stack.Pop() + stack.Pop())), E --> T, T --> T & mult & F ^ (v => stack.Push(stack.Pop() * stack.Pop())), T --> F, F --> leftBrace & E & rightBrace, F --> id ^ (v => stack.Push(v[id].AsInt())) }; Console.WriteLine("Grammar is being tested: \n{0}", grammar); Console.WriteLine("Input is being parsed: {0}\n", INPUT); Console.WriteLine("Parsing process:\n"); parser.Grammar = grammar; parser.Lexer = lexer; var ptBuilder = new SLRParsingTableBuilder(); ptBuilder.SetGrammar(grammar); ptBuilder.ConstructParsingTable(); parser.ParsingTable = ptBuilder.GetTable(); parser.InputAccepted += (sender, eventArgs) => Console.WriteLine("Accepted!"); parser.ParseInput(); Assert.That(stack.Pop(), Is.EqualTo(14)); Assert.That(stack.Count, Is.EqualTo(0)); fileStream.Close(); File.Delete(fileName); }
public void ArithmeticStatementParseTest() { const string INPUT = @"2*(3+4)";//@"2*(3+4)$" ActionTableEntry[,] actionTable; int[,] gotoTable; string fileName; ILexer lexer; Stream fileStream; RegExpTokenizer tokenizer; fileName = Path.GetTempFileName(); File.WriteAllText(fileName, INPUT); tokenizer = new RegExpTokenizer(); tokenizer.SetTransitionFunction(new TableDrivenTransitionFunction()); lexer = new Lexer(tokenizer); fileStream = File.OpenRead(fileName); lexer.SetDataSource(fileStream); actionTable = new ActionTableEntry[12, 6]; #region Populating Action Table actionTable[0, 1].Action = ParserAction.Shift; actionTable[0, 1].Destination = 5; actionTable[0, 4].Action = ParserAction.Shift; actionTable[0, 4].Destination = 4; actionTable[1, 0].Action = ParserAction.Accept; // <-- !!!! actionTable[1, 2].Action = ParserAction.Shift; actionTable[1, 2].Destination = 6; actionTable[2, 0].Action = ParserAction.Reduce; actionTable[2, 0].Destination = 2; actionTable[2, 2].Action = ParserAction.Reduce; actionTable[2, 2].Destination = 2; actionTable[2, 3].Action = ParserAction.Shift; actionTable[2, 3].Destination = 7; actionTable[2, 5].Action = ParserAction.Reduce; actionTable[2, 5].Destination = 2; actionTable[3, 0].Action = ParserAction.Reduce; actionTable[3, 0].Destination = 4; actionTable[3, 2].Action = ParserAction.Reduce; actionTable[3, 2].Destination = 4; actionTable[3, 3].Action = ParserAction.Reduce; actionTable[3, 3].Destination = 4; actionTable[3, 5].Action = ParserAction.Reduce; actionTable[3, 5].Destination = 4; actionTable[4, 1].Action = ParserAction.Shift; actionTable[4, 1].Destination = 5; actionTable[4, 4].Action = ParserAction.Shift; actionTable[4, 4].Destination = 4; actionTable[5, 0].Action = ParserAction.Reduce; actionTable[5, 0].Destination = 6; actionTable[5, 2].Action = ParserAction.Reduce; actionTable[5, 2].Destination = 6; actionTable[5, 3].Action = ParserAction.Reduce; actionTable[5, 3].Destination = 6; actionTable[5, 5].Action = ParserAction.Reduce; actionTable[5, 5].Destination = 6; actionTable[6, 1].Action = ParserAction.Shift; actionTable[6, 1].Destination = 5; actionTable[6, 4].Action = ParserAction.Shift; actionTable[6, 4].Destination = 4; actionTable[7, 1].Action = ParserAction.Shift; actionTable[7, 1].Destination = 5; actionTable[7, 4].Action = ParserAction.Shift; actionTable[7, 4].Destination = 4; actionTable[8, 2].Action = ParserAction.Shift; actionTable[8, 2].Destination = 6; actionTable[8, 5].Action = ParserAction.Shift; actionTable[8, 5].Destination = 11; actionTable[9, 0].Action = ParserAction.Reduce; actionTable[9, 0].Destination = 1; actionTable[9, 2].Action = ParserAction.Reduce; actionTable[9, 2].Destination = 1; actionTable[9, 3].Action = ParserAction.Shift; actionTable[9, 3].Destination = 7; actionTable[9, 5].Action = ParserAction.Reduce; actionTable[9, 5].Destination = 1; actionTable[10, 0].Action = ParserAction.Reduce; actionTable[10, 0].Destination = 3; actionTable[10, 2].Action = ParserAction.Reduce; actionTable[10, 2].Destination = 3; actionTable[10, 3].Action = ParserAction.Reduce; actionTable[10, 3].Destination = 3; actionTable[10, 5].Action = ParserAction.Reduce; actionTable[10, 5].Destination = 3; actionTable[11, 0].Action = ParserAction.Reduce; actionTable[11, 0].Destination = 5; actionTable[11, 2].Action = ParserAction.Reduce; actionTable[11, 2].Destination = 5; actionTable[11, 3].Action = ParserAction.Reduce; actionTable[11, 3].Destination = 5; actionTable[11, 5].Action = ParserAction.Reduce; actionTable[11, 5].Destination = 5; //actionTable[0, 0].Action = ParserAction.Shift; //actionTable[0, 0].Destination = 5; //actionTable[0, 3].Action = ParserAction.Shift; //actionTable[0, 3].Destination = 4; //actionTable[1, 1].Action = ParserAction.Shift; //actionTable[1, 1].Destination = 6; //actionTable[1, 5].Action = ParserAction.Accept; // <-- !!!! //actionTable[2, 1].Action = ParserAction.Reduce; //actionTable[2, 1].Destination = 2; //actionTable[2, 2].Action = ParserAction.Shift; //actionTable[2, 2].Destination = 7; //actionTable[2, 4].Action = ParserAction.Reduce; //actionTable[2, 4].Destination = 2; //actionTable[2, 5].Action = ParserAction.Reduce; //actionTable[2, 5].Destination = 2; //actionTable[3, 1].Action = ParserAction.Reduce; //actionTable[3, 1].Destination = 4; //actionTable[3, 2].Action = ParserAction.Reduce; //actionTable[3, 2].Destination = 4; //actionTable[3, 4].Action = ParserAction.Reduce; //actionTable[3, 4].Destination = 4; //actionTable[3, 5].Action = ParserAction.Reduce; //actionTable[3, 5].Destination = 4; //actionTable[4, 0].Action = ParserAction.Shift; //actionTable[4, 0].Destination = 5; //actionTable[4, 3].Action = ParserAction.Shift; //actionTable[4, 3].Destination = 4; //actionTable[5, 1].Action = ParserAction.Reduce; //actionTable[5, 1].Destination = 6; //actionTable[5, 2].Action = ParserAction.Reduce; //actionTable[5, 2].Destination = 6; //actionTable[5, 4].Action = ParserAction.Reduce; //actionTable[5, 4].Destination = 6; //actionTable[5, 5].Action = ParserAction.Reduce; //actionTable[5, 5].Destination = 6; //actionTable[6, 0].Action = ParserAction.Shift; //actionTable[6, 0].Destination = 5; //actionTable[6, 3].Action = ParserAction.Shift; //actionTable[6, 3].Destination = 4; //actionTable[7, 0].Action = ParserAction.Shift; //actionTable[7, 0].Destination = 5; //actionTable[7, 3].Action = ParserAction.Shift; //actionTable[7, 3].Destination = 4; //actionTable[8, 1].Action = ParserAction.Shift; //actionTable[8, 1].Destination = 6; //actionTable[8, 4].Action = ParserAction.Shift; //actionTable[8, 4].Destination = 11; //actionTable[9, 1].Action = ParserAction.Reduce; //actionTable[9, 1].Destination = 1; //actionTable[9, 2].Action = ParserAction.Shift; //actionTable[9, 2].Destination = 7; //actionTable[9, 4].Action = ParserAction.Reduce; //actionTable[9, 4].Destination = 1; //actionTable[9, 5].Action = ParserAction.Reduce; //actionTable[9, 5].Destination = 1; //actionTable[10, 1].Action = ParserAction.Reduce; //actionTable[10, 1].Destination = 3; //actionTable[10, 2].Action = ParserAction.Reduce; //actionTable[10, 2].Destination = 3; //actionTable[10, 4].Action = ParserAction.Reduce; //actionTable[10, 4].Destination = 3; //actionTable[10, 5].Action = ParserAction.Reduce; //actionTable[10, 5].Destination = 3; //actionTable[11, 1].Action = ParserAction.Reduce; //actionTable[11, 1].Destination = 5; //actionTable[11, 2].Action = ParserAction.Reduce; //actionTable[11, 2].Destination = 5; //actionTable[11, 4].Action = ParserAction.Reduce; //actionTable[11, 4].Destination = 5; //actionTable[11, 5].Action = ParserAction.Reduce; //actionTable[11, 5].Destination = 5; #endregion gotoTable = new int[12, 3]; #region Populating Goto Table gotoTable[0, 0] = 1; gotoTable[0, 1] = 2; gotoTable[0, 2] = 3; gotoTable[4, 0] = 8; gotoTable[4, 1] = 2; gotoTable[4, 2] = 3; gotoTable[6, 1] = 9; gotoTable[6, 2] = 3; gotoTable[7, 2] = 10; #endregion var stack = new Stack<int>(); var parser = new LRParser(); var E = new NonTerminal("E"); var T = new NonTerminal("T"); var F = new NonTerminal("F"); var id = tokenizer.UseTerminal(RegExp.AtLeastOneOf(RegExp.Choice( RegExp.Literal('0'), RegExp.Literal('1'), RegExp.Literal('2'), RegExp.Literal('3'), RegExp.Literal('4'), RegExp.Literal('5'), RegExp.Literal('6'), RegExp.Literal('7'), RegExp.Literal('8'), RegExp.Literal('9')))); var plus = tokenizer.UseTerminal(RegExp.Literal('+')); var mult = tokenizer.UseTerminal(RegExp.Literal('*')); var leftBrace = tokenizer.UseTerminal(RegExp.Literal('(')); var rightBrace = tokenizer.UseTerminal(RegExp.Literal(')')); tokenizer.BuildTransitions(); var grammar = new AugmentedGrammar() { E --> E & plus & T ^ (v => stack.Push(stack.Pop() + stack.Pop())), E --> T, T --> T & mult & F ^ (v => stack.Push(stack.Pop() * stack.Pop())), T --> F, F --> leftBrace & E & rightBrace, F --> id ^ (v => stack.Push(v[id].AsInt())) }; Console.WriteLine("Grammar is being tested: \n{0}", grammar); Console.WriteLine("Input is being parsed: {0}\n", INPUT); Console.WriteLine("Parsing process:\n"); parser.Grammar = grammar; parser.Lexer = lexer; parser.ParsingTable = new LRParsingTable(actionTable, gotoTable); parser.InputAccepted += (sender, eventArgs) => Console.WriteLine("Accepted!"); parser.ParseInput(); Assert.That(stack.Pop(), Is.EqualTo(14)); Assert.That(stack.Count, Is.EqualTo(0)); fileStream.Close(); File.Delete(fileName); }