public void TestReplaceRangeThenInsertAtRightEdge() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "x"); tokens.InsertBefore(4, "y"); // no effect; within range of a replace Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestMismatchedSetError() { Grammar pg = new Grammar( "parser grammar p;\n" + "prog : WHILE ID LCURLY (assign)* RCURLY;\n" + "assign : ID ASSIGN expr SEMI ;\n" + "expr : INT | FLOAT | ID ;\n"); Grammar g = new Grammar(); g.ImportTokenVocabulary(pg); g.FileName = "<string>"; g.SetGrammarContent( "lexer grammar t;\n" + "WHILE : 'while';\n" + "LCURLY : '{';\n" + "RCURLY : '}';\n" + "ASSIGN : '=';\n" + "SEMI : ';';\n" + "ID : ('a'..'z')+ ;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ ;\n"); ICharStream input = new ANTLRStringStream("while x { i=; y=3.42; z=y; }"); Interpreter lexEngine = new Interpreter(g, input); FilteringTokenStream tokens = new FilteringTokenStream(lexEngine); tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99); //System.out.println("tokens="+tokens.toString()); Interpreter parseEngine = new Interpreter(pg, tokens); ParseTree t = parseEngine.Parse("prog"); string result = t.ToStringTree(); string expecting = "(<grammar p> (prog while x { (assign i = (expr MismatchedSetException(10!={5,6,7})))))"; Assert.AreEqual(expecting, result); }
public void TestOverlappingReplace2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 3, "bar"); tokens.Replace(1, 2, "foo"); // cannot split earlier replace Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestReplaceThenReplaceLowerIndexedSuperset() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "xyz"); tokens.Replace(1, 3, "foo"); // overlap, error Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@3,3:3='c',<6>,1:3]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestMultAltLoop() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : ('0'..'9'|'a'|'b')+ ;\n"); int Atype = g.GetTokenType("A"); Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); IToken result = engine.Scan("A"); engine = new Interpreter(g, new ANTLRStringStream("a")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("1234")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("aaa")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("aaaa9")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("b")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("baa")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); }
public void TestTokensRules() /*throws Exception*/ { Grammar pg = new Grammar( "parser grammar p;\n" + "a : (INT|FLOAT|WS)+;\n"); Grammar g = new Grammar(); g.ImportTokenVocabulary(pg); g.FileName = "<string>"; g.SetGrammarContent( "lexer grammar t;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ {channel=99;};\n"); ICharStream input = new ANTLRStringStream("123 139.52"); Interpreter lexEngine = new Interpreter(g, input); CommonTokenStream tokens = new CommonTokenStream(lexEngine); tokens.LT(5); // make sure it grabs all tokens string result = tokens.ToString(); //System.out.println(result); string expecting = "123 139.52"; Assert.AreEqual(expecting, result); }
public void TestInsertInPriorReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 2, "x"); tokens.InsertBefore(1, "0"); Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestToStringStartStop2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n"); // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0 + 2 * 0;"; Assert.AreEqual(expecting, result); tokens.Replace(4, 8, "0"); // replace 3 * 0 with 0 result = tokens.ToString(); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 17); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(4, 8); expecting = "0"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 8); expecting = "x = 0"; Assert.AreEqual(expecting, result); result = tokens.ToString(12, 16); expecting = "2 * 0"; Assert.AreEqual(expecting, result); tokens.InsertAfter(17, "// comment"); result = tokens.ToString(12, 18); expecting = "2 * 0;// comment"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 8); // try again after insert at end expecting = "x = 0"; Assert.AreEqual(expecting, result); }
public void TestSingleRuleRef() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a' B 'c' ;\n" + "B : 'b' ;\n"); int Atype = g.GetTokenType("A"); Interpreter engine = new Interpreter(g, new ANTLRStringStream("abc")); // should ignore the x IToken result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); }
public void TestSimpleLoops() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n"); int Atype = g.GetTokenType("A"); ICharStream input = new ANTLRStringStream("1234.5"); Interpreter engine = new Interpreter(g, input); IToken result = engine.Scan("A"); Assert.AreEqual(Atype, result.Type); }
public void TestSimpleLoop() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "INT : (DIGIT)+ ;\n" + "fragment DIGIT : '0'..'9';\n"); int INTtype = g.GetTokenType("INT"); Interpreter engine = new Interpreter(g, new ANTLRStringStream("12x")); // should ignore the x IToken result = engine.Scan("INT"); Assert.AreEqual(result.Type, INTtype); engine = new Interpreter(g, new ANTLRStringStream("1234")); result = engine.Scan("INT"); Assert.AreEqual(result.Type, INTtype); }
public void TestSimpleAltCharTest() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a' | 'b' | 'c';"); int Atype = g.GetTokenType("A"); Interpreter engine = new Interpreter(g, new ANTLRStringStream("a")); engine = new Interpreter(g, new ANTLRStringStream("b")); IToken result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); engine = new Interpreter(g, new ANTLRStringStream("c")); result = engine.Scan("A"); Assert.AreEqual(result.Type, Atype); }
public void Test2InsertMiddleIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 1, "x" ); tokens.InsertBefore( 1, "y" ); string result = tokens.ToString(); string expecting = "ayxbc"; Assert.AreEqual( expecting, result ); }
public void TestInsertAfterLastIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.InsertAfter(2, "x"); string result = tokens.ToString(); string expecting = "abcx"; Assert.AreEqual(expecting, result); }
public void TestInsertBeforeIndex0() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.InsertBefore(0, "0"); string result = tokens.ToString(); string expecting = "0abc"; assertEquals(expecting, result); }
public void TestReplaceSubsetThenFetch() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "xyz"); string result = tokens.ToString(0, 6); string expecting = "abxyzba"; Assert.AreEqual(expecting, result); }
public void TestReplaceAll() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 6, "x"); string result = tokens.ToString(); string expecting = "x"; assertEquals(expecting, result); }
public void TestReplaceThenDeleteMiddleIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, "x"); tokens.Delete(1); string result = tokens.ToString(); string expecting = "ac"; Assert.AreEqual(expecting, result); }
public void TestInsertThenReplaceSameIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(0, "0"); tokens.Replace(0, "x"); // supercedes insert at 0 string result = tokens.ToString(); string expecting = "0xbc"; Assert.AreEqual(expecting, result); }
public void TestReplaceRangeThenInsertAtLeftEdge() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "x"); tokens.InsertBefore(2, "y"); string result = tokens.ToString(); string expecting = "abyxba"; Assert.AreEqual(expecting, result); }
public void TestCombineInsertOnLeftWithReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 2, "foo"); tokens.InsertBefore(0, "z"); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "zfoo"; assertEquals(expecting, result); }
public void TestCombineInsertOnLeftWithDelete() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Delete(0, 2); tokens.InsertBefore(0, "z"); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "z"; // make sure combo is not znull Assert.AreEqual(expecting, result); }
public void TestOverlappingReplace4() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, 2, "foo"); tokens.Replace(1, 3, "bar"); // wipes prior nested replace string result = tokens.ToString(); string expecting = "abar"; Assert.AreEqual(expecting, result); }
public void TestDropIdenticalReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, 2, "foo"); tokens.Replace(1, 2, "foo"); // drop previous, identical string result = tokens.ToString(); string expecting = "afooc"; Assert.AreEqual(expecting, result); }
public void TestDropPrevCoveredInsert() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(1, "foo"); tokens.Replace(1, 2, "foo"); // kill prev insert string result = tokens.ToString(); string expecting = "afoofoo"; Assert.AreEqual(expecting, result); }
public void TestLeaveAloneDisjointInsert2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 3, "foo"); tokens.InsertBefore(1, "x"); string result = tokens.ToString(); string expecting = "axbfoo"; Assert.AreEqual(expecting, result); }
public void TestInsertBeforeTokenThenDeleteThatToken() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(2, "y"); tokens.Delete(2); string result = tokens.ToString(); string expecting = "aby"; Assert.AreEqual(expecting, result); }
public void TestReplaceSingleMiddleThenOverlappingSuperset() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 2, "xyz"); tokens.Replace(0, 3, "foo"); string result = tokens.ToString(); string expecting = "fooa"; Assert.AreEqual(expecting, result); }
public void TestCombine3Inserts() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(1, "x"); tokens.InsertBefore(0, "y"); tokens.InsertBefore(1, "z"); string result = tokens.ToString(); string expecting = "yazxbc"; Assert.AreEqual(expecting, result); }
public void TestMismatchedTokenError() { Assert.Inconclusive("May be failing on just my port..."); Grammar pg = new Grammar( "parser grammar p;\n" + "prog : WHILE ID LCURLY (assign)* RCURLY;\n" + "assign : ID ASSIGN expr SEMI ;\n" + "expr : INT | FLOAT | ID ;\n"); Grammar g = new Grammar(); g.FileName = Grammar.IGNORE_STRING_IN_GRAMMAR_FILE_NAME + "string"; g.ImportTokenVocabulary(pg); g.SetGrammarContent( "lexer grammar t;\n" + "WHILE : 'while';\n" + "LCURLY : '{';\n" + "RCURLY : '}';\n" + "ASSIGN : '=';\n" + "SEMI : ';';\n" + "ID : ('a'..'z')+ ;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ ;\n"); ICharStream input = new ANTLRStringStream("while x { i=1 y=3.42; z=y; }"); Interpreter lexEngine = new Interpreter(g, input); FilteringTokenStream tokens = new FilteringTokenStream(lexEngine); tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99); //System.out.println("tokens="+tokens.toString()); Interpreter parseEngine = new Interpreter(pg, tokens); ParseTree t = parseEngine.Parse("prog"); string result = t.ToStringTree(); string expecting = "(<grammar p> (prog while x { (assign i = (expr 1) MismatchedTokenException(5!=9))))"; assertEquals(expecting, result); }
public void TestNoViableAltError() { Grammar pg = new Grammar( "parser grammar p;\n" + "prog : WHILE ID LCURLY (assign)* RCURLY;\n" + "assign : ID ASSIGN expr SEMI ;\n" + "expr : {;}INT | FLOAT | ID ;\n"); Grammar g = new Grammar(); g.ImportTokenVocabulary(pg); g.FileName = "<string>"; g.SetGrammarContent( "lexer grammar t;\n" + "WHILE : 'while';\n" + "LCURLY : '{';\n" + "RCURLY : '}';\n" + "ASSIGN : '=';\n" + "SEMI : ';';\n" + "ID : ('a'..'z')+ ;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ ;\n"); ICharStream input = new ANTLRStringStream("while x { i=; y=3.42; z=y; }"); Interpreter lexEngine = new Interpreter(g, input); FilteringTokenStream tokens = new FilteringTokenStream(lexEngine); tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99); //System.out.println("tokens="+tokens.toString()); Interpreter parseEngine = new Interpreter(pg, tokens); ParseTree t = parseEngine.Parse("prog"); string result = t.ToStringTree(); string expecting = "(<grammar p> (prog while x { (assign i = (expr NoViableAltException(10@[4:1: expr : ( INT | FLOAT | ID );])))))"; Assert.AreEqual(expecting, result); }
public void TestToStringStartStop() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "MUL : '*';\n" + "ASSIGN : '=';\n" + "WS : ' '+;\n"); // Tokens: 0123456789 // Input: x = 3 * 0; ICharStream input = new ANTLRStringStream("x = 3 * 0;"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(4, 8, "0"); // replace 3 * 0 with 0 string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(); expecting = "x = 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 9); expecting = "x = 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(4, 8); expecting = "0"; Assert.AreEqual(expecting, result); }
public void TestReplaceRangeThenInsertAtRightEdge() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "x" ); tokens.InsertBefore( 4, "y" ); // no effect; within range of a replace Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestReplaceRangeThenInsertAtLeftEdge() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "x" ); tokens.InsertBefore( 2, "y" ); string result = tokens.ToString(); string expecting = "abyxba"; assertEquals( expecting, result ); }
public void TestReplaceThenReplaceSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); tokens.Replace( 3, 5, "foo" ); // overlaps, error Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <[email protected]:\"foo\"> overlap with previous <[email protected]:\"xyz\">"; assertNotNull( exc ); assertEquals( expecting, exc.Message ); }
public void TestTokensRules() { Grammar pg = new Grammar( "parser grammar p;\n" + "a : (INT|FLOAT|WS)+;\n" ); Grammar g = new Grammar(); g.ImportTokenVocabulary( pg ); g.FileName = "<string>"; g.SetGrammarContent( "lexer grammar t;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ {channel=99;};\n" ); ICharStream input = new ANTLRStringStream( "123 139.52" ); Interpreter lexEngine = new Interpreter( g, input ); CommonTokenStream tokens = new CommonTokenStream( lexEngine ); tokens.LT(5); // make sure it grabs all tokens string result = tokens.ToString(); //System.out.println(result); string expecting = "123 139.52"; Assert.AreEqual( expecting, result ); }
public void TestSimpleLoops() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n" ); int Atype = g.GetTokenType( "A" ); ICharStream input = new ANTLRStringStream( "1234.5" ); Interpreter engine = new Interpreter( g, input ); IToken result = engine.Scan( "A" ); Assert.AreEqual( Atype, result.Type ); }
public void TestSimpleAltCharTest() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a' | 'b' | 'c';" ); int Atype = g.GetTokenType( "A" ); Interpreter engine = new Interpreter( g, new ANTLRStringStream( "a" ) ); engine = new Interpreter( g, new ANTLRStringStream( "b" ) ); IToken result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "c" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); }
public void TestLeaveAloneDisjointInsert2() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 3, "foo" ); tokens.InsertBefore( 1, "x" ); string result = tokens.ToString(); string expecting = "axbfoo"; Assert.AreEqual( expecting, result ); }
public void TestReplaceThenReplaceSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); tokens.Replace( 3, 5, "foo" ); // overlaps, error Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<6>,1:3]..[@5,5:5='b',<5>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestDropPrevCoveredInsert() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 1, "foo" ); tokens.Replace( 1, 2, "foo" ); // kill prev insert string result = tokens.ToString(); string expecting = "afoofoo"; Assert.AreEqual( expecting, result ); }
public void TestSimpleParse() { Grammar pg = new Grammar( "parser grammar p;\n" + "prog : WHILE ID LCURLY (assign)* RCURLY EOF;\n" + "assign : ID ASSIGN expr SEMI ;\n" + "expr : INT | FLOAT | ID ;\n"); Grammar g = new Grammar(); g.ImportTokenVocabulary(pg); g.FileName = Grammar.IGNORE_STRING_IN_GRAMMAR_FILE_NAME + "string"; g.SetGrammarContent( "lexer grammar t;\n" + "WHILE : 'while';\n" + "LCURLY : '{';\n" + "RCURLY : '}';\n" + "ASSIGN : '=';\n" + "SEMI : ';';\n" + "ID : ('a'..'z')+ ;\n" + "INT : (DIGIT)+ ;\n" + "FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" + "fragment DIGIT : '0'..'9';\n" + "WS : (' ')+ ;\n"); ICharStream input = new ANTLRStringStream("while x { i=1; y=3.42; z=y; }"); Interpreter lexEngine = new Interpreter(g, input); FilteringTokenStream tokens = new FilteringTokenStream(lexEngine); tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99); //System.out.println("tokens="+tokens.toString()); Interpreter parseEngine = new Interpreter(pg, tokens); ParseTree t = parseEngine.Parse("prog"); string result = t.ToStringTree(); string expecting = "(<grammar p> (prog while x { (assign i = (expr 1) ;) (assign y = (expr 3.42) ;) (assign z = (expr y) ;) } <EOF>))"; Assert.AreEqual(expecting, result); }
public void TestToStringStartStop() { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "MUL : '*';\n" + "ASSIGN : '=';\n" + "WS : ' '+;\n" ); // Tokens: 0123456789 // Input: x = 3 * 0; ICharStream input = new ANTLRStringStream( "x = 3 * 0;" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0 string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString(); expecting = "x = 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 9 ); expecting = "x = 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 4, 8 ); expecting = "0"; Assert.AreEqual( expecting, result ); }
public void TestToStringStartStop2() { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n" ); // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; ICharStream input = new ANTLRStringStream( "x = 3 * 0 + 2 * 0;" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0 result = tokens.ToString(); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 17 ); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 4, 8 ); expecting = "0"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 8 ); expecting = "x = 0"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 12, 16 ); expecting = "2 * 0"; Assert.AreEqual( expecting, result ); tokens.InsertAfter( 17, "// comment" ); result = tokens.ToString( 12, 18 ); expecting = "2 * 0;// comment"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 8 ); // try again after insert at end expecting = "x = 0"; Assert.AreEqual( expecting, result ); }
public void TestCombineInsertOnLeftWithReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 2, "foo" ); tokens.InsertBefore( 0, "z" ); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "zfoo"; Assert.AreEqual( expecting, result ); }
public void TestOverlappingReplace4() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, 2, "foo" ); tokens.Replace( 1, 3, "bar" ); // wipes prior nested replace string result = tokens.ToString(); string expecting = "abar"; Assert.AreEqual( expecting, result ); }
public void TestOverlappingReplace2() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 3, "bar" ); tokens.Replace( 1, 2, "foo" ); // cannot split earlier replace Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestInsertBeforeTokenThenDeleteThatToken() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(2, "y"); tokens.Delete(2); string result = tokens.ToString(); string expecting = "aby"; Assert.AreEqual(expecting, result); }
public void TestSimpleLoop() { Grammar g = new Grammar( "lexer grammar t;\n" + "INT : (DIGIT)+ ;\n" + "fragment DIGIT : '0'..'9';\n" ); int INTtype = g.GetTokenType( "INT" ); Interpreter engine = new Interpreter( g, new ANTLRStringStream( "12x" ) ); // should ignore the x IToken result = engine.Scan( "INT" ); Assert.AreEqual( result.Type, INTtype ); engine = new Interpreter( g, new ANTLRStringStream( "1234" ) ); result = engine.Scan( "INT" ); Assert.AreEqual( result.Type, INTtype ); }
public void TestInsertInPriorReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 2, "x" ); tokens.InsertBefore( 1, "0" ); Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestSingleRuleRef() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a' B 'c' ;\n" + "B : 'b' ;\n" ); int Atype = g.GetTokenType( "A" ); Interpreter engine = new Interpreter( g, new ANTLRStringStream( "abc" ) ); // should ignore the x IToken result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); }
public void TestReplaceThenInsertAfterLastIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, "x" ); tokens.InsertAfter( 2, "y" ); string result = tokens.ToString(); string expecting = "abxy"; Assert.AreEqual( expecting, result ); }
public void TestMultAltLoop() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : ('0'..'9'|'a'|'b')+ ;\n" ); int Atype = g.GetTokenType( "A" ); Interpreter engine = new Interpreter( g, new ANTLRStringStream( "a" ) ); IToken result = engine.Scan( "A" ); engine = new Interpreter( g, new ANTLRStringStream( "a" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "1234" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "aaa" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "aaaa9" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "b" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); engine = new Interpreter( g, new ANTLRStringStream( "baa" ) ); result = engine.Scan( "A" ); Assert.AreEqual( result.Type, Atype ); }
public void Test2InsertThenReplaceIndex0() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 0, "x" ); tokens.InsertBefore( 0, "y" ); tokens.Replace( 0, "z" ); string result = tokens.ToString(); string expecting = "zbc"; assertEquals( expecting, result ); }
public void TestReplaceThenDeleteMiddleIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, "x" ); tokens.Delete( 1 ); string result = tokens.ToString(); string expecting = "ac"; assertEquals( expecting, result ); }
public void TestReplaceSubsetThenFetch() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); string result = tokens.ToString( 0, 6 ); string expecting = "abxyzba"; Assert.AreEqual( expecting, result ); }
public void TestOverlappingReplace2() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 3, "bar" ); tokens.Replace( 1, 2, "foo" ); // cannot split earlier replace Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <[email protected]:\"foo\"> overlap with previous <[email protected]:\"bar\">"; assertNotNull( exc ); assertEquals( expecting, exc.Message ); }
public void TestReplaceSingleMiddleThenOverlappingSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 2, "xyz" ); tokens.Replace( 0, 3, "foo" ); string result = tokens.ToString(); string expecting = "fooa"; Assert.AreEqual( expecting, result ); }
public void TestDropIdenticalReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, 2, "foo" ); tokens.Replace( 1, 2, "foo" ); // drop previous, identical string result = tokens.ToString(); string expecting = "afooc"; Assert.AreEqual( expecting, result ); }
public void TestInsertThenReplaceSameIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 0, "0" ); tokens.Replace( 0, "x" ); // supercedes insert at 0 string result = tokens.ToString(); string expecting = "0xbc"; Assert.AreEqual( expecting, result ); }