public void TestReplaceRangeThenInsertAtRightEdge() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcccba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 4, "x");
tokens.InsertBefore(4, "y"); // no effect; within range of a replace
Exception exc = null;
try
{
tokens.ToString();
}
catch (ArgumentException iae)
{
exc = iae;
}
string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">";
Assert.IsNotNull(exc);
Assert.AreEqual(expecting, exc.Message);
}
public void TestMismatchedSetError()
{
Grammar pg = new Grammar(
"parser grammar p;\n" +
"prog : WHILE ID LCURLY (assign)* RCURLY;\n" +
"assign : ID ASSIGN expr SEMI ;\n" +
"expr : INT | FLOAT | ID ;\n");
Grammar g = new Grammar();
g.ImportTokenVocabulary(pg);
g.FileName = "<string>";
g.SetGrammarContent(
"lexer grammar t;\n" +
"WHILE : 'while';\n" +
"LCURLY : '{';\n" +
"RCURLY : '}';\n" +
"ASSIGN : '=';\n" +
"SEMI : ';';\n" +
"ID : ('a'..'z')+ ;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ ;\n");
ICharStream input = new ANTLRStringStream("while x { i=; y=3.42; z=y; }");
Interpreter lexEngine = new Interpreter(g, input);
FilteringTokenStream tokens = new FilteringTokenStream(lexEngine);
tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99);
//System.out.println("tokens="+tokens.toString());
Interpreter parseEngine = new Interpreter(pg, tokens);
ParseTree t = parseEngine.Parse("prog");
string result = t.ToStringTree();
string expecting =
"(<grammar p> (prog while x { (assign i = (expr MismatchedSetException(10!={5,6,7})))))";
Assert.AreEqual(expecting, result);
}
public void TestOverlappingReplace2() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(0, 3, "bar");
tokens.Replace(1, 2, "foo"); // cannot split earlier replace
Exception exc = null;
try
{
tokens.ToString();
}
catch (ArgumentException iae)
{
exc = iae;
}
string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">";
Assert.IsNotNull(exc);
Assert.AreEqual(expecting, exc.Message);
}
public void TestReplaceThenReplaceLowerIndexedSuperset() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcccba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 4, "xyz");
tokens.Replace(1, 3, "foo"); // overlap, error
Exception exc = null;
try
{
tokens.ToString();
}
catch (ArgumentException iae)
{
exc = iae;
}
string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@3,3:3='c',<6>,1:3]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">";
Assert.IsNotNull(exc);
Assert.AreEqual(expecting, exc.Message);
}
public void TestMultAltLoop() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : ('0'..'9'|'a'|'b')+ ;\n");
int Atype = g.GetTokenType("A");
Interpreter engine = new Interpreter(g, new ANTLRStringStream("a"));
IToken result = engine.Scan("A");
engine = new Interpreter(g, new ANTLRStringStream("a"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("1234"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("aaa"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("aaaa9"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("b"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("baa"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
}
public void TestTokensRules() /*throws Exception*/
{
Grammar pg = new Grammar(
"parser grammar p;\n" +
"a : (INT|FLOAT|WS)+;\n");
Grammar g = new Grammar();
g.ImportTokenVocabulary(pg);
g.FileName = "<string>";
g.SetGrammarContent(
"lexer grammar t;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ {channel=99;};\n");
ICharStream input = new ANTLRStringStream("123 139.52");
Interpreter lexEngine = new Interpreter(g, input);
CommonTokenStream tokens = new CommonTokenStream(lexEngine);
tokens.LT(5); // make sure it grabs all tokens
string result = tokens.ToString();
//System.out.println(result);
string expecting = "123 139.52";
Assert.AreEqual(expecting, result);
}
public void TestInsertInPriorReplace() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(0, 2, "x");
tokens.InsertBefore(1, "0");
Exception exc = null;
try
{
tokens.ToString();
}
catch (ArgumentException iae)
{
exc = iae;
}
string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">";
Assert.IsNotNull(exc);
Assert.AreEqual(expecting, exc.Message);
}
public void TestToStringStartStop2() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
string result = tokens.ToOriginalString();
string expecting = "x = 3 * 0 + 2 * 0;";
Assert.AreEqual(expecting, result);
tokens.Replace(4, 8, "0"); // replace 3 * 0 with 0
result = tokens.ToString();
expecting = "x = 0 + 2 * 0;";
Assert.AreEqual(expecting, result);
result = tokens.ToString(0, 17);
expecting = "x = 0 + 2 * 0;";
Assert.AreEqual(expecting, result);
result = tokens.ToString(4, 8);
expecting = "0";
Assert.AreEqual(expecting, result);
result = tokens.ToString(0, 8);
expecting = "x = 0";
Assert.AreEqual(expecting, result);
result = tokens.ToString(12, 16);
expecting = "2 * 0";
Assert.AreEqual(expecting, result);
tokens.InsertAfter(17, "// comment");
result = tokens.ToString(12, 18);
expecting = "2 * 0;// comment";
Assert.AreEqual(expecting, result);
result = tokens.ToString(0, 8); // try again after insert at end
expecting = "x = 0";
Assert.AreEqual(expecting, result);
}
public void TestSingleRuleRef() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' B 'c' ;\n" +
"B : 'b' ;\n");
int Atype = g.GetTokenType("A");
Interpreter engine = new Interpreter(g, new ANTLRStringStream("abc")); // should ignore the x
IToken result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
}
public void TestSimpleLoops() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n");
int Atype = g.GetTokenType("A");
ICharStream input = new ANTLRStringStream("1234.5");
Interpreter engine = new Interpreter(g, input);
IToken result = engine.Scan("A");
Assert.AreEqual(Atype, result.Type);
}
public void TestSimpleLoop() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"INT : (DIGIT)+ ;\n" +
"fragment DIGIT : '0'..'9';\n");
int INTtype = g.GetTokenType("INT");
Interpreter engine = new Interpreter(g, new ANTLRStringStream("12x")); // should ignore the x
IToken result = engine.Scan("INT");
Assert.AreEqual(result.Type, INTtype);
engine = new Interpreter(g, new ANTLRStringStream("1234"));
result = engine.Scan("INT");
Assert.AreEqual(result.Type, INTtype);
}
public void TestSimpleAltCharTest() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' | 'b' | 'c';");
int Atype = g.GetTokenType("A");
Interpreter engine = new Interpreter(g, new ANTLRStringStream("a"));
engine = new Interpreter(g, new ANTLRStringStream("b"));
IToken result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
engine = new Interpreter(g, new ANTLRStringStream("c"));
result = engine.Scan("A");
Assert.AreEqual(result.Type, Atype);
}
public void Test2InsertMiddleIndex()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.InsertBefore( 1, "x" );
tokens.InsertBefore( 1, "y" );
string result = tokens.ToString();
string expecting = "ayxbc";
Assert.AreEqual( expecting, result );
}
public void TestInsertAfterLastIndex() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.InsertAfter(2, "x");
string result = tokens.ToString();
string expecting = "abcx";
Assert.AreEqual(expecting, result);
}
public void TestInsertBeforeIndex0() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.InsertBefore(0, "0");
string result = tokens.ToString();
string expecting = "0abc";
assertEquals(expecting, result);
}
public void TestReplaceSubsetThenFetch() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcccba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 4, "xyz");
string result = tokens.ToString(0, 6);
string expecting = "abxyzba";
Assert.AreEqual(expecting, result);
}
public void TestReplaceAll() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcccba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(0, 6, "x");
string result = tokens.ToString();
string expecting = "x";
assertEquals(expecting, result);
}
public void TestReplaceThenDeleteMiddleIndex() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(1, "x");
tokens.Delete(1);
string result = tokens.ToString();
string expecting = "ac";
Assert.AreEqual(expecting, result);
}
public void TestInsertThenReplaceSameIndex() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(0, "0");
tokens.Replace(0, "x"); // supercedes insert at 0
string result = tokens.ToString();
string expecting = "0xbc";
Assert.AreEqual(expecting, result);
}
public void TestReplaceRangeThenInsertAtLeftEdge() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcccba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 4, "x");
tokens.InsertBefore(2, "y");
string result = tokens.ToString();
string expecting = "abyxba";
Assert.AreEqual(expecting, result);
}
public void TestCombineInsertOnLeftWithReplace() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(0, 2, "foo");
tokens.InsertBefore(0, "z"); // combine with left edge of rewrite
string result = tokens.ToString();
string expecting = "zfoo";
assertEquals(expecting, result);
}
public void TestCombineInsertOnLeftWithDelete() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Delete(0, 2);
tokens.InsertBefore(0, "z"); // combine with left edge of rewrite
string result = tokens.ToString();
string expecting = "z"; // make sure combo is not znull
Assert.AreEqual(expecting, result);
}
public void TestOverlappingReplace4() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(1, 2, "foo");
tokens.Replace(1, 3, "bar"); // wipes prior nested replace
string result = tokens.ToString();
string expecting = "abar";
Assert.AreEqual(expecting, result);
}
public void TestDropIdenticalReplace() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(1, 2, "foo");
tokens.Replace(1, 2, "foo"); // drop previous, identical
string result = tokens.ToString();
string expecting = "afooc";
Assert.AreEqual(expecting, result);
}
public void TestDropPrevCoveredInsert() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(1, "foo");
tokens.Replace(1, 2, "foo"); // kill prev insert
string result = tokens.ToString();
string expecting = "afoofoo";
Assert.AreEqual(expecting, result);
}
public void TestLeaveAloneDisjointInsert2() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 3, "foo");
tokens.InsertBefore(1, "x");
string result = tokens.ToString();
string expecting = "axbfoo";
Assert.AreEqual(expecting, result);
}
public void TestInsertBeforeTokenThenDeleteThatToken()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(2, "y");
tokens.Delete(2);
string result = tokens.ToString();
string expecting = "aby";
Assert.AreEqual(expecting, result);
}
public void TestReplaceSingleMiddleThenOverlappingSuperset() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abcba");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(2, 2, "xyz");
tokens.Replace(0, 3, "foo");
string result = tokens.ToString();
string expecting = "fooa";
Assert.AreEqual(expecting, result);
}
public void TestCombine3Inserts() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(1, "x");
tokens.InsertBefore(0, "y");
tokens.InsertBefore(1, "z");
string result = tokens.ToString();
string expecting = "yazxbc";
Assert.AreEqual(expecting, result);
}
public void TestMismatchedTokenError()
{
Assert.Inconclusive("May be failing on just my port...");
Grammar pg = new Grammar(
"parser grammar p;\n" +
"prog : WHILE ID LCURLY (assign)* RCURLY;\n" +
"assign : ID ASSIGN expr SEMI ;\n" +
"expr : INT | FLOAT | ID ;\n");
Grammar g = new Grammar();
g.FileName = Grammar.IGNORE_STRING_IN_GRAMMAR_FILE_NAME + "string";
g.ImportTokenVocabulary(pg);
g.SetGrammarContent(
"lexer grammar t;\n" +
"WHILE : 'while';\n" +
"LCURLY : '{';\n" +
"RCURLY : '}';\n" +
"ASSIGN : '=';\n" +
"SEMI : ';';\n" +
"ID : ('a'..'z')+ ;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ ;\n");
ICharStream input = new ANTLRStringStream("while x { i=1 y=3.42; z=y; }");
Interpreter lexEngine = new Interpreter(g, input);
FilteringTokenStream tokens = new FilteringTokenStream(lexEngine);
tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99);
//System.out.println("tokens="+tokens.toString());
Interpreter parseEngine = new Interpreter(pg, tokens);
ParseTree t = parseEngine.Parse("prog");
string result = t.ToStringTree();
string expecting =
"(<grammar p> (prog while x { (assign i = (expr 1) MismatchedTokenException(5!=9))))";
assertEquals(expecting, result);
}
public void TestNoViableAltError()
{
Grammar pg = new Grammar(
"parser grammar p;\n" +
"prog : WHILE ID LCURLY (assign)* RCURLY;\n" +
"assign : ID ASSIGN expr SEMI ;\n" +
"expr : {;}INT | FLOAT | ID ;\n");
Grammar g = new Grammar();
g.ImportTokenVocabulary(pg);
g.FileName = "<string>";
g.SetGrammarContent(
"lexer grammar t;\n" +
"WHILE : 'while';\n" +
"LCURLY : '{';\n" +
"RCURLY : '}';\n" +
"ASSIGN : '=';\n" +
"SEMI : ';';\n" +
"ID : ('a'..'z')+ ;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ ;\n");
ICharStream input = new ANTLRStringStream("while x { i=; y=3.42; z=y; }");
Interpreter lexEngine = new Interpreter(g, input);
FilteringTokenStream tokens = new FilteringTokenStream(lexEngine);
tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99);
//System.out.println("tokens="+tokens.toString());
Interpreter parseEngine = new Interpreter(pg, tokens);
ParseTree t = parseEngine.Parse("prog");
string result = t.ToStringTree();
string expecting =
"(<grammar p> (prog while x { (assign i = (expr NoViableAltException(10@[4:1: expr : ( INT | FLOAT | ID );])))))";
Assert.AreEqual(expecting, result);
}
public void TestToStringStartStop() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"MUL : '*';\n" +
"ASSIGN : '=';\n" +
"WS : ' '+;\n");
// Tokens: 0123456789
// Input: x = 3 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(4, 8, "0"); // replace 3 * 0 with 0
string result = tokens.ToOriginalString();
string expecting = "x = 3 * 0;";
Assert.AreEqual(expecting, result);
result = tokens.ToString();
expecting = "x = 0;";
Assert.AreEqual(expecting, result);
result = tokens.ToString(0, 9);
expecting = "x = 0;";
Assert.AreEqual(expecting, result);
result = tokens.ToString(4, 8);
expecting = "0";
Assert.AreEqual(expecting, result);
}
public void TestReplaceRangeThenInsertAtRightEdge()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcccba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 4, "x" );
tokens.InsertBefore( 4, "y" ); // no effect; within range of a replace
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">";
Assert.IsNotNull( exc );
Assert.AreEqual( expecting, exc.Message );
}
public void TestReplaceRangeThenInsertAtLeftEdge()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcccba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 4, "x" );
tokens.InsertBefore( 2, "y" );
string result = tokens.ToString();
string expecting = "abyxba";
assertEquals( expecting, result );
}
public void TestReplaceThenReplaceSuperset()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcccba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 4, "xyz" );
tokens.Replace( 3, 5, "foo" ); // overlaps, error
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "replace op boundaries of <[email protected]:\"foo\"> overlap with previous <[email protected]:\"xyz\">";
assertNotNull( exc );
assertEquals( expecting, exc.Message );
}
public void TestTokensRules()
{
Grammar pg = new Grammar(
"parser grammar p;\n" +
"a : (INT|FLOAT|WS)+;\n" );
Grammar g = new Grammar();
g.ImportTokenVocabulary( pg );
g.FileName = "<string>";
g.SetGrammarContent(
"lexer grammar t;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ {channel=99;};\n" );
ICharStream input = new ANTLRStringStream( "123 139.52" );
Interpreter lexEngine = new Interpreter( g, input );
CommonTokenStream tokens = new CommonTokenStream( lexEngine );
tokens.LT(5); // make sure it grabs all tokens
string result = tokens.ToString();
//System.out.println(result);
string expecting = "123 139.52";
Assert.AreEqual( expecting, result );
}
public void TestSimpleLoops()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : ('0'..'9')+ '.' ('0'..'9')* | ('0'..'9')+ ;\n" );
int Atype = g.GetTokenType( "A" );
ICharStream input = new ANTLRStringStream( "1234.5" );
Interpreter engine = new Interpreter( g, input );
IToken result = engine.Scan( "A" );
Assert.AreEqual( Atype, result.Type );
}
public void TestSimpleAltCharTest()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' | 'b' | 'c';" );
int Atype = g.GetTokenType( "A" );
Interpreter engine = new Interpreter( g, new ANTLRStringStream( "a" ) );
engine = new Interpreter( g, new ANTLRStringStream( "b" ) );
IToken result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "c" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
}
public void TestLeaveAloneDisjointInsert2()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 3, "foo" );
tokens.InsertBefore( 1, "x" );
string result = tokens.ToString();
string expecting = "axbfoo";
Assert.AreEqual( expecting, result );
}
public void TestReplaceThenReplaceSuperset()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcccba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 4, "xyz" );
tokens.Replace( 3, 5, "foo" ); // overlaps, error
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<6>,1:3]..[@5,5:5='b',<5>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">";
Assert.IsNotNull( exc );
Assert.AreEqual( expecting, exc.Message );
}
public void TestDropPrevCoveredInsert()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.InsertBefore( 1, "foo" );
tokens.Replace( 1, 2, "foo" ); // kill prev insert
string result = tokens.ToString();
string expecting = "afoofoo";
Assert.AreEqual( expecting, result );
}
public void TestSimpleParse()
{
Grammar pg = new Grammar(
"parser grammar p;\n" +
"prog : WHILE ID LCURLY (assign)* RCURLY EOF;\n" +
"assign : ID ASSIGN expr SEMI ;\n" +
"expr : INT | FLOAT | ID ;\n");
Grammar g = new Grammar();
g.ImportTokenVocabulary(pg);
g.FileName = Grammar.IGNORE_STRING_IN_GRAMMAR_FILE_NAME + "string";
g.SetGrammarContent(
"lexer grammar t;\n" +
"WHILE : 'while';\n" +
"LCURLY : '{';\n" +
"RCURLY : '}';\n" +
"ASSIGN : '=';\n" +
"SEMI : ';';\n" +
"ID : ('a'..'z')+ ;\n" +
"INT : (DIGIT)+ ;\n" +
"FLOAT : (DIGIT)+ '.' (DIGIT)* ;\n" +
"fragment DIGIT : '0'..'9';\n" +
"WS : (' ')+ ;\n");
ICharStream input = new ANTLRStringStream("while x { i=1; y=3.42; z=y; }");
Interpreter lexEngine = new Interpreter(g, input);
FilteringTokenStream tokens = new FilteringTokenStream(lexEngine);
tokens.SetTokenTypeChannel(g.GetTokenType("WS"), 99);
//System.out.println("tokens="+tokens.toString());
Interpreter parseEngine = new Interpreter(pg, tokens);
ParseTree t = parseEngine.Parse("prog");
string result = t.ToStringTree();
string expecting =
"(<grammar p> (prog while x { (assign i = (expr 1) ;) (assign y = (expr 3.42) ;) (assign z = (expr y) ;) } <EOF>))";
Assert.AreEqual(expecting, result);
}
public void TestToStringStartStop()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"MUL : '*';\n" +
"ASSIGN : '=';\n" +
"WS : ' '+;\n" );
// Tokens: 0123456789
// Input: x = 3 * 0;
ICharStream input = new ANTLRStringStream( "x = 3 * 0;" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0
string result = tokens.ToOriginalString();
string expecting = "x = 3 * 0;";
Assert.AreEqual( expecting, result );
result = tokens.ToString();
expecting = "x = 0;";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 0, 9 );
expecting = "x = 0;";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 4, 8 );
expecting = "0";
Assert.AreEqual( expecting, result );
}
public void TestToStringStartStop2()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n" );
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream( "x = 3 * 0 + 2 * 0;" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
string result = tokens.ToOriginalString();
string expecting = "x = 3 * 0 + 2 * 0;";
Assert.AreEqual( expecting, result );
tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0
result = tokens.ToString();
expecting = "x = 0 + 2 * 0;";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 0, 17 );
expecting = "x = 0 + 2 * 0;";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 4, 8 );
expecting = "0";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 0, 8 );
expecting = "x = 0";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 12, 16 );
expecting = "2 * 0";
Assert.AreEqual( expecting, result );
tokens.InsertAfter( 17, "// comment" );
result = tokens.ToString( 12, 18 );
expecting = "2 * 0;// comment";
Assert.AreEqual( expecting, result );
result = tokens.ToString( 0, 8 ); // try again after insert at end
expecting = "x = 0";
Assert.AreEqual( expecting, result );
}
public void TestCombineInsertOnLeftWithReplace()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 0, 2, "foo" );
tokens.InsertBefore( 0, "z" ); // combine with left edge of rewrite
string result = tokens.ToString();
string expecting = "zfoo";
Assert.AreEqual( expecting, result );
}
public void TestOverlappingReplace4()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 1, 2, "foo" );
tokens.Replace( 1, 3, "bar" ); // wipes prior nested replace
string result = tokens.ToString();
string expecting = "abar";
Assert.AreEqual( expecting, result );
}
public void TestOverlappingReplace2()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 0, 3, "bar" );
tokens.Replace( 1, 2, "foo" ); // cannot split earlier replace
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">";
Assert.IsNotNull( exc );
Assert.AreEqual( expecting, exc.Message );
}
public void TestInsertBeforeTokenThenDeleteThatToken()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(2, "y");
tokens.Delete(2);
string result = tokens.ToString();
string expecting = "aby";
Assert.AreEqual(expecting, result);
}
public void TestSimpleLoop()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"INT : (DIGIT)+ ;\n" +
"fragment DIGIT : '0'..'9';\n" );
int INTtype = g.GetTokenType( "INT" );
Interpreter engine = new Interpreter( g, new ANTLRStringStream( "12x" ) ); // should ignore the x
IToken result = engine.Scan( "INT" );
Assert.AreEqual( result.Type, INTtype );
engine = new Interpreter( g, new ANTLRStringStream( "1234" ) );
result = engine.Scan( "INT" );
Assert.AreEqual( result.Type, INTtype );
}
public void TestInsertInPriorReplace()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 0, 2, "x" );
tokens.InsertBefore( 1, "0" );
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">";
Assert.IsNotNull( exc );
Assert.AreEqual( expecting, exc.Message );
}
public void TestSingleRuleRef()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' B 'c' ;\n" +
"B : 'b' ;\n" );
int Atype = g.GetTokenType( "A" );
Interpreter engine = new Interpreter( g, new ANTLRStringStream( "abc" ) ); // should ignore the x
IToken result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
}
public void TestReplaceThenInsertAfterLastIndex()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, "x" );
tokens.InsertAfter( 2, "y" );
string result = tokens.ToString();
string expecting = "abxy";
Assert.AreEqual( expecting, result );
}
public void TestMultAltLoop()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : ('0'..'9'|'a'|'b')+ ;\n" );
int Atype = g.GetTokenType( "A" );
Interpreter engine = new Interpreter( g, new ANTLRStringStream( "a" ) );
IToken result = engine.Scan( "A" );
engine = new Interpreter( g, new ANTLRStringStream( "a" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "1234" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "aaa" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "aaaa9" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "b" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
engine = new Interpreter( g, new ANTLRStringStream( "baa" ) );
result = engine.Scan( "A" );
Assert.AreEqual( result.Type, Atype );
}
public void Test2InsertThenReplaceIndex0()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.InsertBefore( 0, "x" );
tokens.InsertBefore( 0, "y" );
tokens.Replace( 0, "z" );
string result = tokens.ToString();
string expecting = "zbc";
assertEquals( expecting, result );
}
public void TestReplaceThenDeleteMiddleIndex()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 1, "x" );
tokens.Delete( 1 );
string result = tokens.ToString();
string expecting = "ac";
assertEquals( expecting, result );
}
public void TestReplaceSubsetThenFetch()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcccba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 4, "xyz" );
string result = tokens.ToString( 0, 6 );
string expecting = "abxyzba";
Assert.AreEqual( expecting, result );
}
public void TestOverlappingReplace2()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 0, 3, "bar" );
tokens.Replace( 1, 2, "foo" ); // cannot split earlier replace
Exception exc = null;
try
{
tokens.ToString();
}
catch ( ArgumentException iae )
{
exc = iae;
}
string expecting = "replace op boundaries of <[email protected]:\"foo\"> overlap with previous <[email protected]:\"bar\">";
assertNotNull( exc );
assertEquals( expecting, exc.Message );
}
public void TestReplaceSingleMiddleThenOverlappingSuperset()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcba" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 2, 2, "xyz" );
tokens.Replace( 0, 3, "foo" );
string result = tokens.ToString();
string expecting = "fooa";
Assert.AreEqual( expecting, result );
}
public void TestDropIdenticalReplace()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abcc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 1, 2, "foo" );
tokens.Replace( 1, 2, "foo" ); // drop previous, identical
string result = tokens.ToString();
string expecting = "afooc";
Assert.AreEqual( expecting, result );
}
public void TestInsertThenReplaceSameIndex()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.InsertBefore( 0, "0" );
tokens.Replace( 0, "x" ); // supercedes insert at 0
string result = tokens.ToString();
string expecting = "0xbc";
Assert.AreEqual( expecting, result );
}