private static void KillTrailingNewline(TokenRewriteStream tokens, int index) { IList <IToken> all = tokens.GetTokens(); IToken tok = all[index]; IToken after = all[index + 1]; string ws = after.Text; if (ws.StartsWith("\n")) { //Console.Out.WriteLine( "killing WS after action" ); if (ws.Length > 1) { int space = ws.IndexOf(' '); int tab = ws.IndexOf('\t'); if (ws.StartsWith("\n") && space >= 0 || tab >= 0) { return; // do nothing if \n + indent } // otherwise kill all \n ws = ws.Replace("\n", ""); tokens.Replace(after.TokenIndex, ws); } else { tokens.Delete(after.TokenIndex); } } }
public void TestInsertBeforeTokenThenDeleteThatToken() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(2, "y"); tokens.Delete(2); string result = tokens.ToString(); string expecting = "aby"; Assert.AreEqual(expecting, result); }
public void TestReplaceThenDeleteMiddleIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, "x"); tokens.Delete(1); string result = tokens.ToString(); string expecting = "ac"; Assert.AreEqual(expecting, result); }
public void TestCombineInsertOnLeftWithDelete() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Delete(0, 2); tokens.InsertBefore(0, "z"); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "z"; // make sure combo is not znull Assert.AreEqual(expecting, result); }
public static void ACTION(TokenRewriteStream tokens, CommonTree t) { CommonTree parent = (CommonTree)t.Parent; int ptype = parent.Type; if (ptype == ANTLRParser.SCOPE || // we have special rules for these ptype == ANTLRParser.AMPERSAND) { return; } //Console.Out.WriteLine( "ACTION: " + t.Text ); CommonTree root = (CommonTree)t.GetAncestor(ANTLRParser.RULE); if (root != null) { CommonTree rule = (CommonTree)root.GetChild(0); //Console.Out.WriteLine( "rule: " + rule ); if (!char.IsUpper(rule.Text[0])) { tokens.Delete(t.TokenStartIndex, t.TokenStopIndex); KillTrailingNewline(tokens, t.token.TokenIndex); } } }
public virtual void Rewrite(ITreeAdaptor adaptor, CommonTree t, string[] tokenNames) { TreeWizard wiz = new TreeWizard(adaptor, tokenNames); // ACTIONS STUFF wiz.Visit(t, ANTLRParser.ACTION, (tree) => { ACTION(tokens, (CommonTree)tree); }); // ^('@' id ACTION) rule actions wiz.Visit(t, ANTLRParser.AMPERSAND, (tree) => { CommonTree a = (CommonTree)t; CommonTree action = null; if (a.ChildCount == 2) { action = (CommonTree)a.GetChild(1); } else if (a.ChildCount == 3) { action = (CommonTree)a.GetChild(2); } if (action.Type == ANTLRParser.ACTION) { tokens.Delete(a.TokenStartIndex, a.TokenStopIndex); KillTrailingNewline(tokens, action.TokenStopIndex); } }); wiz.Visit(t, ANTLRParser.ACTION, (tree) => { }); // wipe rule arguments wiz.Visit(t, ANTLRParser.ARG, (tree) => { CommonTree a = (CommonTree)t; a = (CommonTree)a.GetChild(0); tokens.Delete(a.token.TokenIndex); KillTrailingNewline(tokens, a.token.TokenIndex); }); // wipe rule return declarations wiz.Visit(t, ANTLRParser.RET, (tree) => { CommonTree a = (CommonTree)t; CommonTree ret = (CommonTree)a.GetChild(0); tokens.Delete(a.token.TokenIndex, ret.token.TokenIndex); }); // comment out semantic predicates wiz.Visit(t, ANTLRParser.SEMPRED, (tree) => { CommonTree a = (CommonTree)t; tokens.Replace(a.token.TokenIndex, "/*" + a.Text + "*/"); }); // comment out semantic predicates wiz.Visit(t, ANTLRParser.GATED_SEMPRED, (tree) => { CommonTree a = (CommonTree)t; string text = tokens.ToString(a.TokenStartIndex, a.TokenStopIndex); tokens.Replace(a.TokenStartIndex, a.TokenStopIndex, "/*" + text + "*/"); }); // comment scope specs wiz.Visit(t, ANTLRParser.SCOPE, (tree) => { CommonTree a = (CommonTree)t; tokens.Delete(a.TokenStartIndex, a.TokenStopIndex); KillTrailingNewline(tokens, a.TokenStopIndex); }); // args r[x,y] -> ^(r [x,y]) wiz.Visit(t, ANTLRParser.ARG_ACTION, (tree) => { CommonTree a = (CommonTree)t; if (a.Parent.Type == ANTLRParser.RULE_REF) { tokens.Delete(a.TokenStartIndex, a.TokenStopIndex); } }); #if false // what is this token type in the C# ported version of the V3 grammar? // ^('=' id ^(RULE_REF [arg])), ... wiz.Visit(t, ANTLRParser.LABEL_ASSIGN, (tree) => { CommonTree a = (CommonTree)t; if (!a.HasAncestor(ANTLRParser.OPTIONS)) { // avoid options CommonTree child = (CommonTree)a.GetChild(0); // kill "id=" tokens.Delete(a.token.TokenIndex); tokens.Delete(child.token.TokenIndex); } }); #endif #if false // what is this token type in the C# ported version of the V3 grammar? // ^('+=' id ^(RULE_REF [arg])), ... wiz.Visit(t, ANTLRParser.LIST_LABEL_ASSIGN, (tree) => { CommonTree a = (CommonTree)t; CommonTree child = (CommonTree)a.GetChild(0); // kill "id+=" tokens.Delete(a.token.TokenIndex); tokens.Delete(child.token.TokenIndex); }); #endif // AST STUFF wiz.Visit(t, ANTLRParser.REWRITE, (tree) => { CommonTree a = (CommonTree)t; CommonTree child = (CommonTree)a.GetChild(0); int stop = child.TokenStopIndex; if (child.Type == ANTLRParser.SEMPRED) { CommonTree rew = (CommonTree)a.GetChild(1); stop = rew.TokenStopIndex; } tokens.Delete(a.token.TokenIndex, stop); KillTrailingNewline(tokens, stop); }); wiz.Visit(t, ANTLRParser.ROOT, (tree) => { tokens.Delete(((CommonTree)t).token.TokenIndex); }); wiz.Visit(t, ANTLRParser.BANG, (tree) => { tokens.Delete(((CommonTree)t).token.TokenIndex); }); }
public void TestReplaceThenDeleteMiddleIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, "x" ); tokens.Delete( 1 ); string result = tokens.ToString(); string expecting = "ac"; Assert.AreEqual( expecting, result ); }
public void TestCombineInsertOnLeftWithDelete() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Delete( 0, 2 ); tokens.InsertBefore( 0, "z" ); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "z"; // make sure combo is not znull Assert.AreEqual( expecting, result ); }
public static void ACTION( TokenRewriteStream tokens, CommonTree t ) { CommonTree parent = (CommonTree)t.Parent; int ptype = parent.Type; if ( ptype == ANTLRParser.SCOPE || // we have special rules for these ptype == ANTLRParser.AMPERSAND ) { return; } //Console.Out.WriteLine( "ACTION: " + t.Text ); CommonTree root = (CommonTree)t.GetAncestor( ANTLRParser.RULE ); if ( root != null ) { CommonTree rule = (CommonTree)root.GetChild( 0 ); //Console.Out.WriteLine( "rule: " + rule ); if ( Rule.GetRuleType( rule.Text ) == RuleType.Parser ) { tokens.Delete( t.TokenStartIndex, t.TokenStopIndex ); KillTrailingNewline( tokens, t.Token.TokenIndex ); } } }
private static void KillTrailingNewline( TokenRewriteStream tokens, int index ) { IList<IToken> all = tokens.GetTokens(); IToken tok = all[index]; IToken after = all[index + 1]; string ws = after.Text; if ( ws.StartsWith( "\n" ) ) { //Console.Out.WriteLine( "killing WS after action" ); if ( ws.Length > 1 ) { int space = ws.IndexOf( ' ' ); int tab = ws.IndexOf( '\t' ); if ( ws.StartsWith( "\n" ) && space >= 0 || tab >= 0 ) { return; // do nothing if \n + indent } // otherwise kill all \n ws = ws.Replace( "\n", "" ); tokens.Replace( after.TokenIndex, ws ); } else { tokens.Delete( after.TokenIndex ); } } }