private static void KillTrailingNewline(TokenRewriteStream tokens, int index)
{
IList <IToken> all = tokens.GetTokens();
IToken tok = all[index];
IToken after = all[index + 1];
string ws = after.Text;
if (ws.StartsWith("\n"))
{
//Console.Out.WriteLine( "killing WS after action" );
if (ws.Length > 1)
{
int space = ws.IndexOf(' ');
int tab = ws.IndexOf('\t');
if (ws.StartsWith("\n") && space >= 0 || tab >= 0)
{
return; // do nothing if \n + indent
}
// otherwise kill all \n
ws = ws.Replace("\n", "");
tokens.Replace(after.TokenIndex, ws);
}
else
{
tokens.Delete(after.TokenIndex);
}
}
}
public void TestInsertBeforeTokenThenDeleteThatToken()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(2, "y");
tokens.Delete(2);
string result = tokens.ToString();
string expecting = "aby";
Assert.AreEqual(expecting, result);
}
public void TestReplaceThenDeleteMiddleIndex() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Replace(1, "x");
tokens.Delete(1);
string result = tokens.ToString();
string expecting = "ac";
Assert.AreEqual(expecting, result);
}
public void TestCombineInsertOnLeftWithDelete() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.Delete(0, 2);
tokens.InsertBefore(0, "z"); // combine with left edge of rewrite
string result = tokens.ToString();
string expecting = "z"; // make sure combo is not znull
Assert.AreEqual(expecting, result);
}
public static void ACTION(TokenRewriteStream tokens, CommonTree t)
{
CommonTree parent = (CommonTree)t.Parent;
int ptype = parent.Type;
if (ptype == ANTLRParser.SCOPE || // we have special rules for these
ptype == ANTLRParser.AMPERSAND)
{
return;
}
//Console.Out.WriteLine( "ACTION: " + t.Text );
CommonTree root = (CommonTree)t.GetAncestor(ANTLRParser.RULE);
if (root != null)
{
CommonTree rule = (CommonTree)root.GetChild(0);
//Console.Out.WriteLine( "rule: " + rule );
if (!char.IsUpper(rule.Text[0]))
{
tokens.Delete(t.TokenStartIndex, t.TokenStopIndex);
KillTrailingNewline(tokens, t.token.TokenIndex);
}
}
}
public virtual void Rewrite(ITreeAdaptor adaptor, CommonTree t, string[] tokenNames)
{
TreeWizard wiz = new TreeWizard(adaptor, tokenNames);
// ACTIONS STUFF
wiz.Visit(t, ANTLRParser.ACTION, (tree) =>
{
ACTION(tokens, (CommonTree)tree);
});
// ^('@' id ACTION) rule actions
wiz.Visit(t, ANTLRParser.AMPERSAND, (tree) =>
{
CommonTree a = (CommonTree)t;
CommonTree action = null;
if (a.ChildCount == 2)
{
action = (CommonTree)a.GetChild(1);
}
else if (a.ChildCount == 3)
{
action = (CommonTree)a.GetChild(2);
}
if (action.Type == ANTLRParser.ACTION)
{
tokens.Delete(a.TokenStartIndex, a.TokenStopIndex);
KillTrailingNewline(tokens, action.TokenStopIndex);
}
});
wiz.Visit(t, ANTLRParser.ACTION, (tree) =>
{
});
// wipe rule arguments
wiz.Visit(t, ANTLRParser.ARG, (tree) =>
{
CommonTree a = (CommonTree)t;
a = (CommonTree)a.GetChild(0);
tokens.Delete(a.token.TokenIndex);
KillTrailingNewline(tokens, a.token.TokenIndex);
});
// wipe rule return declarations
wiz.Visit(t, ANTLRParser.RET, (tree) =>
{
CommonTree a = (CommonTree)t;
CommonTree ret = (CommonTree)a.GetChild(0);
tokens.Delete(a.token.TokenIndex, ret.token.TokenIndex);
});
// comment out semantic predicates
wiz.Visit(t, ANTLRParser.SEMPRED, (tree) =>
{
CommonTree a = (CommonTree)t;
tokens.Replace(a.token.TokenIndex, "/*" + a.Text + "*/");
});
// comment out semantic predicates
wiz.Visit(t, ANTLRParser.GATED_SEMPRED, (tree) =>
{
CommonTree a = (CommonTree)t;
string text = tokens.ToString(a.TokenStartIndex, a.TokenStopIndex);
tokens.Replace(a.TokenStartIndex, a.TokenStopIndex, "/*" + text + "*/");
});
// comment scope specs
wiz.Visit(t, ANTLRParser.SCOPE, (tree) =>
{
CommonTree a = (CommonTree)t;
tokens.Delete(a.TokenStartIndex, a.TokenStopIndex);
KillTrailingNewline(tokens, a.TokenStopIndex);
});
// args r[x,y] -> ^(r [x,y])
wiz.Visit(t, ANTLRParser.ARG_ACTION, (tree) =>
{
CommonTree a = (CommonTree)t;
if (a.Parent.Type == ANTLRParser.RULE_REF)
{
tokens.Delete(a.TokenStartIndex, a.TokenStopIndex);
}
});
#if false // what is this token type in the C# ported version of the V3 grammar?
// ^('=' id ^(RULE_REF [arg])), ...
wiz.Visit(t, ANTLRParser.LABEL_ASSIGN, (tree) =>
{
CommonTree a = (CommonTree)t;
if (!a.HasAncestor(ANTLRParser.OPTIONS))
{
// avoid options
CommonTree child = (CommonTree)a.GetChild(0);
// kill "id="
tokens.Delete(a.token.TokenIndex);
tokens.Delete(child.token.TokenIndex);
}
});
#endif
#if false // what is this token type in the C# ported version of the V3 grammar?
// ^('+=' id ^(RULE_REF [arg])), ...
wiz.Visit(t, ANTLRParser.LIST_LABEL_ASSIGN, (tree) =>
{
CommonTree a = (CommonTree)t;
CommonTree child = (CommonTree)a.GetChild(0);
// kill "id+="
tokens.Delete(a.token.TokenIndex);
tokens.Delete(child.token.TokenIndex);
});
#endif
// AST STUFF
wiz.Visit(t, ANTLRParser.REWRITE, (tree) =>
{
CommonTree a = (CommonTree)t;
CommonTree child = (CommonTree)a.GetChild(0);
int stop = child.TokenStopIndex;
if (child.Type == ANTLRParser.SEMPRED)
{
CommonTree rew = (CommonTree)a.GetChild(1);
stop = rew.TokenStopIndex;
}
tokens.Delete(a.token.TokenIndex, stop);
KillTrailingNewline(tokens, stop);
});
wiz.Visit(t, ANTLRParser.ROOT, (tree) =>
{
tokens.Delete(((CommonTree)t).token.TokenIndex);
});
wiz.Visit(t, ANTLRParser.BANG, (tree) =>
{
tokens.Delete(((CommonTree)t).token.TokenIndex);
});
}
public void TestReplaceThenDeleteMiddleIndex()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Replace( 1, "x" );
tokens.Delete( 1 );
string result = tokens.ToString();
string expecting = "ac";
Assert.AreEqual( expecting, result );
}
public void TestInsertBeforeTokenThenDeleteThatToken()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n");
ICharStream input = new ANTLRStringStream("abc");
Interpreter lexEngine = new Interpreter(g, input);
TokenRewriteStream tokens = new TokenRewriteStream(lexEngine);
tokens.Fill();
tokens.InsertBefore(2, "y");
tokens.Delete(2);
string result = tokens.ToString();
string expecting = "aby";
Assert.AreEqual(expecting, result);
}
public void TestCombineInsertOnLeftWithDelete()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a';\n" +
"B : 'b';\n" +
"C : 'c';\n" );
ICharStream input = new ANTLRStringStream( "abc" );
Interpreter lexEngine = new Interpreter( g, input );
TokenRewriteStream tokens = new TokenRewriteStream( lexEngine );
tokens.Fill();
tokens.Delete( 0, 2 );
tokens.InsertBefore( 0, "z" ); // combine with left edge of rewrite
string result = tokens.ToString();
string expecting = "z"; // make sure combo is not znull
Assert.AreEqual( expecting, result );
}
public static void ACTION( TokenRewriteStream tokens, CommonTree t )
{
CommonTree parent = (CommonTree)t.Parent;
int ptype = parent.Type;
if ( ptype == ANTLRParser.SCOPE || // we have special rules for these
ptype == ANTLRParser.AMPERSAND )
{
return;
}
//Console.Out.WriteLine( "ACTION: " + t.Text );
CommonTree root = (CommonTree)t.GetAncestor( ANTLRParser.RULE );
if ( root != null )
{
CommonTree rule = (CommonTree)root.GetChild( 0 );
//Console.Out.WriteLine( "rule: " + rule );
if ( Rule.GetRuleType( rule.Text ) == RuleType.Parser )
{
tokens.Delete( t.TokenStartIndex, t.TokenStopIndex );
KillTrailingNewline( tokens, t.Token.TokenIndex );
}
}
}
private static void KillTrailingNewline( TokenRewriteStream tokens, int index )
{
IList<IToken> all = tokens.GetTokens();
IToken tok = all[index];
IToken after = all[index + 1];
string ws = after.Text;
if ( ws.StartsWith( "\n" ) )
{
//Console.Out.WriteLine( "killing WS after action" );
if ( ws.Length > 1 )
{
int space = ws.IndexOf( ' ' );
int tab = ws.IndexOf( '\t' );
if ( ws.StartsWith( "\n" ) && space >= 0 || tab >= 0 )
{
return; // do nothing if \n + indent
}
// otherwise kill all \n
ws = ws.Replace( "\n", "" );
tokens.Replace( after.TokenIndex, ws );
}
else
{
tokens.Delete( after.TokenIndex );
}
}
}