public virtual void ParseAndRewrite() { ProcessArgs(args); ICharStream input = null; if (filename != null) { input = new ANTLRStringStream(File.ReadAllText(filename), filename); } else { input = new ANTLRReaderStream(Console.In, ANTLRReaderStream.InitialBufferSize, ANTLRReaderStream.ReadBufferSize); } // BUILD AST ANTLRLexer lex = new ANTLRLexer(input); tokens = new TokenRewriteStream(lex); ANTLRParser g = new ANTLRParser(tokens); Grammar grammar = new Grammar(); var r = g.grammar_(grammar); CommonTree t = (CommonTree)r.Tree; if (tree_option) { Console.Out.WriteLine(t.ToStringTree()); } Rewrite(g.TreeAdaptor, t, g.TokenNames); }
public void TestReplaceRangeThenInsertAtRightEdge() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "x"); tokens.InsertBefore(4, "y"); // no effect; within range of a replace Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestInsertInPriorReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 2, "x"); tokens.InsertBefore(1, "0"); Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public static IList <ActionChunk> TranslateActionChunk(OutputModelFactory factory, RuleFunction rf, string action, ActionAST node) { IToken tokenWithinAction = node.Token; ActionTranslator translator = new ActionTranslator(factory, node); translator.rf = rf; factory.GetGrammar().tool.Log("action-translator", "translate " + action); string altLabel = node.GetAltLabel(); if (rf != null) { translator.nodeContext = rf.ruleCtx; if (altLabel != null) { AltLabelStructDecl decl; rf.altLabelCtxs.TryGetValue(altLabel, out decl); translator.nodeContext = decl; } } ANTLRStringStream @in = new ANTLRStringStream(action); @in.Line = tokenWithinAction.Line; @in.CharPositionInLine = tokenWithinAction.CharPositionInLine; ActionSplitter trigger = new ActionSplitter(@in, translator); // forces eval, triggers listener methods trigger.GetActionTokens(); return(translator.chunks); }
public void TestReplaceThenReplaceLowerIndexedSuperset() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "xyz"); tokens.Replace(1, 3, "foo"); // overlap, error Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@3,3:3='c',<6>,1:3]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
public void TestOverlappingReplace2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 3, "bar"); tokens.Replace(1, 2, "foo"); // cannot split earlier replace Exception exc = null; try { tokens.ToString(); } catch (ArgumentException iae) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">"; Assert.IsNotNull(exc); Assert.AreEqual(expecting, exc.Message); }
protected void checkPrediction(DFA dfa, string input, int expected) //throws Exception { ANTLRStringStream stream = new ANTLRStringStream(input); assertEquals(dfa.Predict(stream), expected); }
public virtual void ProcessNested(IToken actionToken) { ANTLRStringStream @in = new ANTLRStringStream(actionToken.Text); @in.Line = actionToken.Line; @in.CharPositionInLine = actionToken.CharPositionInLine; ActionSplitter splitter = new ActionSplitter(@in, this); // forces eval, triggers listener methods splitter.GetActionTokens(); }
public static List <BekProgram> DefsFromString(string programs) { var res = new List <BekProgram>(); try { var input = new Antlr.Runtime.ANTLRStringStream(programs); var lexer = new bekLexer(input); var tokens = new Antlr.Runtime.CommonTokenStream(lexer); var parser = new bekParser(tokens); var resp = parser.BekPgms(); foreach (BekPgm cur in resp) { var stab = new Symtab(cur); //TypeChecker.TypeCheck(cur, stab); var p = new BekProgram(cur, stab); //Library.PerformExpansions(p); res.Add(p); } } catch (Antlr.Runtime.MismatchedTokenException e) { string tok = (e.Token != null ? "'" + e.Token.Text + "'" : (e.Character >= 0 ? Microsoft.Automata.StringUtility.Escape((char)e.Character) : "")); string msg = "unexpected token " + tok; if (tok != "" && 0 <= e.Expecting && e.Expecting < ParserImpl.bekParser.tokenNames.Length) { msg += string.Format(" expecting {0}", ParserImpl.bekParser.tokenNames[e.Expecting]); } throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.FailedPredicateException e) { string msg = string.Format("unexpected '{0}' failed {1}", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString()), e.PredicateText); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.NoViableAltException e) { string msg = string.Format("unexpected '{0}' no alternatives", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.RecognitionException e) { string msg = string.Format("unexpected '{0}'", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (BekParseException e) { throw e; } catch (Exception e) { throw new BekParseException(1, 1, e.Message); } return(res); }
public virtual void ExamineAction() { //System.out.println("examine "+actionToken); ANTLRStringStream @in = new ANTLRStringStream(actionToken.Text); @in.Line = actionToken.Line; @in.CharPositionInLine = actionToken.CharPositionInLine; ActionSplitter splitter = new ActionSplitter(@in, this); // forces eval, triggers listener methods node.chunks = splitter.GetActionTokens(); }
public static bool ActionIsContextDependent(ActionAST actionAST) { ANTLRStringStream @in = new ANTLRStringStream(actionAST.Token.Text); @in.Line = actionAST.Token.Line; @in.CharPositionInLine = actionAST.Token.CharPositionInLine; var listener = new ContextDependentListener(); ActionSplitter splitter = new ActionSplitter(@in, listener); // forces eval, triggers listener methods splitter.GetActionTokens(); return listener.dependent; }
public virtual void ProcessNested(IToken actionToken) { ANTLRStringStream @in = new ANTLRStringStream(actionToken.Text); @in.Line = actionToken.Line; @in.CharPositionInLine = actionToken.CharPositionInLine; ActionSplitter splitter = new ActionSplitter(@in, this); // forces eval, triggers listener methods splitter.GetActionTokens(); }
public void TestToStringStartStop2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n"); // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0 + 2 * 0;"; Assert.AreEqual(expecting, result); tokens.Replace(4, 8, "0"); // replace 3 * 0 with 0 result = tokens.ToString(); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 17); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual(expecting, result); result = tokens.ToString(4, 8); expecting = "0"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 8); expecting = "x = 0"; Assert.AreEqual(expecting, result); result = tokens.ToString(12, 16); expecting = "2 * 0"; Assert.AreEqual(expecting, result); tokens.InsertAfter(17, "// comment"); result = tokens.ToString(12, 18); expecting = "2 * 0;// comment"; Assert.AreEqual(expecting, result); result = tokens.ToString(0, 8); // try again after insert at end expecting = "x = 0"; Assert.AreEqual(expecting, result); }
public static List<BekProgram> DefsFromString(string programs) { var res = new List<BekProgram>(); try { var input = new Antlr.Runtime.ANTLRStringStream(programs); var lexer = new bekLexer(input); var tokens = new Antlr.Runtime.CommonTokenStream(lexer); var parser = new bekParser(tokens); var resp = parser.BekPgms(); foreach (BekPgm cur in resp) { var stab = new Symtab(cur); //TypeChecker.TypeCheck(cur, stab); var p = new BekProgram(cur, stab); //Library.PerformExpansions(p); res.Add(p); } } catch (Antlr.Runtime.MismatchedTokenException e) { string tok = (e.Token != null ? "'" + e.Token.Text + "'" : (e.Character >= 0 ? Microsoft.Automata.StringUtility.Escape((char)e.Character) : "")); string msg = "unexpected token " + tok; if (tok != "" && 0 <= e.Expecting && e.Expecting < ParserImpl.bekParser.tokenNames.Length) msg += string.Format(" expecting {0}", ParserImpl.bekParser.tokenNames[e.Expecting]); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.FailedPredicateException e) { string msg = string.Format("unexpected '{0}' failed {1}", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString()), e.PredicateText); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.NoViableAltException e) { string msg = string.Format("unexpected '{0}' no alternatives", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.RecognitionException e) { string msg = string.Format("unexpected '{0}'", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (BekParseException e) { throw e; } catch (Exception e) { throw new BekParseException(1, 1, e.Message); } return res; }
public virtual void ExamineAction() { //System.out.println("examine "+actionToken); ANTLRStringStream @in = new ANTLRStringStream(actionToken.Text); @in.Line = actionToken.Line; @in.CharPositionInLine = actionToken.CharPositionInLine; ActionSplitter splitter = new ActionSplitter(@in, this); // forces eval, triggers listener methods node.chunks = splitter.GetActionTokens(); }
public static bool ActionIsContextDependent(ActionAST actionAST) { ANTLRStringStream @in = new ANTLRStringStream(actionAST.Token.Text); @in.Line = actionAST.Token.Line; @in.CharPositionInLine = actionAST.Token.CharPositionInLine; var listener = new ContextDependentListener(); ActionSplitter splitter = new ActionSplitter(@in, listener); // forces eval, triggers listener methods splitter.GetActionTokens(); return(listener.dependent); }
public void TestInsertBeforeIndex0() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.InsertBefore(0, "0"); string result = tokens.ToString(); string expecting = "0abc"; assertEquals(expecting, result); }
/// <summary> /// Parse a sequence of queries. /// Eeach query in the sequence must end with a ';'. /// Output also the individual query strings. /// </summary> public static List <Expression> ParseQueries(string query_sequence, out List <string> query_strings) { try { var input = new Antlr.Runtime.ANTLRStringStream(query_sequence); var lexer = new queryLexer(input); var tokens = new Antlr.Runtime.CommonTokenStream(lexer); var parser = new queryParser(tokens); var res = parser.Queries(); query_strings = res.Item2; return(res.Item1); } catch (Antlr.Runtime.MismatchedTokenException e) { string tok = (e.Token != null ? "'" + e.Token.Text + "'" : (e.Character >= 0 ? StringUtility.Escape((char)e.Character) : "")); string msg = "unexpected token " + tok; if (tok != "" && 0 <= e.Expecting && e.Expecting < queryParser.tokenNames.Length) { msg += string.Format(" expecting {0}", queryParser.tokenNames[e.Expecting]); } throw new QueryParseException(e.Line, e.CharPositionInLine + 1, msg); } catch (Antlr.Runtime.FailedPredicateException e) { string msg = string.Format("unexpected '{0}' failed {1}", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString()), e.PredicateText); throw new QueryParseException(e.Line, e.CharPositionInLine + 1, msg); } catch (Antlr.Runtime.NoViableAltException e) { string msg = string.Format("unexpected '{0}' no alternatives", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new QueryParseException(e.Line, e.CharPositionInLine + 1, msg); } catch (Antlr.Runtime.RecognitionException e) { string msg = string.Format("unexpected '{0}'", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new QueryParseException(e.Line, e.CharPositionInLine + 1, msg); } catch (QueryParseException e) { throw e; } catch (Exception e) { throw new QueryParseException(1, 1, e.Message); } }
public void TestInsertAfterLastIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.InsertAfter(2, "x"); string result = tokens.ToString(); string expecting = "abcx"; Assert.AreEqual(expecting, result); }
public void Test2InsertMiddleIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 1, "x" ); tokens.InsertBefore( 1, "y" ); string result = tokens.ToString(); string expecting = "ayxbc"; Assert.AreEqual( expecting, result ); }
public void TestReplaceSubsetThenFetch() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "xyz"); string result = tokens.ToString(0, 6); string expecting = "abxyzba"; Assert.AreEqual(expecting, result); }
public void TestReplaceAll() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 6, "x"); string result = tokens.ToString(); string expecting = "x"; assertEquals(expecting, result); }
internal static expr ParseExprFromString(string str) { try { var input = new Antlr.Runtime.ANTLRStringStream(str); var lexer = new bekLexer(input); var tokens = new Antlr.Runtime.CommonTokenStream(lexer); var parser = new bekParser(tokens); return(parser.Comp_expr()); } catch (Antlr.Runtime.MismatchedTokenException e) { string tok = (e.Token != null ? "'" + e.Token.Text + "'" : (e.Character >= 0 ? Microsoft.Automata.StringUtility.Escape((char)e.Character) : "")); string msg = "unexpected token " + tok; if (tok != "" && 0 <= e.Expecting && e.Expecting < ParserImpl.bekParser.tokenNames.Length) { msg += string.Format(" expecting {0}", ParserImpl.bekParser.tokenNames[e.Expecting]); } throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.FailedPredicateException e) { string msg = string.Format("unexpected '{0}' failed {1}", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString()), e.PredicateText); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.NoViableAltException e) { string msg = string.Format("unexpected '{0}' no alternatives", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.RecognitionException e) { string msg = string.Format("unexpected '{0}'", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (BekParseException e) { throw e; } catch (Exception e) { throw new BekParseException(1, 1, e.Message); } }
public void TestInsertBeforeTokenThenDeleteThatToken() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(2, "y"); tokens.Delete(2); string result = tokens.ToString(); string expecting = "aby"; Assert.AreEqual(expecting, result); }
public void TestInsertThenReplaceSameIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(0, "0"); tokens.Replace(0, "x"); // supercedes insert at 0 string result = tokens.ToString(); string expecting = "0xbc"; Assert.AreEqual(expecting, result); }
public void TestReplaceThenDeleteMiddleIndex() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, "x"); tokens.Delete(1); string result = tokens.ToString(); string expecting = "ac"; Assert.AreEqual(expecting, result); }
public void TestOverlappingReplace4() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, 2, "foo"); tokens.Replace(1, 3, "bar"); // wipes prior nested replace string result = tokens.ToString(); string expecting = "abar"; Assert.AreEqual(expecting, result); }
public void TestLeaveAloneDisjointInsert2() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 3, "foo"); tokens.InsertBefore(1, "x"); string result = tokens.ToString(); string expecting = "axbfoo"; Assert.AreEqual(expecting, result); }
public void TestDropPrevCoveredInsert() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(1, "foo"); tokens.Replace(1, 2, "foo"); // kill prev insert string result = tokens.ToString(); string expecting = "afoofoo"; Assert.AreEqual(expecting, result); }
public void TestDropIdenticalReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(1, 2, "foo"); tokens.Replace(1, 2, "foo"); // drop previous, identical string result = tokens.ToString(); string expecting = "afooc"; Assert.AreEqual(expecting, result); }
public void TestCombineInsertOnLeftWithDelete() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Delete(0, 2); tokens.InsertBefore(0, "z"); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "z"; // make sure combo is not znull Assert.AreEqual(expecting, result); }
public void TestReplaceSingleMiddleThenOverlappingSuperset() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 2, "xyz"); tokens.Replace(0, 3, "foo"); string result = tokens.ToString(); string expecting = "fooa"; Assert.AreEqual(expecting, result); }
public void TestReplaceRangeThenInsertAtLeftEdge() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abcccba"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(2, 4, "x"); tokens.InsertBefore(2, "y"); string result = tokens.ToString(); string expecting = "abyxba"; Assert.AreEqual(expecting, result); }
public void TestCombineInsertOnLeftWithReplace() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.Replace(0, 2, "foo"); tokens.InsertBefore(0, "z"); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "zfoo"; assertEquals(expecting, result); }
public void TestCombine3Inserts() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(1, "x"); tokens.InsertBefore(0, "y"); tokens.InsertBefore(1, "z"); string result = tokens.ToString(); string expecting = "yazxbc"; Assert.AreEqual(expecting, result); }
public void TestInsertInPriorReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 2, "x" ); tokens.InsertBefore( 1, "0" ); Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@1,1:1='b',<5>,1:1]:\"0\"> within boundaries of previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@2,2:2='c',<6>,1:2]:\"x\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestInsertThenReplaceSameIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 0, "0" ); tokens.Replace( 0, "x" ); // supercedes insert at 0 string result = tokens.ToString(); string expecting = "0xbc"; Assert.AreEqual( expecting, result ); }
public void TestReplaceRangeThenInsertAtLeftEdge() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "x" ); tokens.InsertBefore( 2, "y" ); string result = tokens.ToString(); string expecting = "abyxba"; assertEquals( expecting, result ); }
public void TestReplaceRangeThenInsertAtRightEdge() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "x" ); tokens.InsertBefore( 4, "y" ); // no effect; within range of a replace Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "insert op <InsertBeforeOp@[@4,4:4='c',<6>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"x\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestInsertBeforeTokenThenDeleteThatToken() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n"); ICharStream input = new ANTLRStringStream("abc"); Interpreter lexEngine = new Interpreter(g, input); TokenRewriteStream tokens = new TokenRewriteStream(lexEngine); tokens.Fill(); tokens.InsertBefore(2, "y"); tokens.Delete(2); string result = tokens.ToString(); string expecting = "aby"; Assert.AreEqual(expecting, result); }
public void TestLeaveAloneDisjointInsert2() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 3, "foo" ); tokens.InsertBefore( 1, "x" ); string result = tokens.ToString(); string expecting = "axbfoo"; Assert.AreEqual( expecting, result ); }
public void TestDropIdenticalReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, 2, "foo" ); tokens.Replace( 1, 2, "foo" ); // drop previous, identical string result = tokens.ToString(); string expecting = "afooc"; Assert.AreEqual( expecting, result ); }
public void TestOverlappingReplace2() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 3, "bar" ); tokens.Replace( 1, 2, "foo" ); // cannot split earlier replace Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@1,1:1='b',<5>,1:1]..[@2,2:2='c',<6>,1:2]:\"foo\"> overlap with previous <ReplaceOp@[@0,0:0='a',<4>,1:0]..[@3,3:3='c',<6>,1:3]:\"bar\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void TestOverlappingReplace4() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, 2, "foo" ); tokens.Replace( 1, 3, "bar" ); // wipes prior nested replace string result = tokens.ToString(); string expecting = "abar"; Assert.AreEqual( expecting, result ); }
public static IList<ActionChunk> TranslateActionChunk(OutputModelFactory factory, RuleFunction rf, string action, ActionAST node) { IToken tokenWithinAction = node.Token; ActionTranslator translator = new ActionTranslator(factory, node); translator.rf = rf; factory.GetGrammar().tool.Log("action-translator", "translate " + action); string altLabel = node.GetAltLabel(); if (rf != null) { translator.nodeContext = rf.ruleCtx; if (altLabel != null) { AltLabelStructDecl decl; rf.altLabelCtxs.TryGetValue(altLabel, out decl); translator.nodeContext = decl; } } ANTLRStringStream @in = new ANTLRStringStream(action); @in.Line = tokenWithinAction.Line; @in.CharPositionInLine = tokenWithinAction.CharPositionInLine; ActionSplitter trigger = new ActionSplitter(@in, translator); // forces eval, triggers listener methods trigger.GetActionTokens(); return translator.chunks; }
/** For testing; builds trees, does sem anal */ public Grammar(string fileName, string grammarText, Grammar tokenVocabSource, [Nullable] ANTLRToolListener listener) { this.text = grammarText; this.fileName = fileName; this.tool = new AntlrTool(); this.tool.AddListener(listener); Antlr.Runtime.ANTLRStringStream @in = new Antlr.Runtime.ANTLRStringStream(grammarText); @in.name = fileName; this.ast = tool.Parse(fileName, @in); if (ast == null) { throw new NotSupportedException(); } if (ast.tokenStream == null) { throw new InvalidOperationException("expected ast to have a token stream"); } this.tokenStream = ast.tokenStream; this.originalTokenStream = this.tokenStream; // ensure each node has pointer to surrounding grammar Antlr.Runtime.Tree.TreeVisitor v = new Antlr.Runtime.Tree.TreeVisitor(new GrammarASTAdaptor()); v.Visit(ast, new SetPointersAction(this)); InitTokenSymbolTables(); if (tokenVocabSource != null) { ImportVocab(tokenVocabSource); } tool.Process(this, false); }
public void TestReplaceSingleMiddleThenOverlappingSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 2, "xyz" ); tokens.Replace( 0, 3, "foo" ); string result = tokens.ToString(); string expecting = "fooa"; Assert.AreEqual( expecting, result ); }
public void TestToStringStartStop() { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "MUL : '*';\n" + "ASSIGN : '=';\n" + "WS : ' '+;\n" ); // Tokens: 0123456789 // Input: x = 3 * 0; ICharStream input = new ANTLRStringStream( "x = 3 * 0;" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0 string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString(); expecting = "x = 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 9 ); expecting = "x = 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 4, 8 ); expecting = "0"; Assert.AreEqual( expecting, result ); }
public void TestToStringStartStop2() { Grammar g = new Grammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n" ); // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; ICharStream input = new ANTLRStringStream( "x = 3 * 0 + 2 * 0;" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); string result = tokens.ToOriginalString(); string expecting = "x = 3 * 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); tokens.Replace( 4, 8, "0" ); // replace 3 * 0 with 0 result = tokens.ToString(); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 17 ); expecting = "x = 0 + 2 * 0;"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 4, 8 ); expecting = "0"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 8 ); expecting = "x = 0"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 12, 16 ); expecting = "2 * 0"; Assert.AreEqual( expecting, result ); tokens.InsertAfter( 17, "// comment" ); result = tokens.ToString( 12, 18 ); expecting = "2 * 0;// comment"; Assert.AreEqual( expecting, result ); result = tokens.ToString( 0, 8 ); // try again after insert at end expecting = "x = 0"; Assert.AreEqual( expecting, result ); }
public virtual GrammarRootAST ParseGrammar(string fileName) { try { string file = fileName; if (!Path.IsPathRooted(file)) { file = Path.Combine(inputDirectory, fileName); } string fileContent = File.ReadAllText(file, Encoding.GetEncoding(grammarEncoding)); char[] fileChars = fileContent.ToCharArray(); ANTLRStringStream @in = new ANTLRStringStream(fileChars, fileChars.Length, fileName); GrammarRootAST t = Parse(fileName, @in); return t; } catch (IOException ioe) { errMgr.ToolError(ErrorType.CANNOT_OPEN_FILE, ioe, fileName); } return null; }
public void TestDropPrevCoveredInsert() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 1, "foo" ); tokens.Replace( 1, 2, "foo" ); // kill prev insert string result = tokens.ToString(); string expecting = "afoofoo"; Assert.AreEqual( expecting, result ); }
public void TestReplaceSubsetThenFetch() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); string result = tokens.ToString( 0, 6 ); string expecting = "abxyzba"; Assert.AreEqual( expecting, result ); }
public void TestReplaceThenDeleteMiddleIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 1, "x" ); tokens.Delete( 1 ); string result = tokens.ToString(); string expecting = "ac"; assertEquals( expecting, result ); }
public void TestReplaceThenReplaceSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); tokens.Replace( 3, 5, "foo" ); // overlaps, error Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <ReplaceOp@[@3,3:3='c',<6>,1:3]..[@5,5:5='b',<5>,1:5]:\"foo\"> overlap with previous <ReplaceOp@[@2,2:2='c',<6>,1:2]..[@4,4:4='c',<6>,1:4]:\"xyz\">"; Assert.IsNotNull( exc ); Assert.AreEqual( expecting, exc.Message ); }
public void Test2InsertThenReplaceIndex0() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.InsertBefore( 0, "x" ); tokens.InsertBefore( 0, "y" ); tokens.Replace( 0, "z" ); string result = tokens.ToString(); string expecting = "zbc"; assertEquals( expecting, result ); }
public void TestReplaceThenInsertAfterLastIndex() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, "x" ); tokens.InsertAfter( 2, "y" ); string result = tokens.ToString(); string expecting = "abxy"; Assert.AreEqual( expecting, result ); }
internal static expr ParseExprFromString(string str) { try { var input = new Antlr.Runtime.ANTLRStringStream(str); var lexer = new bekLexer(input); var tokens = new Antlr.Runtime.CommonTokenStream(lexer); var parser = new bekParser(tokens); return parser.Comp_expr(); } catch (Antlr.Runtime.MismatchedTokenException e) { string tok = (e.Token != null ? "'" + e.Token.Text + "'" : (e.Character >= 0 ? Microsoft.Automata.StringUtility.Escape((char)e.Character) : "")); string msg = "unexpected token " + tok; if (tok != "" && 0 <= e.Expecting && e.Expecting < ParserImpl.bekParser.tokenNames.Length) msg += string.Format(" expecting {0}", ParserImpl.bekParser.tokenNames[e.Expecting]); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.FailedPredicateException e) { string msg = string.Format("unexpected '{0}' failed {1}", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString()), e.PredicateText); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.NoViableAltException e) { string msg = string.Format("unexpected '{0}' no alternatives", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (Antlr.Runtime.RecognitionException e) { string msg = string.Format("unexpected '{0}'", (e.Token != null ? e.Token.Text : ((char)e.Character).ToString())); throw new BekParseException(e.Line, e.CharPositionInLine, msg); } catch (BekParseException e) { throw e; } catch (Exception e) { throw new BekParseException(1, 1, e.Message); } }
public void TestReplaceThenReplaceSuperset() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abcccba" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 2, 4, "xyz" ); tokens.Replace( 3, 5, "foo" ); // overlaps, error Exception exc = null; try { tokens.ToString(); } catch ( ArgumentException iae ) { exc = iae; } string expecting = "replace op boundaries of <[email protected]:\"foo\"> overlap with previous <[email protected]:\"xyz\">"; assertNotNull( exc ); assertEquals( expecting, exc.Message ); }
/** * Try current dir then dir of g then lib dir * @param g * @param nameNode The node associated with the imported grammar name. */ public virtual Grammar LoadImportedGrammar(Grammar g, GrammarAST nameNode) { string name = nameNode.Text; Grammar imported; if (!importedGrammars.TryGetValue(name, out imported) || imported == null) { g.tool.Log("grammar", "load " + name + " from " + g.fileName); string importedFile = null; foreach (string extension in ALL_GRAMMAR_EXTENSIONS) { importedFile = GetImportedGrammarFile(g, name + extension); if (importedFile != null) { break; } } if (importedFile == null) { errMgr.GrammarError(ErrorType.CANNOT_FIND_IMPORTED_GRAMMAR, g.fileName, nameNode.Token, name); return null; } string absolutePath = Path.GetFullPath(importedFile); string fileContent = File.ReadAllText(absolutePath, Encoding.GetEncoding(grammarEncoding)); char[] fileChars = fileContent.ToCharArray(); ANTLRStringStream @in = new ANTLRStringStream(fileChars, fileChars.Length, importedFile); GrammarRootAST root = Parse(g.fileName, @in); if (root == null) { return null; } imported = CreateGrammar(root); imported.fileName = absolutePath; importedGrammars[root.GetGrammarName()] = imported; } return imported; }
public void TestCombineInsertOnLeftWithReplace() { Grammar g = new Grammar( "lexer grammar t;\n" + "A : 'a';\n" + "B : 'b';\n" + "C : 'c';\n" ); ICharStream input = new ANTLRStringStream( "abc" ); Interpreter lexEngine = new Interpreter( g, input ); TokenRewriteStream tokens = new TokenRewriteStream( lexEngine ); tokens.Fill(); tokens.Replace( 0, 2, "foo" ); tokens.InsertBefore( 0, "z" ); // combine with left edge of rewrite string result = tokens.ToString(); string expecting = "zfoo"; Assert.AreEqual( expecting, result ); }