public void Execute(Repl repl, ReplParser.RupContext tree, bool piped) { var expr = tree.StringLiteral()?.GetText(); expr = expr?.Substring(1, expr.Length - 2); var doc = repl.stack.Peek(); var pr = ParsingResultsFactory.Create(doc); var aparser = pr.Parser; var atree = pr.ParseTree; List <IParseTree> nodes = null; if (expr != null) { using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); } } var results = LanguageServer.Transform.RemoveUselessParentheses(doc, nodes); repl.EnactEdits(results); }
public void Execute(Repl repl, ReplParser.UnulliteralContext tree, bool piped) { var type = tree.uclc()?.GetText(); var expr = tree.StringLiteral()?.GetText(); expr = expr?.Substring(1, expr.Length - 2); var doc = repl.stack.Peek(); var pr = ParsingResultsFactory.Create(doc); var aparser = pr.Parser; var atree = pr.ParseTree; if (expr != null) { using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree as TerminalNodeImpl).ToList(); var results = LanguageServer.Transform.UnUpperLowerCaseLiteral(nodes, type == "uc", doc); repl.EnactEdits(results); } } else { var results = LanguageServer.Transform.UnUpperLowerCaseLiteral(null, type == "uc", doc); repl.EnactEdits(results); } }
public void Execute(Repl repl, ReplParser.XgrepContext tree, bool piped) { var expr = repl.GetArg(tree.arg()); IParseTree[] atrees; Parser parser; Lexer lexer; string text; string fn; ITokenStream tokstream; if (piped) { var lines = repl.input_output_stack.Pop(); var serializeOptions = new JsonSerializerOptions(); serializeOptions.Converters.Add(new AntlrJson.ParseTreeConverter()); serializeOptions.WriteIndented = false; var parse_info = JsonSerializer.Deserialize <AntlrJson.ParsingResultSet>(lines, serializeOptions); text = parse_info.Text; fn = parse_info.FileName; atrees = parse_info.Nodes; parser = parse_info.Parser; lexer = parse_info.Lexer; tokstream = parse_info.Stream; } else { var doc = repl.stack.Peek(); var pr = ParsingResultsFactory.Create(doc); parser = pr.Parser; lexer = pr.Lexer; text = pr.Code; fn = pr.FullFileName; tokstream = pr.TokStream; IParseTree atree = pr.ParseTree; atrees = new IParseTree[] { atree }; } org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); IParseTree root = atrees.First().Root(); var ate = new AntlrTreeEditing.AntlrDOM.ConvertToDOM(); using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = ate.Try(root, parser)) { var l = atrees.Select(t => ate.FindDomNode(t)); var nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, l.ToArray()) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToArray(); var serializeOptions = new JsonSerializerOptions(); serializeOptions.Converters.Add(new AntlrJson.ParseTreeConverter()); serializeOptions.WriteIndented = false; var parse_info_out = new AntlrJson.ParsingResultSet() { Text = text, FileName = fn, Lexer = lexer, Parser = parser, Stream = tokstream, Nodes = nodes }; string js1 = JsonSerializer.Serialize(parse_info_out, serializeOptions); repl.input_output_stack.Push(js1); } }
public void Execute(Repl repl, ReplParser.ReorderContext tree, bool piped) { Dictionary <string, string> results = new Dictionary <string, string>(); var doc = repl.stack.Peek(); string expr = null; if (tree.modes() != null) { results = LanguageServer.Transform.SortModes(doc); } else { LspAntlr.ReorderType order = default; if (tree.alpha() != null) { order = LspAntlr.ReorderType.Alphabetically; } else if (tree.bfs() != null) { order = LspAntlr.ReorderType.BFS; expr = tree.bfs().StringLiteral().GetText(); } else if (tree.dfs() != null) { order = LspAntlr.ReorderType.DFS; expr = tree.dfs().StringLiteral().GetText(); } else { throw new Exception("unknown sorting type"); } List <IParseTree> nodes = null; if (expr != null) { expr = expr.Substring(1, expr.Length - 2); var pr = ParsingResultsFactory.Create(doc); var aparser = pr.Parser; var atree = pr.ParseTree; using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); } } results = LanguageServer.Transform.ReorderParserRules(doc, order, nodes); } repl.EnactEdits(results); }
public void Execute(Repl repl, ReplParser.UnfoldContext tree, bool piped) { var expr = repl.GetArg(tree.arg()); var doc = repl.stack.Peek(); var pr = ParsingResultsFactory.Create(doc); var aparser = pr.Parser; var atree = pr.ParseTree; using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree as TerminalNodeImpl).ToList(); var results = LanguageServer.Transform.Unfold(nodes, doc); repl.EnactEdits(results); } }
public void Execute(Repl repl, ReplParser.HasContext tree, bool piped) { var graph = tree.GRAPH() != null; var expr = repl.GetArg(tree.arg()); var doc = repl.stack.Peek(); var pr = ParsingResultsFactory.Create(doc); var aparser = pr.Parser; var atree = pr.ParseTree; using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser)) { List <IParseTree> nodes = null; if (expr != null) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); nodes = engine.parseExpression(expr, new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); } if (tree.DR() != null) { var result = LanguageServer.Transform.HasDirectRec(doc, nodes); foreach (var r in result) { System.Console.WriteLine(r); } } else if (tree.IR() != null) { var result = LanguageServer.Transform.HasIndirectRec(nodes, graph, doc); foreach (var r in result) { System.Console.WriteLine(r); } } else { throw new Exception("unknown check"); } } }
public void Try(string ffn, string input, ref Dictionary <string, string> results) { bool convert_undefined_to_terminals = true; string now = DateTime.Now.ToString(); StringBuilder errors = new StringBuilder(); StringBuilder sb = new StringBuilder(); AntlrInputStream str = new AntlrInputStream(input); BisonLexer lexer = new BisonLexer(str); CommonTokenStream tokens = new CommonTokenStream(lexer); BisonParser parser = new BisonParser(tokens); ErrorListenerBison <IToken> elistener = new ErrorListenerBison <IToken>(parser, lexer, tokens, errors); parser.AddErrorListener(elistener); BisonParser.InputContext tree = parser.input(); string error_file_name = ffn.Replace(".g4", ".txt"); if (elistener.had_error) { results.Add(ffn.Replace(".y", ".txt"), errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Get list of tokens. Convert this to a list of capitalized names. Dictionary <string, string> terminals = new Dictionary <string, string>(); Dictionary <string, string> nonterminals = new Dictionary <string, string>(); List <Tuple <string, List <List <string> > > > rules = new List <Tuple <string, List <List <string> > > >(); // Collect terminals. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//token_decls//token_decl/id[position() = 1]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); Antlr4.Runtime.Tree.IParseTree parent; foreach (var token in nodes) { string tok = token.GetText(); string cap_tok = tok.Length == 1 ? char.ToUpper(tok[0]).ToString() : (char.ToUpper(tok[0]) + tok.Substring(1)); terminals.Add(tok, cap_tok); } } // Collect rules. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rules", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement)); foreach (var rule in nodes) { var r = rule.AntlrIParseTree; var lhs = r.GetChild(0).GetText(); string cap_tok = lhs.Length == 1 ? char.ToLower(lhs[0]).ToString() : (char.ToLower(lhs[0]) + lhs.Substring(1)); nonterminals[lhs] = cap_tok; var rhs = new List <List <string> >(); var rhses = engine.parseExpression( @".//rhses_1/rhs", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { rule }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement)); foreach (var r1 in rhses) { rhs.Add(new List <string>()); var sym = engine.parseExpression( @".//symbol", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { r1 }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var s in sym) { List <string> l = rhs.Last(); l.Insert(0, s.GetText()); } } rules.Add(new Tuple <string, List <List <string> > >(lhs, rhs)); } } // First, collect information about the grammar. // BisonGrammarListener listener = new BisonGrammarListener(); // ParseTreeWalker.Default.Walk(listener, tree); if (convert_undefined_to_terminals) { foreach (Tuple <string, List <List <string> > > r in rules) { List <List <string> > rhs = r.Item2; foreach (List <string> s in rhs) { foreach (string c in s) { if (rules.Where(rr => rr.Item1 == c).Any()) { continue; } if (terminals.ContainsKey(c)) { continue; } if (c[0] == '\'') { continue; } if (c == "error") { continue; } // RHS symbol is not a non-terminal and not a %token terminal. // Enter it as a terminal. string tok = c; string cap_tok = tok.Length == 1 ? char.ToUpper(tok[0]).ToString() : (char.ToUpper(tok[0]) + tok.Substring(1)); terminals.Add(tok, cap_tok); // Note that in the error list. errors.AppendLine("Symbol " + c + " not declared, assuming it is a terminal."); } } } } // Convert any nonterminals which are keywords in Antlr... foreach (Tuple <string, List <List <string> > > r in rules) { if (!terminals.ContainsKey(r.Item1) && (r.Item1 == "options" || r.Item1 == "grammar" || r.Item1 == "tokenVocab" || r.Item1 == "lexer" || r.Item1 == "parser" || r.Item1 == "rule" )) { if (!nonterminals.ContainsKey(r.Item1)) { nonterminals.Add(r.Item1, r.Item1 + "_nonterminal"); } } else { if (!nonterminals.ContainsKey(r.Item1)) { nonterminals.Add(r.Item1, r.Item1); } } } // Get the name of the grammar. string name = System.IO.Path.GetFileNameWithoutExtension(ffn); sb.AppendLine("// Combined Antlr4 grammar generated by Antlrvsix."); sb.AppendLine("// Input grammar: " + ffn); sb.AppendLine("// Date: " + now); sb.AppendLine(); sb.AppendLine("grammar " + name + ";"); // Output the rules. foreach (Tuple <string, List <List <string> > > r in rules) { string lhs = r.Item1; lhs = nonterminals[lhs]; nonterminals.TryGetValue(lhs, out string nlhs); if (nlhs != null && nlhs != "") { lhs = nlhs; } sb.Append(lhs); List <List <string> > rhs = r.Item2; bool first = true; foreach (List <string> s in rhs) { sb.Append(first ? " :" : " |"); first = false; foreach (string c in s) { var re = c; if (terminals.ContainsKey(re)) { re = terminals[re]; } else if (nonterminals.ContainsKey(re)) { re = nonterminals[re]; } sb.Append(" " + re); } sb.AppendLine(); } sb.AppendLine(" ;"); } // Check for use of error symbol. The semantics is for the parser to look for // the ERROR token. bool found = false; foreach (Tuple <string, List <List <string> > > r in rules) { List <List <string> > rhs = r.Item2; foreach (List <string> s in rhs) { foreach (string c in s) { if (c == "error") { found = true; goto bigexit; } } } } bigexit: if (found) { sb.AppendLine("error : ERROR ;"); } results.Add(ffn.Replace(".y", ".g4"), sb.ToString()); results.Add(ffn.Replace(".y", ".txt"), errors.ToString()); }
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv2Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv2Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammar_(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g2") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g2") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove "header". using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//header_", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Remove classDef action blocks for now. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//classDef/actionBlock", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Let's take care of options first. That's because we can't // determine if this is a combined grammar or not. // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var options = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeOptionsSpec)", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); var nodes = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeParserOptionsSpec) /(option | lexerOption) [id/* [ text() = 'output' or text() = 'backtrack' or text() = 'buildAST' or text() = 'classHeaderSuffix' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); foreach (var opt in options) { if (opt.ChildCount == 3) { TreeEdits.Delete(opt); } } } // Delete rule options. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rule_/ruleOptionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/ruleOptionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Parser and Lexer in One Definition using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var parser_nodes = engine.parseExpression( @"//parserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var lexer_nodes = engine.parseExpression( @"//lexerSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var tree_nodes = engine.parseExpression( @"//treeParserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); // Can this be a "combined" grammar? That can happen only if // one parser and one lexer decl and no tree decl. Options for // the lexer must be removed too. if (parser_nodes.Count() == 1 && lexer_nodes.Count() == 1 && tree_nodes.Count() == 0) { var lexerSpec = lexer_nodes.First() as ANTLRv2Parser.LexerSpecContext; if (lexerSpec.lexerOptionsSpec() == null) { // Nuke lexer class decl because it's a combined grammar. TreeEdits.Delete(lexerSpec); lexerSpec = null; } // Rewrite the parser spec. var parserSpec = parser_nodes.First() as ANTLRv2Parser.ParserSpecContext; var c = parserSpec.CLASS(); var i = parserSpec.id(); var e = parserSpec.EXTENDS(); var p = parserSpec.PARSER(); var s = parserSpec.superClass(); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.GRAMMAR) { Line = -1, Column = -1, Text = "grammar" }); text_before.TryGetValue(c as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(c, (in IParseTree n, out bool cc) => { cc = false; return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv3Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv3Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammarDef(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g3") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g3") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//grammarDef/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//grammarDef/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Fix options in the beginning of rules. // See https://theantlrguy.atlassian.net/wiki/spaces/ANTLR3/pages/2687029/Rule+and+subrule+options using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//rule_/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Use new tokens{} syntax using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//tokensSpec /tokenSpec /SEMI", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); if (nodes.Any()) { // Delete tha last ";" in tokens list--change in syntax. var last = nodes.Last(); TreeEdits.Delete(last); // Replace all remaining ";" with ",". TreeEdits.Replace(tree, (in IParseTree n, out bool c) => { c = true; if (!nodes.Contains(n) && n != last) { return(null); } var t = n as TerminalNodeImpl; var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.COMMA) { Line = -1, Column = -1, Text = "," }); text_before.TryGetValue(t, out string v); if (v != null) { text_before.Add(new_sym, v); } return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new W3CebnfLexer(str); var tokens = new CommonTokenStream(lexer); var parser = new W3CebnfParser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.prods(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(suffix) ? (error_file_name.Substring(0, error_file_name.Length - suffix.Length) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(suffix) ? (new_ffn.Substring(0, new_ffn.Length - suffix.Length) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } var name = System.IO.Path.GetFileName(ffn); name = System.IO.Path.GetFileNameWithoutExtension(ffn); var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); TreeEdits.InsertBefore(tree.GetChild(0), "grammar " + name + ";" + Environment.NewLine); using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rhs", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.InsertAfter(n, " ;"); } } using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//PPEQ", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Replace(n, ":"); } } // Convert double-quoted string literals to single quote. { using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//STRING", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { var text = n.GetText(); if (text.Length == 0) { continue; } if (text[0] != '"') { continue; } text = text.Substring(1, text.Length - 2); StringBuilder ss = new StringBuilder(); ss.Append("'"); foreach (var c in text) { if (c == '"') { ss.Append("\\"); } else { ss.Append(c); } } ss.Append("'"); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.STRING_LITERAL) { Line = -1, Column = -1, Text = ss.ToString() }); text_before.TryGetValue(n as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(n, new_sym); } } } StringBuilder sb = new StringBuilder(); TreeEdits.Reconstruct(sb, tree, text_before); var new_code = sb.ToString(); results.Add(new_ffn, new_code); results.Add(ffn.Replace(suffix, ".txt"), errors.ToString()); }