public override void Parse(string code, out CommonTokenStream TokStream, out Parser Parser, out Lexer Lexer, out IParseTree ParseTree) { IParseTree pt = null; // Set up Antlr to parse input grammar. byte[] byteArray = Encoding.UTF8.GetBytes(code); AntlrInputStream ais = new AntlrInputStream( new StreamReader( new MemoryStream(byteArray)).ReadToEnd()); var lexer = new W3CebnfLexer(ais); CommonTokenStream cts = new CommonTokenStream(lexer); var parser = new W3CebnfParser(cts); lexer.RemoveErrorListeners(); var lexer_error_listener = new ErrorListener <int>(parser, lexer, this.QuietAfter); lexer.AddErrorListener(lexer_error_listener); parser.RemoveErrorListeners(); var parser_error_listener = new ErrorListener <IToken>(parser, lexer, this.QuietAfter); parser.AddErrorListener(parser_error_listener); try { pt = parser.prods(); } catch (Exception) { // Parsing error. } TokStream = cts; Parser = parser; Lexer = lexer; ParseTree = pt; }
public override void Parse(ParsingResults pd, bool bail) { string ffn = pd.FullFileName; string code = pd.Code; if (ffn == null) { return; } if (code == null) { return; } this.QuietAfter = pd.QuietAfter; IParseTree pt = null; // Set up Antlr to parse input grammar. byte[] byteArray = Encoding.UTF8.GetBytes(code); AntlrInputStream ais = new AntlrInputStream( new StreamReader( new MemoryStream(byteArray)).ReadToEnd()) { name = ffn }; var lexer = new W3CebnfLexer(ais); CommonTokenStream cts = new CommonTokenStream(lexer); var parser = new W3CebnfParser(cts); lexer.RemoveErrorListeners(); var lexer_error_listener = new ErrorListener <int>(parser, lexer, pd.QuietAfter); lexer.AddErrorListener(lexer_error_listener); parser.RemoveErrorListeners(); var parser_error_listener = new ErrorListener <IToken>(parser, lexer, pd.QuietAfter); parser.AddErrorListener(parser_error_listener); BailErrorHandler bail_error_handler = null; if (bail) { bail_error_handler = new BailErrorHandler(); parser.ErrorHandler = bail_error_handler; } try { pt = parser.prods(); } catch (Exception) { // Parsing error. } //StringBuilder sb = new StringBuilder(); //TreeSerializer.ParenthesizedAST(pt, sb, "", cts); //string fn = System.IO.Path.GetFileName(ffn); //fn = "c:\\temp\\" + fn; //System.IO.File.WriteAllText(fn, sb.ToString()); if (parser_error_listener.had_error || lexer_error_listener.had_error || (bail_error_handler != null && bail_error_handler.had_error)) { System.Console.Error.WriteLine("Error in parse of " + ffn); } else { System.Console.Error.WriteLine("Parse completed of " + ffn); } pd.TokStream = cts; pd.Parser = parser; pd.Lexer = lexer; pd.ParseTree = pt; Stack <IParseTree> stack = new Stack <IParseTree>(); stack.Push(pt); while (stack.Any()) { var x = stack.Pop(); if (x is TerminalNodeImpl leaf) { } else { var y = x as AttributedParseTreeNode; if (y != null) { y.ParserDetails = pd; } for (int i = 0; i < x.ChildCount; ++i) { var c = x.GetChild(i); if (c != null) { stack.Push(c); } } } } }
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv3Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv3Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammarDef(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g3") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g3") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//grammarDef/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//grammarDef/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Fix options in the beginning of rules. // See https://theantlrguy.atlassian.net/wiki/spaces/ANTLR3/pages/2687029/Rule+and+subrule+options using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//rule_/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Use new tokens{} syntax using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//tokensSpec /tokenSpec /SEMI", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); if (nodes.Any()) { // Delete tha last ";" in tokens list--change in syntax. var last = nodes.Last(); TreeEdits.Delete(last); // Replace all remaining ";" with ",". TreeEdits.Replace(tree, (in IParseTree n, out bool c) => { c = true; if (!nodes.Contains(n) && n != last) { return(null); } var t = n as TerminalNodeImpl; var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.COMMA) { Line = -1, Column = -1, Text = "," }); text_before.TryGetValue(t, out string v); if (v != null) { text_before.Add(new_sym, v); } return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv2Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv2Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammar_(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g2") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g2") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove "header". using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//header_", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Remove classDef action blocks for now. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//classDef/actionBlock", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Let's take care of options first. That's because we can't // determine if this is a combined grammar or not. // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var options = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeOptionsSpec)", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); var nodes = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeParserOptionsSpec) /(option | lexerOption) [id/* [ text() = 'output' or text() = 'backtrack' or text() = 'buildAST' or text() = 'classHeaderSuffix' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); foreach (var opt in options) { if (opt.ChildCount == 3) { TreeEdits.Delete(opt); } } } // Delete rule options. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rule_/ruleOptionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/ruleOptionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Parser and Lexer in One Definition using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var parser_nodes = engine.parseExpression( @"//parserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var lexer_nodes = engine.parseExpression( @"//lexerSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var tree_nodes = engine.parseExpression( @"//treeParserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); // Can this be a "combined" grammar? That can happen only if // one parser and one lexer decl and no tree decl. Options for // the lexer must be removed too. if (parser_nodes.Count() == 1 && lexer_nodes.Count() == 1 && tree_nodes.Count() == 0) { var lexerSpec = lexer_nodes.First() as ANTLRv2Parser.LexerSpecContext; if (lexerSpec.lexerOptionsSpec() == null) { // Nuke lexer class decl because it's a combined grammar. TreeEdits.Delete(lexerSpec); lexerSpec = null; } // Rewrite the parser spec. var parserSpec = parser_nodes.First() as ANTLRv2Parser.ParserSpecContext; var c = parserSpec.CLASS(); var i = parserSpec.id(); var e = parserSpec.EXTENDS(); var p = parserSpec.PARSER(); var s = parserSpec.superClass(); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.GRAMMAR) { Line = -1, Column = -1, Text = "grammar" }); text_before.TryGetValue(c as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(c, (in IParseTree n, out bool cc) => { cc = false; return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new W3CebnfLexer(str); var tokens = new CommonTokenStream(lexer); var parser = new W3CebnfParser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.prods(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(suffix) ? (error_file_name.Substring(0, error_file_name.Length - suffix.Length) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(suffix) ? (new_ffn.Substring(0, new_ffn.Length - suffix.Length) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } var name = System.IO.Path.GetFileName(ffn); name = System.IO.Path.GetFileNameWithoutExtension(ffn); var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); TreeEdits.InsertBefore(tree.GetChild(0), "grammar " + name + ";" + Environment.NewLine); using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rhs", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.InsertAfter(n, " ;"); } } using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//PPEQ", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Replace(n, ":"); } } // Convert double-quoted string literals to single quote. { using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//STRING", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { var text = n.GetText(); if (text.Length == 0) { continue; } if (text[0] != '"') { continue; } text = text.Substring(1, text.Length - 2); StringBuilder ss = new StringBuilder(); ss.Append("'"); foreach (var c in text) { if (c == '"') { ss.Append("\\"); } else { ss.Append(c); } } ss.Append("'"); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.STRING_LITERAL) { Line = -1, Column = -1, Text = ss.ToString() }); text_before.TryGetValue(n as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(n, new_sym); } } } StringBuilder sb = new StringBuilder(); TreeEdits.Reconstruct(sb, tree, text_before); var new_code = sb.ToString(); results.Add(new_ffn, new_code); results.Add(ffn.Replace(suffix, ".txt"), errors.ToString()); }