public override void EnterCat([NotNull] lbnfParser.CatContext context) { if (context.Identifier() == null) { return; } var id = context.Identifier().GetText(); if (context.Parent is lbnfParser.DefContext || context.Parent is lbnfParser.CatContext) { var frontier = TreeEdits.Frontier(context); var list = frontier.Select(t => t.Symbol).ToList(); ISymbol sym = new NonterminalSymbol(id, list); _pd.RootScope.define(ref sym); CombinedScopeSymbol s = (CombinedScopeSymbol)sym; _pd.Attributes[context] = new List <CombinedScopeSymbol>() { s }; foreach (var tr in frontier) { _pd.Attributes[tr] = new List <CombinedScopeSymbol>() { s } } ; } } }
public override void EnterCat([NotNull] lbnfParser.CatContext context) { if (context.Identifier() == null) { return; } var id = context.Identifier().GetText(); if (!(context.Parent is lbnfParser.ItemContext)) { return; } var frontier = TreeEdits.Frontier(context); List <ISymbol> prior = _pd.RootScope.LookupType(id).ToList(); if (!prior.Any()) { var listp = frontier.Select(t => t.Symbol).ToList(); ISymbol symp = new NonterminalSymbol(id, listp); _pd.RootScope.define(ref symp); prior = _pd.RootScope.LookupType(id).ToList(); } var list = frontier.Select(t => t.Symbol).ToList(); List <CombinedScopeSymbol> new_attrs = new List <CombinedScopeSymbol>(); CombinedScopeSymbol s = new RefSymbol(list, prior); new_attrs.Add(s); _pd.Attributes[context] = new_attrs; _pd.Attributes[context] = new List <CombinedScopeSymbol>() { s }; foreach (var tr in frontier) { _pd.Attributes[tr] = new List <CombinedScopeSymbol>() { s } } ; } }
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv2Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv2Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammar_(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g2") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g2") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove "header". using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//header_", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Remove classDef action blocks for now. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//classDef/actionBlock", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Delete(n); } } // Let's take care of options first. That's because we can't // determine if this is a combined grammar or not. // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var options = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeOptionsSpec)", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList(); var nodes = engine.parseExpression( @"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeParserOptionsSpec) /(option | lexerOption) [id/* [ text() = 'output' or text() = 'backtrack' or text() = 'buildAST' or text() = 'classHeaderSuffix' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); foreach (var opt in options) { if (opt.ChildCount == 3) { TreeEdits.Delete(opt); } } } // Delete rule options. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rule_/ruleOptionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' or text() = 'k' or text() = 'exportVocab' or text() = 'testLiterals' or text() = 'interactive' or text() = 'charVocabulary' or text() = 'defaultErrorHandler' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/ruleOptionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Parser and Lexer in One Definition using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var parser_nodes = engine.parseExpression( @"//parserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var lexer_nodes = engine.parseExpression( @"//lexerSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); var tree_nodes = engine.parseExpression( @"//treeParserSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); // Can this be a "combined" grammar? That can happen only if // one parser and one lexer decl and no tree decl. Options for // the lexer must be removed too. if (parser_nodes.Count() == 1 && lexer_nodes.Count() == 1 && tree_nodes.Count() == 0) { var lexerSpec = lexer_nodes.First() as ANTLRv2Parser.LexerSpecContext; if (lexerSpec.lexerOptionsSpec() == null) { // Nuke lexer class decl because it's a combined grammar. TreeEdits.Delete(lexerSpec); lexerSpec = null; } // Rewrite the parser spec. var parserSpec = parser_nodes.First() as ANTLRv2Parser.ParserSpecContext; var c = parserSpec.CLASS(); var i = parserSpec.id(); var e = parserSpec.EXTENDS(); var p = parserSpec.PARSER(); var s = parserSpec.superClass(); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.GRAMMAR) { Line = -1, Column = -1, Text = "grammar" }); text_before.TryGetValue(c as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(c, (in IParseTree n, out bool cc) => { cc = false; return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new ANTLRv3Lexer(str); var tokens = new CommonTokenStream(lexer); var parser = new ANTLRv3Parser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.grammarDef(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(".g3") ? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name; error_file_name = error_file_name.EndsWith(".g") ? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(".g3") ? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn; new_ffn = new_ffn.EndsWith(".g") ? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } // Transforms derived from two sources: // https://github.com/senseidb/sensei/pull/23 var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); // Remove unused options at top of grammar def. // This specifically looks at the options at the top of the file, // not rule-based options. That will be handled separately below. using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//grammarDef/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//grammarDef/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Fix options in the beginning of rules. // See https://theantlrguy.atlassian.net/wiki/spaces/ANTLR3/pages/2687029/Rule+and+subrule+options using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); // Allow language, tokenVocab, TokenLabelType, superClass var nodes = engine.parseExpression( @"//rule_/optionsSpec /option [id /(TOKEN_REF | RULE_REF) [text() = 'output' or text() = 'backtrack' or text() = 'memoize' or text() = 'ASTLabelType' or text() = 'rewrite' ]]", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); TreeEdits.Delete(nodes); var options = engine.parseExpression( @"//rule_/optionsSpec", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var os in options) { if (os.ChildCount == 3) { TreeEdits.Delete(os); } } } // Use new tokens{} syntax using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//tokensSpec /tokenSpec /SEMI", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); if (nodes.Any()) { // Delete tha last ";" in tokens list--change in syntax. var last = nodes.Last(); TreeEdits.Delete(last); // Replace all remaining ";" with ",". TreeEdits.Replace(tree, (in IParseTree n, out bool c) => { c = true; if (!nodes.Contains(n) && n != last) { return(null); } var t = n as TerminalNodeImpl; var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.COMMA) { Line = -1, Column = -1, Text = "," }); text_before.TryGetValue(t, out string v); if (v != null) { text_before.Add(new_sym, v); } return(new_sym); });
public void Try(string ffn, string input, ref Dictionary <string, string> results) { var now = DateTime.Now.ToString(); var errors = new StringBuilder(); var str = new AntlrInputStream(input); var lexer = new W3CebnfLexer(str); var tokens = new CommonTokenStream(lexer); var parser = new W3CebnfParser(tokens); var elistener = new ErrorListener <IToken>(parser, lexer, 0); parser.AddErrorListener(elistener); var tree = parser.prods(); var error_file_name = ffn; error_file_name = error_file_name.EndsWith(suffix) ? (error_file_name.Substring(0, error_file_name.Length - suffix.Length) + ".txt") : error_file_name; var new_ffn = ffn; new_ffn = new_ffn.EndsWith(suffix) ? (new_ffn.Substring(0, new_ffn.Length - suffix.Length) + ".g4") : new_ffn; if (elistener.had_error) { results.Add(error_file_name, errors.ToString()); return; } else { errors.AppendLine("File " + ffn + " parsed successfully."); errors.AppendLine("Date: " + now); } var name = System.IO.Path.GetFileName(ffn); name = System.IO.Path.GetFileNameWithoutExtension(ffn); var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree); TreeEdits.InsertBefore(tree.GetChild(0), "grammar " + name + ";" + Environment.NewLine); using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//rhs", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.InsertAfter(n, " ;"); } } using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//PPEQ", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { TreeEdits.Replace(n, ":"); } } // Convert double-quoted string literals to single quote. { using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser)) { org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine(); var nodes = engine.parseExpression( @"//STRING", new StaticContextBuilder()).evaluate( dynamicContext, new object[] { dynamicContext.Document }) .Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree); foreach (var n in nodes) { var text = n.GetText(); if (text.Length == 0) { continue; } if (text[0] != '"') { continue; } text = text.Substring(1, text.Length - 2); StringBuilder ss = new StringBuilder(); ss.Append("'"); foreach (var c in text) { if (c == '"') { ss.Append("\\"); } else { ss.Append(c); } } ss.Append("'"); var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.STRING_LITERAL) { Line = -1, Column = -1, Text = ss.ToString() }); text_before.TryGetValue(n as TerminalNodeImpl, out string v); if (v != null) { text_before.Add(new_sym, v); } TreeEdits.Replace(n, new_sym); } } } StringBuilder sb = new StringBuilder(); TreeEdits.Reconstruct(sb, tree, text_before); var new_code = sb.ToString(); results.Add(new_ffn, new_code); results.Add(ffn.Replace(suffix, ".txt"), errors.ToString()); }
public static List <DiagnosticInfo> PerformAnalysis(Document document) { List <DiagnosticInfo> result = new List <DiagnosticInfo>(); // Check if initial file is a grammar. ParsingResults pd_parser = ParsingResultsFactory.Create(document) as ParsingResults; if (pd_parser == null) { throw new LanguageServerException("A grammar file is not selected. Please select one first."); } Transform.ExtractGrammarType egt = new Transform.ExtractGrammarType(); ParseTreeWalker.Default.Walk(egt, pd_parser.ParseTree); bool is_grammar = egt.Type == Transform.ExtractGrammarType.GrammarType.Parser || egt.Type == Transform.ExtractGrammarType.GrammarType.Combined || egt.Type == Transform.ExtractGrammarType.GrammarType.Lexer; if (!is_grammar) { throw new LanguageServerException("A grammar file is not selected. Please select one first."); } // Find all other grammars by walking dependencies (import, vocab, file names). HashSet <string> read_files = new HashSet <string> { document.FullPath }; Dictionary <Workspaces.Document, List <TerminalNodeImpl> > every_damn_literal = new Dictionary <Workspaces.Document, List <TerminalNodeImpl> >(); for (; ;) { int before_count = read_files.Count; foreach (string f in read_files) { List <string> additional = ParsingResults.InverseImports.Where( t => t.Value.Contains(f)).Select( t => t.Key).ToList(); read_files = read_files.Union(additional).ToHashSet(); } foreach (string f in read_files) { var additional = ParsingResults.InverseImports.Where( t => t.Key == f).Select( t => t.Value); foreach (var t in additional) { read_files = read_files.Union(t).ToHashSet(); } } int after_count = read_files.Count; if (after_count == before_count) { break; } } { if (pd_parser.AllNodes != null) { int[] histogram = new int[pd_parser.Map.Length]; var fun = pd_parser.Classify; IEnumerable <IParseTree> it = pd_parser.AllNodes.Where(n => n is TerminalNodeImpl); foreach (var n in it) { var t = n as TerminalNodeImpl; int i = -1; try { i = pd_parser.Classify(pd_parser, pd_parser.Attributes, t); if (i >= 0) { histogram[i]++; } } catch (Exception) { } } for (int j = 0; j < histogram.Length; ++j) { string i = "Parser type " + j + " " + histogram[j]; result.Add( new DiagnosticInfo() { Document = document.FullPath, Severify = DiagnosticInfo.Severity.Info, Start = 0, End = 0, Message = i }); } } } //IParseTree rule = null; //var tarjan = new TarjanSCC<string, DirectedEdge<string>>(graph); //List<string> ordered = new List<string>(); //var sccs = tarjan.Compute(); //foreach (var scc in sccs) //{ // if (scc.Value.Count() <= 1) continue; // var k = scc.Key; // var v = scc.Value; // string i = "Participates in cycle " + // string.Join(" => ", scc.Value); // var (start, end) = table.rules.Where(r => r.LHS == k).Select(r => // { // var lmt = TreeEdits.LeftMostToken(r.rule); // var source_interval = lmt.SourceInterval; // int a = source_interval.a; // int b = source_interval.b; // IToken ta = pd_parser.TokStream.Get(a); // IToken tb = pd_parser.TokStream.Get(b); // var st = ta.StartIndex; // var ed = tb.StopIndex + 1; // return (st, ed); // }).FirstOrDefault(); // result.Add( // new DiagnosticInfo() // { // Document = document.FullPath, // Severify = DiagnosticInfo.Severity.Info, // Start = start, // End = end, // Message = i // }); //} // Check for useless lexer tokens. List <string> unused = new List <string>(); var pt = pd_parser.ParseTree; var l1 = TreeEdits.FindTopDown(pt, (in IParseTree t, out bool c) => { c = true; if (t is ANTLRv4Parser.LexerRuleSpecContext) { c = false; return(t); } return(null); }).ToList();