public void Execute(Repl repl, ReplParser.RupContext tree, bool piped)
{
var expr = tree.StringLiteral()?.GetText();
expr = expr?.Substring(1, expr.Length - 2);
var doc = repl.stack.Peek();
var pr = ParsingResultsFactory.Create(doc);
var aparser = pr.Parser;
var atree = pr.ParseTree;
List <IParseTree> nodes = null;
if (expr != null)
{
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList();
}
}
var results = LanguageServer.Transform.RemoveUselessParentheses(doc, nodes);
repl.EnactEdits(results);
}
public void Execute(Repl repl, ReplParser.UnulliteralContext tree, bool piped)
{
var type = tree.uclc()?.GetText();
var expr = tree.StringLiteral()?.GetText();
expr = expr?.Substring(1, expr.Length - 2);
var doc = repl.stack.Peek();
var pr = ParsingResultsFactory.Create(doc);
var aparser = pr.Parser;
var atree = pr.ParseTree;
if (expr != null)
{
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree as TerminalNodeImpl).ToList();
var results = LanguageServer.Transform.UnUpperLowerCaseLiteral(nodes, type == "uc", doc);
repl.EnactEdits(results);
}
}
else
{
var results = LanguageServer.Transform.UnUpperLowerCaseLiteral(null, type == "uc", doc);
repl.EnactEdits(results);
}
}
public void Execute(Repl repl, ReplParser.XgrepContext tree, bool piped)
{
var expr = repl.GetArg(tree.arg());
IParseTree[] atrees;
Parser parser;
Lexer lexer;
string text;
string fn;
ITokenStream tokstream;
if (piped)
{
var lines = repl.input_output_stack.Pop();
var serializeOptions = new JsonSerializerOptions();
serializeOptions.Converters.Add(new AntlrJson.ParseTreeConverter());
serializeOptions.WriteIndented = false;
var parse_info = JsonSerializer.Deserialize <AntlrJson.ParsingResultSet>(lines, serializeOptions);
text = parse_info.Text;
fn = parse_info.FileName;
atrees = parse_info.Nodes;
parser = parse_info.Parser;
lexer = parse_info.Lexer;
tokstream = parse_info.Stream;
}
else
{
var doc = repl.stack.Peek();
var pr = ParsingResultsFactory.Create(doc);
parser = pr.Parser;
lexer = pr.Lexer;
text = pr.Code;
fn = pr.FullFileName;
tokstream = pr.TokStream;
IParseTree atree = pr.ParseTree;
atrees = new IParseTree[] { atree };
}
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
IParseTree root = atrees.First().Root();
var ate = new AntlrTreeEditing.AntlrDOM.ConvertToDOM();
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = ate.Try(root, parser))
{
var l = atrees.Select(t => ate.FindDomNode(t));
var nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, l.ToArray())
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToArray();
var serializeOptions = new JsonSerializerOptions();
serializeOptions.Converters.Add(new AntlrJson.ParseTreeConverter());
serializeOptions.WriteIndented = false;
var parse_info_out = new AntlrJson.ParsingResultSet()
{
Text = text, FileName = fn, Lexer = lexer, Parser = parser, Stream = tokstream, Nodes = nodes
};
string js1 = JsonSerializer.Serialize(parse_info_out, serializeOptions);
repl.input_output_stack.Push(js1);
}
}
public void Execute(Repl repl, ReplParser.ReorderContext tree, bool piped)
{
Dictionary <string, string> results = new Dictionary <string, string>();
var doc = repl.stack.Peek();
string expr = null;
if (tree.modes() != null)
{
results = LanguageServer.Transform.SortModes(doc);
}
else
{
LspAntlr.ReorderType order = default;
if (tree.alpha() != null)
{
order = LspAntlr.ReorderType.Alphabetically;
}
else if (tree.bfs() != null)
{
order = LspAntlr.ReorderType.BFS;
expr = tree.bfs().StringLiteral().GetText();
}
else if (tree.dfs() != null)
{
order = LspAntlr.ReorderType.DFS;
expr = tree.dfs().StringLiteral().GetText();
}
else
{
throw new Exception("unknown sorting type");
}
List <IParseTree> nodes = null;
if (expr != null)
{
expr = expr.Substring(1, expr.Length - 2);
var pr = ParsingResultsFactory.Create(doc);
var aparser = pr.Parser;
var atree = pr.ParseTree;
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList();
}
}
results = LanguageServer.Transform.ReorderParserRules(doc, order, nodes);
}
repl.EnactEdits(results);
}
public void Execute(Repl repl, ReplParser.UnfoldContext tree, bool piped)
{
var expr = repl.GetArg(tree.arg());
var doc = repl.stack.Peek();
var pr = ParsingResultsFactory.Create(doc);
var aparser = pr.Parser;
var atree = pr.ParseTree;
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree as TerminalNodeImpl).ToList();
var results = LanguageServer.Transform.Unfold(nodes, doc);
repl.EnactEdits(results);
}
}
public void Execute(Repl repl, ReplParser.HasContext tree, bool piped)
{
var graph = tree.GRAPH() != null;
var expr = repl.GetArg(tree.arg());
var doc = repl.stack.Peek();
var pr = ParsingResultsFactory.Create(doc);
var aparser = pr.Parser;
var atree = pr.ParseTree;
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext = new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(atree, aparser))
{
List <IParseTree> nodes = null;
if (expr != null)
{
org.eclipse.wst.xml.xpath2.processor.Engine engine = new org.eclipse.wst.xml.xpath2.processor.Engine();
nodes = engine.parseExpression(expr,
new StaticContextBuilder()).evaluate(dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList();
}
if (tree.DR() != null)
{
var result = LanguageServer.Transform.HasDirectRec(doc, nodes);
foreach (var r in result)
{
System.Console.WriteLine(r);
}
}
else if (tree.IR() != null)
{
var result = LanguageServer.Transform.HasIndirectRec(nodes, graph, doc);
foreach (var r in result)
{
System.Console.WriteLine(r);
}
}
else
{
throw new Exception("unknown check");
}
}
}
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
bool convert_undefined_to_terminals = true;
string now = DateTime.Now.ToString();
StringBuilder errors = new StringBuilder();
StringBuilder sb = new StringBuilder();
AntlrInputStream str = new AntlrInputStream(input);
BisonLexer lexer = new BisonLexer(str);
CommonTokenStream tokens = new CommonTokenStream(lexer);
BisonParser parser = new BisonParser(tokens);
ErrorListenerBison <IToken> elistener = new ErrorListenerBison <IToken>(parser, lexer, tokens, errors);
parser.AddErrorListener(elistener);
BisonParser.InputContext tree = parser.input();
string error_file_name = ffn.Replace(".g4", ".txt");
if (elistener.had_error)
{
results.Add(ffn.Replace(".y", ".txt"), errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
// Get list of tokens. Convert this to a list of capitalized names.
Dictionary <string, string> terminals = new Dictionary <string, string>();
Dictionary <string, string> nonterminals = new Dictionary <string, string>();
List <Tuple <string, List <List <string> > > > rules = new List <Tuple <string, List <List <string> > > >();
// Collect terminals.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//token_decls//token_decl/id[position() = 1]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
Antlr4.Runtime.Tree.IParseTree parent;
foreach (var token in nodes)
{
string tok = token.GetText();
string cap_tok = tok.Length == 1 ? char.ToUpper(tok[0]).ToString() :
(char.ToUpper(tok[0]) + tok.Substring(1));
terminals.Add(tok, cap_tok);
}
}
// Collect rules.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//rules",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement));
foreach (var rule in nodes)
{
var r = rule.AntlrIParseTree;
var lhs = r.GetChild(0).GetText();
string cap_tok = lhs.Length == 1 ? char.ToLower(lhs[0]).ToString() :
(char.ToLower(lhs[0]) + lhs.Substring(1));
nonterminals[lhs] = cap_tok;
var rhs = new List <List <string> >();
var rhses = engine.parseExpression(
@".//rhses_1/rhs",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { rule })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement));
foreach (var r1 in rhses)
{
rhs.Add(new List <string>());
var sym = engine.parseExpression(
@".//symbol",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { r1 })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var s in sym)
{
List <string> l = rhs.Last();
l.Insert(0, s.GetText());
}
}
rules.Add(new Tuple <string, List <List <string> > >(lhs, rhs));
}
}
// First, collect information about the grammar.
// BisonGrammarListener listener = new BisonGrammarListener();
// ParseTreeWalker.Default.Walk(listener, tree);
if (convert_undefined_to_terminals)
{
foreach (Tuple <string, List <List <string> > > r in rules)
{
List <List <string> > rhs = r.Item2;
foreach (List <string> s in rhs)
{
foreach (string c in s)
{
if (rules.Where(rr => rr.Item1 == c).Any())
{
continue;
}
if (terminals.ContainsKey(c))
{
continue;
}
if (c[0] == '\'')
{
continue;
}
if (c == "error")
{
continue;
}
// RHS symbol is not a non-terminal and not a %token terminal.
// Enter it as a terminal.
string tok = c;
string cap_tok = tok.Length == 1 ? char.ToUpper(tok[0]).ToString() :
(char.ToUpper(tok[0]) + tok.Substring(1));
terminals.Add(tok, cap_tok);
// Note that in the error list.
errors.AppendLine("Symbol " + c + " not declared, assuming it is a terminal.");
}
}
}
}
// Convert any nonterminals which are keywords in Antlr...
foreach (Tuple <string, List <List <string> > > r in rules)
{
if (!terminals.ContainsKey(r.Item1) &&
(r.Item1 == "options" ||
r.Item1 == "grammar" ||
r.Item1 == "tokenVocab" ||
r.Item1 == "lexer" ||
r.Item1 == "parser" ||
r.Item1 == "rule"
))
{
if (!nonterminals.ContainsKey(r.Item1))
{
nonterminals.Add(r.Item1, r.Item1 + "_nonterminal");
}
}
else
{
if (!nonterminals.ContainsKey(r.Item1))
{
nonterminals.Add(r.Item1, r.Item1);
}
}
}
// Get the name of the grammar.
string name = System.IO.Path.GetFileNameWithoutExtension(ffn);
sb.AppendLine("// Combined Antlr4 grammar generated by Antlrvsix.");
sb.AppendLine("// Input grammar: " + ffn);
sb.AppendLine("// Date: " + now);
sb.AppendLine();
sb.AppendLine("grammar " + name + ";");
// Output the rules.
foreach (Tuple <string, List <List <string> > > r in rules)
{
string lhs = r.Item1;
lhs = nonterminals[lhs];
nonterminals.TryGetValue(lhs, out string nlhs);
if (nlhs != null && nlhs != "")
{
lhs = nlhs;
}
sb.Append(lhs);
List <List <string> > rhs = r.Item2;
bool first = true;
foreach (List <string> s in rhs)
{
sb.Append(first ? " :" : " |");
first = false;
foreach (string c in s)
{
var re = c;
if (terminals.ContainsKey(re))
{
re = terminals[re];
}
else if (nonterminals.ContainsKey(re))
{
re = nonterminals[re];
}
sb.Append(" " + re);
}
sb.AppendLine();
}
sb.AppendLine(" ;");
}
// Check for use of error symbol. The semantics is for the parser to look for
// the ERROR token.
bool found = false;
foreach (Tuple <string, List <List <string> > > r in rules)
{
List <List <string> > rhs = r.Item2;
foreach (List <string> s in rhs)
{
foreach (string c in s)
{
if (c == "error")
{
found = true;
goto bigexit;
}
}
}
}
bigexit:
if (found)
{
sb.AppendLine("error : ERROR ;");
}
results.Add(ffn.Replace(".y", ".g4"), sb.ToString());
results.Add(ffn.Replace(".y", ".txt"), errors.ToString());
}
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new ANTLRv2Lexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new ANTLRv2Parser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.grammar_();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(".g2")
? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name;
error_file_name = error_file_name.EndsWith(".g")
? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(".g2")
? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn;
new_ffn = new_ffn.EndsWith(".g")
? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
// Transforms derived from two sources:
// https://github.com/senseidb/sensei/pull/23
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
// Remove "header".
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//header_",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Delete(n);
}
}
// Remove classDef action blocks for now.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//classDef/actionBlock",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Delete(n);
}
}
// Let's take care of options first. That's because we can't
// determine if this is a combined grammar or not.
// Remove unused options at top of grammar def.
// This specifically looks at the options at the top of the file,
// not rule-based options. That will be handled separately below.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var options = engine.parseExpression(
@"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeOptionsSpec)",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList();
var nodes = engine.parseExpression(
@"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeParserOptionsSpec)
/(option | lexerOption)
[id/*
[
text() = 'output'
or text() = 'backtrack'
or text() = 'buildAST'
or text() = 'classHeaderSuffix'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
or text() = 'k'
or text() = 'exportVocab'
or text() = 'testLiterals'
or text() = 'interactive'
or text() = 'charVocabulary'
or text() = 'defaultErrorHandler'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
foreach (var opt in options)
{
if (opt.ChildCount == 3)
{
TreeEdits.Delete(opt);
}
}
}
// Delete rule options.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//rule_/ruleOptionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
or text() = 'k'
or text() = 'exportVocab'
or text() = 'testLiterals'
or text() = 'interactive'
or text() = 'charVocabulary'
or text() = 'defaultErrorHandler'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//rule_/ruleOptionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Parser and Lexer in One Definition
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var parser_nodes = engine.parseExpression(
@"//parserSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
var lexer_nodes = engine.parseExpression(
@"//lexerSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
var tree_nodes = engine.parseExpression(
@"//treeParserSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
// Can this be a "combined" grammar? That can happen only if
// one parser and one lexer decl and no tree decl. Options for
// the lexer must be removed too.
if (parser_nodes.Count() == 1 && lexer_nodes.Count() == 1 && tree_nodes.Count() == 0)
{
var lexerSpec = lexer_nodes.First() as ANTLRv2Parser.LexerSpecContext;
if (lexerSpec.lexerOptionsSpec() == null)
{
// Nuke lexer class decl because it's a combined grammar.
TreeEdits.Delete(lexerSpec);
lexerSpec = null;
}
// Rewrite the parser spec.
var parserSpec = parser_nodes.First() as ANTLRv2Parser.ParserSpecContext;
var c = parserSpec.CLASS();
var i = parserSpec.id();
var e = parserSpec.EXTENDS();
var p = parserSpec.PARSER();
var s = parserSpec.superClass();
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.GRAMMAR)
{
Line = -1, Column = -1, Text = "grammar"
});
text_before.TryGetValue(c as TerminalNodeImpl, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
TreeEdits.Replace(c, (in IParseTree n, out bool cc) =>
{
cc = false;
return(new_sym);
});
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new ANTLRv3Lexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new ANTLRv3Parser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.grammarDef();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(".g3")
? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name;
error_file_name = error_file_name.EndsWith(".g")
? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(".g3")
? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn;
new_ffn = new_ffn.EndsWith(".g")
? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
// Transforms derived from two sources:
// https://github.com/senseidb/sensei/pull/23
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
// Remove unused options at top of grammar def.
// This specifically looks at the options at the top of the file,
// not rule-based options. That will be handled separately below.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
// Allow language, tokenVocab, TokenLabelType, superClass
var nodes = engine.parseExpression(
@"//grammarDef/optionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//grammarDef/optionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Fix options in the beginning of rules.
// See https://theantlrguy.atlassian.net/wiki/spaces/ANTLR3/pages/2687029/Rule+and+subrule+options
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
// Allow language, tokenVocab, TokenLabelType, superClass
var nodes = engine.parseExpression(
@"//rule_/optionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//rule_/optionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Use new tokens{} syntax
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//tokensSpec
/tokenSpec
/SEMI",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
if (nodes.Any())
{
// Delete tha last ";" in tokens list--change in syntax.
var last = nodes.Last();
TreeEdits.Delete(last);
// Replace all remaining ";" with ",".
TreeEdits.Replace(tree, (in IParseTree n, out bool c) =>
{
c = true;
if (!nodes.Contains(n) && n != last)
{
return(null);
}
var t = n as TerminalNodeImpl;
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.COMMA)
{
Line = -1, Column = -1, Text = ","
});
text_before.TryGetValue(t, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
return(new_sym);
});
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new W3CebnfLexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new W3CebnfParser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.prods();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(suffix)
? (error_file_name.Substring(0, error_file_name.Length - suffix.Length) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(suffix)
? (new_ffn.Substring(0, new_ffn.Length - suffix.Length) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
var name = System.IO.Path.GetFileName(ffn);
name = System.IO.Path.GetFileNameWithoutExtension(ffn);
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
TreeEdits.InsertBefore(tree.GetChild(0), "grammar " + name + ";" + Environment.NewLine);
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//rhs",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.InsertAfter(n, " ;");
}
}
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//PPEQ",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Replace(n,
":");
}
}
// Convert double-quoted string literals to single quote.
{
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//STRING",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
var text = n.GetText();
if (text.Length == 0)
{
continue;
}
if (text[0] != '"')
{
continue;
}
text = text.Substring(1, text.Length - 2);
StringBuilder ss = new StringBuilder();
ss.Append("'");
foreach (var c in text)
{
if (c == '"')
{
ss.Append("\\");
}
else
{
ss.Append(c);
}
}
ss.Append("'");
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.STRING_LITERAL)
{
Line = -1, Column = -1, Text = ss.ToString()
});
text_before.TryGetValue(n as TerminalNodeImpl, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
TreeEdits.Replace(n, new_sym);
}
}
}
StringBuilder sb = new StringBuilder();
TreeEdits.Reconstruct(sb, tree, text_before);
var new_code = sb.ToString();
results.Add(new_ffn, new_code);
results.Add(ffn.Replace(suffix, ".txt"), errors.ToString());
}
