public override void Parse(string code,
out CommonTokenStream TokStream,
out Parser Parser,
out Lexer Lexer,
out IParseTree ParseTree)
{
IParseTree pt = null;
// Set up Antlr to parse input grammar.
byte[] byteArray = Encoding.UTF8.GetBytes(code);
AntlrInputStream ais = new AntlrInputStream(
new StreamReader(
new MemoryStream(byteArray)).ReadToEnd());
var lexer = new W3CebnfLexer(ais);
CommonTokenStream cts = new CommonTokenStream(lexer);
var parser = new W3CebnfParser(cts);
lexer.RemoveErrorListeners();
var lexer_error_listener = new ErrorListener <int>(parser, lexer, this.QuietAfter);
lexer.AddErrorListener(lexer_error_listener);
parser.RemoveErrorListeners();
var parser_error_listener = new ErrorListener <IToken>(parser, lexer, this.QuietAfter);
parser.AddErrorListener(parser_error_listener);
try
{
pt = parser.prods();
}
catch (Exception)
{
// Parsing error.
}
TokStream = cts;
Parser = parser;
Lexer = lexer;
ParseTree = pt;
}
public override void Parse(ParsingResults pd, bool bail)
{
string ffn = pd.FullFileName;
string code = pd.Code;
if (ffn == null)
{
return;
}
if (code == null)
{
return;
}
this.QuietAfter = pd.QuietAfter;
IParseTree pt = null;
// Set up Antlr to parse input grammar.
byte[] byteArray = Encoding.UTF8.GetBytes(code);
AntlrInputStream ais = new AntlrInputStream(
new StreamReader(
new MemoryStream(byteArray)).ReadToEnd())
{
name = ffn
};
var lexer = new W3CebnfLexer(ais);
CommonTokenStream cts = new CommonTokenStream(lexer);
var parser = new W3CebnfParser(cts);
lexer.RemoveErrorListeners();
var lexer_error_listener = new ErrorListener <int>(parser, lexer, pd.QuietAfter);
lexer.AddErrorListener(lexer_error_listener);
parser.RemoveErrorListeners();
var parser_error_listener = new ErrorListener <IToken>(parser, lexer, pd.QuietAfter);
parser.AddErrorListener(parser_error_listener);
BailErrorHandler bail_error_handler = null;
if (bail)
{
bail_error_handler = new BailErrorHandler();
parser.ErrorHandler = bail_error_handler;
}
try
{
pt = parser.prods();
}
catch (Exception)
{
// Parsing error.
}
//StringBuilder sb = new StringBuilder();
//TreeSerializer.ParenthesizedAST(pt, sb, "", cts);
//string fn = System.IO.Path.GetFileName(ffn);
//fn = "c:\\temp\\" + fn;
//System.IO.File.WriteAllText(fn, sb.ToString());
if (parser_error_listener.had_error || lexer_error_listener.had_error || (bail_error_handler != null && bail_error_handler.had_error))
{
System.Console.Error.WriteLine("Error in parse of " + ffn);
}
else
{
System.Console.Error.WriteLine("Parse completed of " + ffn);
}
pd.TokStream = cts;
pd.Parser = parser;
pd.Lexer = lexer;
pd.ParseTree = pt;
Stack <IParseTree> stack = new Stack <IParseTree>();
stack.Push(pt);
while (stack.Any())
{
var x = stack.Pop();
if (x is TerminalNodeImpl leaf)
{
}
else
{
var y = x as AttributedParseTreeNode;
if (y != null)
{
y.ParserDetails = pd;
}
for (int i = 0; i < x.ChildCount; ++i)
{
var c = x.GetChild(i);
if (c != null)
{
stack.Push(c);
}
}
}
}
}
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new ANTLRv3Lexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new ANTLRv3Parser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.grammarDef();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(".g3")
? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name;
error_file_name = error_file_name.EndsWith(".g")
? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(".g3")
? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn;
new_ffn = new_ffn.EndsWith(".g")
? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
// Transforms derived from two sources:
// https://github.com/senseidb/sensei/pull/23
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
// Remove unused options at top of grammar def.
// This specifically looks at the options at the top of the file,
// not rule-based options. That will be handled separately below.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
// Allow language, tokenVocab, TokenLabelType, superClass
var nodes = engine.parseExpression(
@"//grammarDef/optionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//grammarDef/optionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Fix options in the beginning of rules.
// See https://theantlrguy.atlassian.net/wiki/spaces/ANTLR3/pages/2687029/Rule+and+subrule+options
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
// Allow language, tokenVocab, TokenLabelType, superClass
var nodes = engine.parseExpression(
@"//rule_/optionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//rule_/optionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Use new tokens{} syntax
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//tokensSpec
/tokenSpec
/SEMI",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
if (nodes.Any())
{
// Delete tha last ";" in tokens list--change in syntax.
var last = nodes.Last();
TreeEdits.Delete(last);
// Replace all remaining ";" with ",".
TreeEdits.Replace(tree, (in IParseTree n, out bool c) =>
{
c = true;
if (!nodes.Contains(n) && n != last)
{
return(null);
}
var t = n as TerminalNodeImpl;
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.COMMA)
{
Line = -1, Column = -1, Text = ","
});
text_before.TryGetValue(t, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
return(new_sym);
});
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new ANTLRv2Lexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new ANTLRv2Parser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.grammar_();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(".g2")
? (error_file_name.Substring(0, error_file_name.Length - 3) + ".txt") : error_file_name;
error_file_name = error_file_name.EndsWith(".g")
? (error_file_name.Substring(0, error_file_name.Length - 2) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(".g2")
? (new_ffn.Substring(0, new_ffn.Length - 3) + ".g4") : new_ffn;
new_ffn = new_ffn.EndsWith(".g")
? (new_ffn.Substring(0, new_ffn.Length - 2) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
// Transforms derived from two sources:
// https://github.com/senseidb/sensei/pull/23
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
// Remove "header".
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//header_",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Delete(n);
}
}
// Remove classDef action blocks for now.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//classDef/actionBlock",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Delete(n);
}
}
// Let's take care of options first. That's because we can't
// determine if this is a combined grammar or not.
// Remove unused options at top of grammar def.
// This specifically looks at the options at the top of the file,
// not rule-based options. That will be handled separately below.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var options = engine.parseExpression(
@"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeOptionsSpec)",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree).ToList();
var nodes = engine.parseExpression(
@"//(fileOptionsSpec | parserOptionsSpec | lexerOptionsSpec | treeParserOptionsSpec)
/(option | lexerOption)
[id/*
[
text() = 'output'
or text() = 'backtrack'
or text() = 'buildAST'
or text() = 'classHeaderSuffix'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
or text() = 'k'
or text() = 'exportVocab'
or text() = 'testLiterals'
or text() = 'interactive'
or text() = 'charVocabulary'
or text() = 'defaultErrorHandler'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
foreach (var opt in options)
{
if (opt.ChildCount == 3)
{
TreeEdits.Delete(opt);
}
}
}
// Delete rule options.
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//rule_/ruleOptionsSpec
/option
[id
/(TOKEN_REF | RULE_REF)
[text() = 'output'
or text() = 'backtrack'
or text() = 'memoize'
or text() = 'ASTLabelType'
or text() = 'rewrite'
or text() = 'k'
or text() = 'exportVocab'
or text() = 'testLiterals'
or text() = 'interactive'
or text() = 'charVocabulary'
or text() = 'defaultErrorHandler'
]]",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
TreeEdits.Delete(nodes);
var options = engine.parseExpression(
@"//rule_/ruleOptionsSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var os in options)
{
if (os.ChildCount == 3)
{
TreeEdits.Delete(os);
}
}
}
// Parser and Lexer in One Definition
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var parser_nodes = engine.parseExpression(
@"//parserSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
var lexer_nodes = engine.parseExpression(
@"//lexerSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
var tree_nodes = engine.parseExpression(
@"//treeParserSpec",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
// Can this be a "combined" grammar? That can happen only if
// one parser and one lexer decl and no tree decl. Options for
// the lexer must be removed too.
if (parser_nodes.Count() == 1 && lexer_nodes.Count() == 1 && tree_nodes.Count() == 0)
{
var lexerSpec = lexer_nodes.First() as ANTLRv2Parser.LexerSpecContext;
if (lexerSpec.lexerOptionsSpec() == null)
{
// Nuke lexer class decl because it's a combined grammar.
TreeEdits.Delete(lexerSpec);
lexerSpec = null;
}
// Rewrite the parser spec.
var parserSpec = parser_nodes.First() as ANTLRv2Parser.ParserSpecContext;
var c = parserSpec.CLASS();
var i = parserSpec.id();
var e = parserSpec.EXTENDS();
var p = parserSpec.PARSER();
var s = parserSpec.superClass();
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.GRAMMAR)
{
Line = -1, Column = -1, Text = "grammar"
});
text_before.TryGetValue(c as TerminalNodeImpl, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
TreeEdits.Replace(c, (in IParseTree n, out bool cc) =>
{
cc = false;
return(new_sym);
});
public void Try(string ffn, string input, ref Dictionary <string, string> results)
{
var now = DateTime.Now.ToString();
var errors = new StringBuilder();
var str = new AntlrInputStream(input);
var lexer = new W3CebnfLexer(str);
var tokens = new CommonTokenStream(lexer);
var parser = new W3CebnfParser(tokens);
var elistener = new ErrorListener <IToken>(parser, lexer, 0);
parser.AddErrorListener(elistener);
var tree = parser.prods();
var error_file_name = ffn;
error_file_name = error_file_name.EndsWith(suffix)
? (error_file_name.Substring(0, error_file_name.Length - suffix.Length) + ".txt") : error_file_name;
var new_ffn = ffn;
new_ffn = new_ffn.EndsWith(suffix)
? (new_ffn.Substring(0, new_ffn.Length - suffix.Length) + ".g4") : new_ffn;
if (elistener.had_error)
{
results.Add(error_file_name, errors.ToString());
return;
}
else
{
errors.AppendLine("File " + ffn + " parsed successfully.");
errors.AppendLine("Date: " + now);
}
var name = System.IO.Path.GetFileName(ffn);
name = System.IO.Path.GetFileNameWithoutExtension(ffn);
var(text_before, other) = TreeEdits.TextToLeftOfLeaves(tokens, tree);
TreeEdits.InsertBefore(tree.GetChild(0), "grammar " + name + ";" + Environment.NewLine);
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//rhs",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.InsertAfter(n, " ;");
}
}
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//PPEQ",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
TreeEdits.Replace(n,
":");
}
}
// Convert double-quoted string literals to single quote.
{
using (AntlrTreeEditing.AntlrDOM.AntlrDynamicContext dynamicContext =
new AntlrTreeEditing.AntlrDOM.ConvertToDOM().Try(tree, parser))
{
org.eclipse.wst.xml.xpath2.processor.Engine engine =
new org.eclipse.wst.xml.xpath2.processor.Engine();
var nodes = engine.parseExpression(
@"//STRING",
new StaticContextBuilder()).evaluate(
dynamicContext, new object[] { dynamicContext.Document })
.Select(x => (x.NativeValue as AntlrTreeEditing.AntlrDOM.AntlrElement).AntlrIParseTree);
foreach (var n in nodes)
{
var text = n.GetText();
if (text.Length == 0)
{
continue;
}
if (text[0] != '"')
{
continue;
}
text = text.Substring(1, text.Length - 2);
StringBuilder ss = new StringBuilder();
ss.Append("'");
foreach (var c in text)
{
if (c == '"')
{
ss.Append("\\");
}
else
{
ss.Append(c);
}
}
ss.Append("'");
var new_sym = new TerminalNodeImpl(new CommonToken(ANTLRv4Lexer.STRING_LITERAL)
{
Line = -1, Column = -1, Text = ss.ToString()
});
text_before.TryGetValue(n as TerminalNodeImpl, out string v);
if (v != null)
{
text_before.Add(new_sym, v);
}
TreeEdits.Replace(n, new_sym);
}
}
}
StringBuilder sb = new StringBuilder();
TreeEdits.Reconstruct(sb, tree, text_before);
var new_code = sb.ToString();
results.Add(new_ffn, new_code);
results.Add(ffn.Replace(suffix, ".txt"), errors.ToString());
}
