public BuildDependencyGenerator(AntlrTool tool, Grammar g)
{
this.tool = tool;
this.g = g;
string language = g.GetOptionString("language");
generator = new CodeGenerator(tool, g, language);
}
public LeftFactoringRuleTransformer([NotNull] GrammarRootAST ast, [NotNull] IDictionary<string, Rule> rules, [NotNull] Grammar g)
{
this._ast = ast;
this._rules = rules;
this._g = g;
this._tool = g.tool;
}
public LeftRecursiveRuleTransformer(GrammarRootAST ast, ICollection<Rule> rules, Grammar g)
{
this.ast = ast;
this.rules = rules;
this.g = g;
this.tool = g.tool;
}
protected override void GenRecognizerFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate outputFileST)
{
// Before we write this, and cause it to generate its string,
// we need to add all the string literals that we are going to match
//
outputFileST.Add("literals", strings);
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
public LeftRecursiveRuleAnalyzer(GrammarAST ruleAST,
AntlrTool tool, string ruleName, string language)
: base(new CommonTreeNodeStream(new GrammarASTAdaptor(ruleAST.Token.InputStream), ruleAST))
{
this.tool = tool;
this.ruleName = ruleName;
this.language = language;
this.tokenStream = ruleAST.g.tokenStream;
if (this.tokenStream == null)
{
throw new InvalidOperationException("grammar must have a token stream");
}
LoadPrecRuleTemplates();
}
public LeftRecursiveRuleAnalyzer(GrammarAST ruleAST,
AntlrTool tool, string ruleName, string language)
: base(new CommonTreeNodeStream(new GrammarASTAdaptor(ruleAST.Token.InputStream), ruleAST))
{
this.tool = tool;
this.ruleName = ruleName;
this.language = language;
this.tokenStream = ruleAST.g.tokenStream;
if (this.tokenStream == null)
{
throw new InvalidOperationException("grammar must have a token stream");
}
LoadPrecRuleTemplates();
}
protected override void GenRecognizerHeaderFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName)
{
// Pick up the file name we are generating. This method will return a
// a file suffixed with .c, so we must substring and add the extName
// to it as we cannot assign into strings in Java.
//
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
fileName = fileName.Substring(0, fileName.Length - 2) + extName;
generator.Write(headerFileST, fileName);
}
protected override void GenRecognizerHeaderFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName)
{
//Its better we remove the EOF Token, as it would have been defined everywhere in C.
//we define it later as "EOF_TOKEN" instead of "EOF"
IList <object> tokens = (IList <object>)headerFileST.GetAttribute("tokens");
for (int i = 0; i < tokens.Count; ++i)
{
bool can_break = false;
object tok = tokens[i];
if (tok is Aggregate)
{
Aggregate atok = (Aggregate)tok;
foreach (var pair in atok.Properties)
{
if (pair.Value.Equals("EOF"))
{
tokens.RemoveAt(i);
can_break = true;
break;
}
}
}
if (can_break)
{
break;
}
}
// Pick up the file name we are generating. This method will return a
// a file suffixed with .c, so we must substring and add the extName
// to it as we cannot assign into strings in Java.
//
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
fileName = fileName.Substring(0, fileName.Length - 4) + extName;
generator.Write(headerFileST, fileName);
}
private static TemplateGroup LoadPrecRuleTemplates(AntlrTool tool)
{
string templateDirs = tool.TemplatesDirectory;
TemplateGroup group;
if (!recRuleTemplatesCache.TryGetValue(templateDirs, out group))
{
string fileName = CodeGenerator.FindTemplateFile(templateDirs.Split(':'), "LeftRecursiveRules.stg");
group = new ToolTemplateGroupFile(fileName);
if (!group.IsDefined("recRuleName"))
{
recRuleTemplatesCache[templateDirs] = group;
}
else
{
ErrorManager.Error(ErrorManager.MSG_MISSING_CODE_GEN_TEMPLATES, "PrecRules");
return(null);
}
}
return(group);
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, Template outputFileST)
{
if (!grammar.IsRoot)
{
Grammar rootGrammar = grammar.composite.RootGrammar;
string actionScope = grammar.GetDefaultActionScope(grammar.type);
IDictionary <string, object> actions;
object rootNamespace;
if (rootGrammar.Actions.TryGetValue(actionScope, out actions) && actions.TryGetValue("namespace", out rootNamespace))
{
if (!grammar.Actions.TryGetValue(actionScope, out actions))
{
actions = new Dictionary <string, object>();
grammar.Actions[actionScope] = actions;
}
actions["namespace"] = rootNamespace;
}
}
generator.Templates.RegisterRenderer(typeof(string), new StringRenderer(generator, this));
base.GenRecognizerFile(tool, generator, grammar, outputFileST);
}
public LexerGrammar(AntlrTool tool, GrammarRootAST ast)
: base(tool, ast)
{
}
public ToolANTLRLexer(ICharStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST)
{
/*
Below is an experimental attempt at providing a few named action blocks
that are printed in both lexer and parser files from combined grammars.
ANTLR appears to first generate a parser, then generate an independent lexer,
and then generate code from that. It keeps the combo/parser grammar object
and the lexer grammar object, as well as their respective code generator and
target instances, completely independent. So, while a bit hack-ish, this is
a solution that should work without having to modify Terrence Parr's
core tool code.
- sharedActionBlocks is a class variable containing a hash map
- if this method is called with a combo grammar, and the action map
in the grammar contains an entry for the named scope "all",
add an entry to sharedActionBlocks mapping the grammar name to
the "all" action map.
- if this method is called with an `implicit lexer'
(one that's extracted from a combo grammar), check to see if
there's an entry in sharedActionBlocks for the lexer's grammar name.
- if there is an action map entry, place it in the lexer's action map
- the recognizerFile template has code to place the
"all" actions appropriately
problems:
- This solution assumes that the parser will be generated
before the lexer. If that changes at some point, this will
not work.
- I have not investigated how this works with delegation yet
Kyle Yetter - March 25, 2010
*/
if (grammar.type == GrammarType.Combined)
{
IDictionary<string, object> all;
if (grammar.Actions.TryGetValue("all", out all))
sharedActionBlocks[grammar.name] = all;
}
else if (grammar.implicitLexer)
{
IDictionary<string, object> shared;
if (sharedActionBlocks.TryGetValue(grammar.name, out shared))
grammar.Actions["all"] = shared;
}
generator.Templates.RegisterRenderer(typeof(string), new RubyRenderer());
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
public CodeGenerator([NotNull] AntlrTool tool, [NotNull] Grammar g, string language)
{
this.g = g;
this.tool = tool;
this.language = language != null ? language : DEFAULT_LANGUAGE;
}
public GrammarTransformPipeline(Grammar g, AntlrTool tool)
{
this.g = g;
this.tool = tool;
}
protected override void GenRecognizerHeaderFile( AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName )
{
// Pick up the file name we are generating. This method will return a
// a file suffixed with .c, so we must substring and add the extName
// to it as we cannot assign into strings in Java.
///
string fileName = generator.GetRecognizerFileName( grammar.name, grammar.type );
fileName = fileName.Substring( 0, fileName.Length - 2 ) + extName;
generator.Write( headerFileST, fileName );
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST)
{
generator.Templates.RegisterRenderer(typeof(string), new StringRenderer(generator, this));
base.GenRecognizerFile(tool, generator, grammar, outputFileST);
}
public bool Execute()
{
try
{
string executable = null;
if (!UseCSharpGenerator)
{
try
{
if (!string.IsNullOrEmpty(JavaExecutable))
{
executable = JavaExecutable;
}
else
{
string javaHome = JavaHome;
executable = Path.Combine(Path.Combine(javaHome, "bin"), "java.exe");
if (!File.Exists(executable))
{
executable = Path.Combine(Path.Combine(javaHome, "bin"), "java");
}
}
}
catch (NotSupportedException)
{
// Fall back to using the new code generation tools
UseCSharpGenerator = true;
}
}
if (UseCSharpGenerator)
{
#if NETSTANDARD
string framework = "netstandard";
string extension = ".dll";
#else
string framework = "net45";
string extension = ".exe";
#endif
executable = Path.Combine(Path.Combine(Path.GetDirectoryName(ToolPath), framework), "Antlr4" + extension);
}
List <string> arguments = new List <string>();
if (!UseCSharpGenerator)
{
arguments.Add("-cp");
arguments.Add(ToolPath);
arguments.Add("org.antlr.v4.CSharpTool");
}
arguments.Add("-o");
arguments.Add(OutputPath);
if (!string.IsNullOrEmpty(Encoding))
{
arguments.Add("-encoding");
arguments.Add(Encoding);
}
if (GenerateListener)
{
arguments.Add("-listener");
}
else
{
arguments.Add("-no-listener");
}
if (GenerateVisitor)
{
arguments.Add("-visitor");
}
else
{
arguments.Add("-no-visitor");
}
if (ForceAtn)
{
arguments.Add("-Xforce-atn");
}
if (AbstractGrammar)
{
arguments.Add("-Dabstract=true");
}
if (!string.IsNullOrEmpty(TargetLanguage))
{
// Since the C# target currently produces the same code for all target framework versions, we can
// avoid bugs with support for newer frameworks by just passing CSharp as the language and allowing
// the tool to use a default.
arguments.Add("-Dlanguage=" + TargetLanguage);
}
if (!string.IsNullOrEmpty(TargetNamespace))
{
arguments.Add("-package");
arguments.Add(TargetNamespace);
}
arguments.AddRange(SourceCodeFiles);
if (UseCSharpGenerator)
{
if (UseInternalAccessModifier)
{
arguments.Add("-DuseInternalAccessModifier");
}
if (ExcludeClsCompliantAttribute)
{
arguments.Add("-DexcludeClsCompliantAttribute");
}
if (IncludeDebuggerNonUserCodeAttribute)
{
arguments.Add("-DincludeDebuggerNonUserCodeAttribute");
}
}
#if NETSTANDARD
if (UseCSharpGenerator)
{
var outWriter = new StringWriter();
var errorWriter = new StringWriter();
try
{
var antlr = new AntlrTool(arguments.ToArray())
{
ConsoleOut = outWriter,
ConsoleError = errorWriter
};
antlr.ProcessGrammarsOnCommandLine();
return(antlr.errMgr.GetNumErrors() == 0);
}
finally
{
foreach (var line in outWriter.ToString().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
{
HandleOutputDataReceived(line);
}
foreach (var line in errorWriter.ToString().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
{
HandleErrorDataReceived(line);
}
}
}
#endif
ProcessStartInfo startInfo = new ProcessStartInfo(executable, JoinArguments(arguments))
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
this.BuildMessages.Add(new BuildMessage(TraceLevel.Info, "Executing command: \"" + startInfo.FileName + "\" " + startInfo.Arguments, "", 0, 0));
Process process = new Process();
process.StartInfo = startInfo;
process.ErrorDataReceived += HandleErrorDataReceived;
process.OutputDataReceived += HandleOutputDataReceived;
process.Start();
process.BeginErrorReadLine();
process.BeginOutputReadLine();
process.StandardInput.Dispose();
process.WaitForExit();
return(process.ExitCode == 0);
//using (LoggingTraceListener traceListener = new LoggingTraceListener(_buildMessages))
//{
// SetTraceListener(traceListener);
// ProcessArgs(args.ToArray());
// process();
//}
//_generatedCodeFiles.AddRange(GetGeneratedFiles().Where(file => LanguageSourceExtensions.Contains(Path.GetExtension(file), StringComparer.OrdinalIgnoreCase)));
//int errorCount = GetNumErrors();
//return errorCount == 0;
}
catch (Exception e)
{
if (e is TargetInvocationException && e.InnerException != null)
{
e = e.InnerException;
}
_buildMessages.Add(new BuildMessage(e.Message));
throw;
}
}
protected override void GenRecognizerHeaderFile( AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName )
{
StringTemplateGroup templates = generator.Templates;
generator.Write( headerFileST, grammar.name + extName );
}
public OutputModelWalker(AntlrTool tool, TemplateGroup templates)
{
this.tool = tool;
this.templates = templates;
}
public LexerGrammar(AntlrTool tool, GrammarRootAST ast)
: base(tool, ast)
{
}
public DefaultToolListener(AntlrTool tool)
{
this.tool = tool;
}
protected override void GenRecognizerHeaderFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName)
{
//Its better we remove the EOF Token, as it would have been defined everywhere in C.
//we define it later as "EOF_TOKEN" instead of "EOF"
IList<object> tokens = (IList<object>)headerFileST.GetAttribute("tokens");
for (int i = 0; i < tokens.Count; ++i)
{
bool can_break = false;
object tok = tokens[i];
if (tok is Aggregate)
{
Aggregate atok = (Aggregate)tok;
foreach (var pair in atok.Properties)
{
if (pair.Value.Equals("EOF"))
{
tokens.RemoveAt(i);
can_break = true;
break;
}
}
}
if (can_break)
break;
}
// Pick up the file name we are generating. This method will return a
// a file suffixed with .c, so we must substring and add the extName
// to it as we cannot assign into strings in Java.
///
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
fileName = fileName.Substring(0, fileName.Length - 4) + extName;
generator.Write(headerFileST, fileName);
}
protected override void GenRecognizerFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate outputFileST)
{
// Before we write this, and cause it to generate its string,
// we need to add all the string literals that we are going to match
//
outputFileST.Add("literals", strings);
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
protected StringTemplate chooseWhereCyclicDFAsGo(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate recognizerST,
StringTemplate cyclicDFAST)
{
return recognizerST;
}
public GrammarTransformPipeline(Grammar g, AntlrTool tool)
{
this.g = g;
this.tool = tool;
}
public ToolANTLRParser(ITokenStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
public CodeGenerator([NotNull] AntlrTool tool, [NotNull] Grammar g, string language)
{
this.g = g;
this.tool = tool;
this.language = language != null ? language : DEFAULT_LANGUAGE;
}
protected override void GenRecognizerHeaderFile( AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName )
{
generator.Write( headerFileST, grammar.name + Grammar.grammarTypeToFileNameSuffix[(int)grammar.type] + extName );
}
public ErrorManager(AntlrTool tool)
{
this.tool = tool;
}
public OutputModelWalker(AntlrTool tool, TemplateGroup templates)
{
this.tool = tool;
this.templates = templates;
}
public ErrorManager(AntlrTool tool)
{
this.tool = tool;
}
public DefaultToolListener(AntlrTool tool)
{
this.tool = tool;
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST)
{
generator.Templates.RegisterRenderer(typeof(string), new RubyRenderer());
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, Template outputFileST)
{
if (!grammar.IsRoot)
{
Grammar rootGrammar = grammar.composite.RootGrammar;
string actionScope = grammar.GetDefaultActionScope(grammar.type);
IDictionary<string, object> actions;
object rootNamespace;
if (rootGrammar.Actions.TryGetValue(actionScope, out actions) && actions.TryGetValue("namespace", out rootNamespace))
{
if (!grammar.Actions.TryGetValue(actionScope, out actions))
{
actions = new Dictionary<string, object>();
grammar.Actions[actionScope] = actions;
}
actions["namespace"] = rootNamespace;
}
}
generator.Templates.RegisterRenderer(typeof(string), new StringRenderer(generator, this));
base.GenRecognizerFile(tool, generator, grammar, outputFileST);
}
/** Load a vocab file {@code <vocabName>.tokens} and return mapping. */
public virtual IDictionary <string, int> Load()
{
IDictionary <string, int> tokens = new LinkedHashMap <string, int>();
int maxTokenType = -1;
string fullFile = GetImportedVocabFile();
AntlrTool tool = g.tool;
string vocabName = g.GetOptionString("tokenVocab");
try {
Regex tokenDefPattern = new Regex("([^\n]+?)[ \\t]*?=[ \\t]*?([0-9]+)");
string[] lines;
if (tool.grammarEncoding != null)
{
lines = File.ReadAllLines(fullFile, Encoding.GetEncoding(tool.grammarEncoding));
}
else
{
lines = File.ReadAllLines(fullFile);
}
for (int i = 0; i < lines.Length; i++)
{
string tokenDef = lines[i];
int lineNum = i + 1;
Match matcher = tokenDefPattern.Match(tokenDef);
if (matcher.Success)
{
string tokenID = matcher.Groups[1].Value;
string tokenTypeS = matcher.Groups[2].Value;
int tokenType;
if (!int.TryParse(tokenTypeS, out tokenType))
{
tool.errMgr.ToolError(ErrorType.TOKENS_FILE_SYNTAX_ERROR,
vocabName + CodeGenerator.VOCAB_FILE_EXTENSION,
" bad token type: " + tokenTypeS,
lineNum);
tokenType = TokenTypes.Invalid;
}
tool.Log("grammar", "import " + tokenID + "=" + tokenType);
tokens[tokenID] = tokenType;
maxTokenType = Math.Max(maxTokenType, tokenType);
lineNum++;
}
else
{
if (tokenDef.Length > 0) // ignore blank lines
{
tool.errMgr.ToolError(ErrorType.TOKENS_FILE_SYNTAX_ERROR,
vocabName + CodeGenerator.VOCAB_FILE_EXTENSION,
" bad token def: " + tokenDef,
lineNum);
}
}
}
}
catch (FileNotFoundException) {
GrammarAST inTree = g.ast.GetOptionAST("tokenVocab");
string inTreeValue = inTree.Token.Text;
if (vocabName.Equals(inTreeValue))
{
tool.errMgr.GrammarError(ErrorType.CANNOT_FIND_TOKENS_FILE_REFD_IN_GRAMMAR,
g.fileName,
inTree.Token,
fullFile);
}
else // must be from -D option on cmd-line not token in tree
{
tool.errMgr.ToolError(ErrorType.CANNOT_FIND_TOKENS_FILE_GIVEN_ON_CMDLINE,
fullFile,
g.name);
}
}
catch (Exception e) {
tool.errMgr.ToolError(ErrorType.ERROR_READING_TOKENS_FILE,
e,
fullFile,
e.Message);
}
return(tokens);
}
public ToolANTLRLexer(ICharStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
public Grammar(AntlrTool tool, [NotNull] GrammarRootAST ast)
{
if (ast == null)
{
throw new ArgumentNullException(nameof(ast));
}
if (ast.tokenStream == null)
{
throw new ArgumentException("ast must have a token stream", nameof(ast));
}
this.tool = tool;
this.ast = ast;
this.name = (ast.GetChild(0)).Text;
this.tokenStream = ast.tokenStream;
this.originalTokenStream = this.tokenStream;
InitTokenSymbolTables();
}
/** For testing; builds trees, does sem anal */
public Grammar(string fileName, string grammarText, Grammar tokenVocabSource, [Nullable] ANTLRToolListener listener)
{
this.text = grammarText;
this.fileName = fileName;
this.tool = new AntlrTool();
this.tool.AddListener(listener);
Antlr.Runtime.ANTLRStringStream @in = new Antlr.Runtime.ANTLRStringStream(grammarText);
@in.name = fileName;
this.ast = tool.Parse(fileName, @in);
if (ast == null)
{
throw new NotSupportedException();
}
if (ast.tokenStream == null)
{
throw new InvalidOperationException("expected ast to have a token stream");
}
this.tokenStream = ast.tokenStream;
this.originalTokenStream = this.tokenStream;
// ensure each node has pointer to surrounding grammar
Antlr.Runtime.Tree.TreeVisitor v = new Antlr.Runtime.Tree.TreeVisitor(new GrammarASTAdaptor());
v.Visit(ast, new SetPointersAction(this));
InitTokenSymbolTables();
if (tokenVocabSource != null)
{
ImportVocab(tokenVocabSource);
}
tool.Process(this, false);
}
/** convenience method for Tool.loadGrammar() */
public static Grammar Load(string fileName)
{
AntlrTool antlr = new AntlrTool();
return antlr.LoadGrammar(fileName);
}
private static TemplateGroup LoadPrecRuleTemplates(AntlrTool tool)
{
string templateDirs = tool.TemplatesDirectory;
TemplateGroup group;
if (!recRuleTemplatesCache.TryGetValue(templateDirs, out group))
{
string fileName = CodeGenerator.FindTemplateFile(templateDirs.Split(':'), "LeftRecursiveRules.stg");
group = new TemplateGroupFile(fileName);
if (!group.IsDefined("recRuleName"))
{
recRuleTemplatesCache[templateDirs] = group;
}
else
{
ErrorManager.Error(ErrorManager.MSG_MISSING_CODE_GEN_TEMPLATES, "PrecRules");
return null;
}
}
return group;
}
public ToolANTLRParser(ITokenStream input, AntlrTool tool)
: base(input)
{
this.tool = tool;
}
