/** Given the grammar to which we are attached, walk the AST associated
* with that grammar to create NFAs. Then create the DFAs for all
* decision points in the grammar by converting the NFAs to DFAs.
* Finally, walk the AST again to generate code.
*
* Either 1 or 2 files are written:
*
* recognizer: the main parser/lexer/treewalker item
* header file: language like C/C++ need extern definitions
*
* The target, such as JavaTarget, dictates which files get written.
*/
public virtual StringTemplate GenRecognizer()
{
//[email protected]("### generate "+grammar.name+" recognizer");
// LOAD OUTPUT TEMPLATES
LoadTemplates( language );
if ( templates == null )
{
return null;
}
// CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA
if ( ErrorManager.DoNotAttemptAnalysis() )
{
return null;
}
target.PerformGrammarAnalysis( this, grammar );
// some grammar analysis errors will not yield reliable DFA
if ( ErrorManager.DoNotAttemptCodeGen() )
{
return null;
}
// OPTIMIZE DFA
DFAOptimizer optimizer = new DFAOptimizer( grammar );
optimizer.Optimize();
// OUTPUT FILE (contains recognizerST)
outputFileST = templates.GetInstanceOf( "outputFile" );
// HEADER FILE
if ( templates.IsDefined( "headerFile" ) )
{
headerFileST = templates.GetInstanceOf( "headerFile" );
}
else
{
// create a dummy to avoid null-checks all over code generator
headerFileST = new StringTemplate( templates, "" );
headerFileST.Name = "dummy-header-file";
}
bool filterMode = grammar.GetOption( "filter" ) != null &&
grammar.GetOption( "filter" ).Equals( "true" );
bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption ||
grammar.SyntacticPredicates != null ||
filterMode;
// TODO: move this down further because generating the recognizer
// alters the model with info on who uses predefined properties etc...
// The actions here might refer to something.
// The only two possible output files are available at this point.
// Verify action scopes are ok for target and dump actions into output
// Templates can say <actions.parser.header> for example.
var actions = grammar.Actions;
VerifyActionScopesOkForTarget( actions );
// translate $x::y references
TranslateActionAttributeReferences( actions );
StringTemplate gateST = templates.GetInstanceOf( "actionGate" );
if ( filterMode )
{
// if filtering, we need to set actions to execute at backtracking
// level 1 not 0.
gateST = templates.GetInstanceOf( "filteringActionGate" );
}
grammar.SetSynPredGateIfNotAlready( gateST );
headerFileST.SetAttribute( "actions", actions );
outputFileST.SetAttribute( "actions", actions );
headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
headerFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
outputFileST.SetAttribute( "backtracking", canBacktrack );
headerFileST.SetAttribute( "backtracking", canBacktrack );
// turn on memoize attribute at grammar level so we can create ruleMemo.
// each rule has memoize attr that hides this one, indicating whether
// it needs to save results
string memoize = (string)grammar.GetOption( "memoize" );
outputFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
headerFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
outputFileST.SetAttribute( "trace", trace );
headerFileST.SetAttribute( "trace", trace );
outputFileST.SetAttribute( "profile", profile );
headerFileST.SetAttribute( "profile", profile );
// RECOGNIZER
if ( grammar.type == GrammarType.Lexer )
{
recognizerST = templates.GetInstanceOf( "lexer" );
outputFileST.SetAttribute( "LEXER", true );
headerFileST.SetAttribute( "LEXER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
else if ( grammar.type == GrammarType.Parser ||
grammar.type == GrammarType.Combined )
{
recognizerST = templates.GetInstanceOf( "parser" );
outputFileST.SetAttribute( "PARSER", true );
headerFileST.SetAttribute( "PARSER", true );
}
else
{
recognizerST = templates.GetInstanceOf( "treeParser" );
outputFileST.SetAttribute( "TREE_PARSER", true );
headerFileST.SetAttribute( "TREE_PARSER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
outputFileST.SetAttribute( "recognizer", recognizerST );
headerFileST.SetAttribute( "recognizer", recognizerST );
outputFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
headerFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
string targetAppropriateFileNameString =
target.GetTargetStringLiteralFromString( grammar.FileName );
outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
{
// GENERATE RECOGNIZER
// Walk the AST holding the input grammar, this time generating code
// Decisions are generated by using the precomputed DFAs
// Fill in the various templates with data
CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) );
try
{
gen.grammar_( grammar,
recognizerST,
outputFileST,
headerFileST );
}
catch ( RecognitionException re )
{
ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE,
re );
}
}
GenTokenTypeConstants( recognizerST );
GenTokenTypeConstants( outputFileST );
GenTokenTypeConstants( headerFileST );
if ( grammar.type != GrammarType.Lexer )
{
GenTokenTypeNames( recognizerST );
GenTokenTypeNames( outputFileST );
GenTokenTypeNames( headerFileST );
}
// Now that we know what synpreds are used, we can set into template
HashSet<string> synpredNames = null;
if ( grammar.synPredNamesUsedInDFA.Count > 0 )
{
synpredNames = grammar.synPredNamesUsedInDFA;
}
outputFileST.SetAttribute( "synpreds", synpredNames );
headerFileST.SetAttribute( "synpreds", synpredNames );
// all recognizers can see Grammar object
recognizerST.SetAttribute( "grammar", grammar );
// WRITE FILES
try
{
target.GenRecognizerFile( tool, this, grammar, outputFileST );
if ( templates.IsDefined( "headerFile" ) )
{
StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" );
target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() );
}
// write out the vocab interchange file; used by antlr,
// does not change per target
StringTemplate tokenVocabSerialization = GenTokenVocabOutput();
string vocabFileName = VocabFileName;
if ( vocabFileName != null )
{
Write( tokenVocabSerialization, vocabFileName );
}
//[email protected](outputFileST.getDOTForDependencyGraph(false));
}
catch ( IOException ioe )
{
ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE,
VocabFileName,
ioe );
}
/*
[email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs);
[email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups);
*/
return outputFileST;
}
/** Given a template constructor action like %foo(a={...}) in
* an action, translate it to the appropriate template constructor
* from the templateLib. This translates a *piece* of the action.
*/
public virtual StringTemplate TranslateTemplateConstructor( string ruleName,
int outerAltNum,
IToken actionToken,
string templateActionText )
{
GrammarAST rewriteTree = null;
{
// first, parse with antlr.g
//[email protected]("translate template: "+templateActionText);
ANTLRLexer lexer = new ANTLRLexer( new Antlr.Runtime.ANTLRStringStream( templateActionText ) );
lexer.Filename = grammar.FileName;
//lexer.setTokenObjectClass( "antlr.TokenWithIndex" );
//TokenStreamRewriteEngine tokenBuffer = new TokenStreamRewriteEngine( lexer );
//tokenBuffer.discard( ANTLRParser.WS );
//tokenBuffer.discard( ANTLRParser.ML_COMMENT );
//tokenBuffer.discard( ANTLRParser.COMMENT );
//tokenBuffer.discard( ANTLRParser.SL_COMMENT );
ANTLRParser parser = new ANTLRParser( new Antlr.Runtime.CommonTokenStream( lexer ) );
parser.FileName = grammar.FileName;
//parser.setASTNodeClass( "org.antlr.tool.GrammarAST" );
try
{
ANTLRParser.rewrite_template_return result = parser.rewrite_template();
rewriteTree = (GrammarAST)result.Tree;
}
catch ( RecognitionException /*re*/ )
{
ErrorManager.GrammarError( ErrorManager.MSG_INVALID_TEMPLATE_ACTION,
grammar,
actionToken,
templateActionText );
}
catch ( Exception tse )
{
ErrorManager.InternalError( "can't parse template action", tse );
}
}
{
// then translate via codegen.g
CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( rewriteTree ) );
gen.Init( grammar );
gen.currentRuleName = ruleName;
gen.outerAltNum = outerAltNum;
StringTemplate st = null;
try
{
st = gen.rewrite_template();
}
catch ( RecognitionException re )
{
ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE,
re );
}
return st;
}
}
