/** Given the grammar to which we are attached, walk the AST associated * with that grammar to create NFAs. Then create the DFAs for all * decision points in the grammar by converting the NFAs to DFAs. * Finally, walk the AST again to generate code. * * Either 1 or 2 files are written: * * recognizer: the main parser/lexer/treewalker item * header file: language like C/C++ need extern definitions * * The target, such as JavaTarget, dictates which files get written. */ public virtual StringTemplate GenRecognizer() { //[email protected]("### generate "+grammar.name+" recognizer"); // LOAD OUTPUT TEMPLATES LoadTemplates( language ); if ( templates == null ) { return null; } // CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA if ( ErrorManager.DoNotAttemptAnalysis() ) { return null; } target.PerformGrammarAnalysis( this, grammar ); // some grammar analysis errors will not yield reliable DFA if ( ErrorManager.DoNotAttemptCodeGen() ) { return null; } // OPTIMIZE DFA DFAOptimizer optimizer = new DFAOptimizer( grammar ); optimizer.Optimize(); // OUTPUT FILE (contains recognizerST) outputFileST = templates.GetInstanceOf( "outputFile" ); // HEADER FILE if ( templates.IsDefined( "headerFile" ) ) { headerFileST = templates.GetInstanceOf( "headerFile" ); } else { // create a dummy to avoid null-checks all over code generator headerFileST = new StringTemplate( templates, "" ); headerFileST.Name = "dummy-header-file"; } bool filterMode = grammar.GetOption( "filter" ) != null && grammar.GetOption( "filter" ).Equals( "true" ); bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption || grammar.SyntacticPredicates != null || filterMode; // TODO: move this down further because generating the recognizer // alters the model with info on who uses predefined properties etc... // The actions here might refer to something. // The only two possible output files are available at this point. // Verify action scopes are ok for target and dump actions into output // Templates can say <actions.parser.header> for example. var actions = grammar.Actions; VerifyActionScopesOkForTarget( actions ); // translate $x::y references TranslateActionAttributeReferences( actions ); StringTemplate gateST = templates.GetInstanceOf( "actionGate" ); if ( filterMode ) { // if filtering, we need to set actions to execute at backtracking // level 1 not 0. gateST = templates.GetInstanceOf( "filteringActionGate" ); } grammar.SetSynPredGateIfNotAlready( gateST ); headerFileST.SetAttribute( "actions", actions ); outputFileST.SetAttribute( "actions", actions ); headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); headerFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); outputFileST.SetAttribute( "backtracking", canBacktrack ); headerFileST.SetAttribute( "backtracking", canBacktrack ); // turn on memoize attribute at grammar level so we can create ruleMemo. // each rule has memoize attr that hides this one, indicating whether // it needs to save results string memoize = (string)grammar.GetOption( "memoize" ); outputFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); headerFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); outputFileST.SetAttribute( "trace", trace ); headerFileST.SetAttribute( "trace", trace ); outputFileST.SetAttribute( "profile", profile ); headerFileST.SetAttribute( "profile", profile ); // RECOGNIZER if ( grammar.type == GrammarType.Lexer ) { recognizerST = templates.GetInstanceOf( "lexer" ); outputFileST.SetAttribute( "LEXER", true ); headerFileST.SetAttribute( "LEXER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } else if ( grammar.type == GrammarType.Parser || grammar.type == GrammarType.Combined ) { recognizerST = templates.GetInstanceOf( "parser" ); outputFileST.SetAttribute( "PARSER", true ); headerFileST.SetAttribute( "PARSER", true ); } else { recognizerST = templates.GetInstanceOf( "treeParser" ); outputFileST.SetAttribute( "TREE_PARSER", true ); headerFileST.SetAttribute( "TREE_PARSER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } outputFileST.SetAttribute( "recognizer", recognizerST ); headerFileST.SetAttribute( "recognizer", recognizerST ); outputFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); headerFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); string targetAppropriateFileNameString = target.GetTargetStringLiteralFromString( grammar.FileName ); outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); { // GENERATE RECOGNIZER // Walk the AST holding the input grammar, this time generating code // Decisions are generated by using the precomputed DFAs // Fill in the various templates with data CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) ); try { gen.grammar_( grammar, recognizerST, outputFileST, headerFileST ); } catch ( RecognitionException re ) { ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE, re ); } } GenTokenTypeConstants( recognizerST ); GenTokenTypeConstants( outputFileST ); GenTokenTypeConstants( headerFileST ); if ( grammar.type != GrammarType.Lexer ) { GenTokenTypeNames( recognizerST ); GenTokenTypeNames( outputFileST ); GenTokenTypeNames( headerFileST ); } // Now that we know what synpreds are used, we can set into template HashSet<string> synpredNames = null; if ( grammar.synPredNamesUsedInDFA.Count > 0 ) { synpredNames = grammar.synPredNamesUsedInDFA; } outputFileST.SetAttribute( "synpreds", synpredNames ); headerFileST.SetAttribute( "synpreds", synpredNames ); // all recognizers can see Grammar object recognizerST.SetAttribute( "grammar", grammar ); // WRITE FILES try { target.GenRecognizerFile( tool, this, grammar, outputFileST ); if ( templates.IsDefined( "headerFile" ) ) { StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" ); target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() ); } // write out the vocab interchange file; used by antlr, // does not change per target StringTemplate tokenVocabSerialization = GenTokenVocabOutput(); string vocabFileName = VocabFileName; if ( vocabFileName != null ) { Write( tokenVocabSerialization, vocabFileName ); } //[email protected](outputFileST.getDOTForDependencyGraph(false)); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE, VocabFileName, ioe ); } /* [email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs); [email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups); */ return outputFileST; }
/** Given a template constructor action like %foo(a={...}) in * an action, translate it to the appropriate template constructor * from the templateLib. This translates a *piece* of the action. */ public virtual StringTemplate TranslateTemplateConstructor( string ruleName, int outerAltNum, IToken actionToken, string templateActionText ) { GrammarAST rewriteTree = null; { // first, parse with antlr.g //[email protected]("translate template: "+templateActionText); ANTLRLexer lexer = new ANTLRLexer( new Antlr.Runtime.ANTLRStringStream( templateActionText ) ); lexer.Filename = grammar.FileName; //lexer.setTokenObjectClass( "antlr.TokenWithIndex" ); //TokenStreamRewriteEngine tokenBuffer = new TokenStreamRewriteEngine( lexer ); //tokenBuffer.discard( ANTLRParser.WS ); //tokenBuffer.discard( ANTLRParser.ML_COMMENT ); //tokenBuffer.discard( ANTLRParser.COMMENT ); //tokenBuffer.discard( ANTLRParser.SL_COMMENT ); ANTLRParser parser = new ANTLRParser( new Antlr.Runtime.CommonTokenStream( lexer ) ); parser.FileName = grammar.FileName; //parser.setASTNodeClass( "org.antlr.tool.GrammarAST" ); try { ANTLRParser.rewrite_template_return result = parser.rewrite_template(); rewriteTree = (GrammarAST)result.Tree; } catch ( RecognitionException /*re*/ ) { ErrorManager.GrammarError( ErrorManager.MSG_INVALID_TEMPLATE_ACTION, grammar, actionToken, templateActionText ); } catch ( Exception tse ) { ErrorManager.InternalError( "can't parse template action", tse ); } } { // then translate via codegen.g CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( rewriteTree ) ); gen.Init( grammar ); gen.currentRuleName = ruleName; gen.outerAltNum = outerAltNum; StringTemplate st = null; try { st = gen.rewrite_template(); } catch ( RecognitionException re ) { ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE, re ); } return st; } }