public string Convert(Model.TemplateSource.PropertyModel source) { string result = ""; string templateString = sourceStructure.GetPropertyTemplate(source.TypeName); Antlr3.ST.StringTemplate template = new Antlr3.ST.StringTemplate(); template.Template = templateString; template.SetAttribute("property", source); result = template.ToString(); return result; }
public void TestNewlineLiterals() /*throws Exception*/ { Grammar g = new Grammar( "lexer grammar T;\n" + "A : '\\n\\n' ;\n" // ANTLR sees '\n\n' ); string expecting = "match(\"\\n\\n\")"; AntlrTool antlr = newTool(); antlr.SetOutputDirectory(null); // write to /dev/null CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); g.CodeGenerator = generator; generator.GenRecognizer(); // codegen phase sets some vars we need StringTemplate codeST = generator.RecognizerST; string code = codeST.ToString(); int m = code.IndexOf("match(\""); string found = code.Substring(m, expecting.Length); assertEquals(expecting, found); }
public void TestIndirectTemplateConstructor() /*throws Exception*/ { string action = "x = %({\"foo\"})(name={$ID.text});"; string expecting = "x = templateLib.getInstanceOf(\"foo\"," + LINE_SEP + " new STAttrMap().put(\"name\", (ID1!=null?ID1.getText():null)));"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener(equeue); Grammar g = new Grammar( "grammar t;\n" + "options {\n" + " output=template;\n" + "}\n" + "\n" + "a : ID {" + action + "}\n" + " ;\n" + "\n" + "ID : 'a';\n"); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator(generator, "a", new CommonToken(ANTLRParser.ACTION, action), 1); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup(".", typeof(AngleBracketTemplateLexer)); StringTemplate actionST = new StringTemplate(templates, rawTranslation); string found = actionST.ToString(); assertNoErrors(equeue); assertEquals(expecting, found); }
public void TestSetAttr() /*throws Exception*/ { string action = "%x.y = z;"; string expecting = "(x).setAttribute(\"y\", z);"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener(equeue); Grammar g = new Grammar( "grammar t;\n" + "options {\n" + " output=template;\n" + "}\n" + "\n" + "a : ID {" + action + "}\n" + " ;\n" + "\n" + "ID : 'a';\n"); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator(generator, "a", new CommonToken(ANTLRParser.ACTION, action), 1); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup(".", typeof(AngleBracketTemplateLexer)); StringTemplate actionST = new StringTemplate(templates, rawTranslation); string found = actionST.ToString(); assertNoErrors(equeue); assertEquals(expecting, found); }
public override String ToString() { line = 0; charPositionInLine = 0; if (offendingToken != null) { line = offendingToken.Line; charPositionInLine = offendingToken.CharPositionInLine; } // TODO: actually set the right Grammar instance to get the filename // TODO: have to update all v2 grammar files for this. or use errormanager and tool to get the current grammar if (g != null) { file = g.FileName; } StringTemplate st = GetMessageTemplate(); if (arg != null) { st.SetAttribute("arg", arg); } return(base.ToString(st)); }
public override String ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; String fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); if (probe.dfa.IsTokensRuleDecision) { // alts are token rules, convert to the names instead of numbers for (int i = 0; i < alts.Length; i++) { int altI = alts[i]; String tokenName = probe.GetTokenNameForTokensRuleAlt(altI); // reset the line/col to the token definition NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName); line = ruleStart.associatedASTNode.Line; charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine; st.SetAttribute("tokens", tokenName); } } else { // regular alt numbers, show the alts st.SetAttribute("alts", alts); } return(base.ToString(st)); }
public void TestRefToRuleWithNoReturnValue() /*throws Exception*/ { ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener(equeue); string grammarStr = "grammar P;\n" + "a : x=b ;\n" + "b : B ;\n" + "B : 'b' ;\n"; Grammar g = new Grammar(grammarStr); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); g.CodeGenerator = generator; StringTemplate recogST = generator.GenRecognizer(); string code = recogST.ToString(); assertTrue("not expecting label", code.IndexOf("x=b();") < 0); assertEquals("unexpected errors: " + equeue, 0, equeue.errors.Count); }
public virtual string ToString(StringTemplate messageST) { // setup the location locationST = ErrorManager.GetLocationFormat(); reportST = ErrorManager.GetReportFormat(); messageFormatST = ErrorManager.GetMessageFormat(); bool locationValid = false; if (line != -1) { locationST.SetAttribute("line", line); locationValid = true; } if (charPositionInLine != -1) { locationST.SetAttribute("column", charPositionInLine + 1); locationValid = true; } if (file != null) { locationST.SetAttribute("file", file); locationValid = true; } messageFormatST.SetAttribute("id", msgID); messageFormatST.SetAttribute("text", messageST); if (locationValid) { reportST.SetAttribute("location", locationST); } reportST.SetAttribute("message", messageFormatST); reportST.SetAttribute("type", ErrorManager.GetMessageType(msgID)); return(reportST.ToString()); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } var labels = probe.GetSampleNonDeterministicInputSequence(problemState); string input = probe.GetInputSequenceDisplay(labels); StringTemplate st = GetMessageTemplate(); List <int> alts = new List <int>(); alts.addAll(problemState.AltSet); alts.Sort(); st.SetAttribute("danglingAlts", alts); st.SetAttribute("input", input); return(base.ToString(st)); }
protected override void GenRecognizerHeaderFile( AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate headerFileST, string extName ) { StringTemplateGroup templates = generator.Templates; generator.Write( headerFileST, grammar.name + extName ); }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. We know this is an NFA * for a rule so don't traverse edges to other rules and * don't go past rule end state. */ protected virtual void WalkRuleNFACreatingDOT(StringTemplate dot, State s) { if (markedStates.Contains(s.stateNumber)) { return; // already visited this node } markedStates.Add(s.stateNumber); // mark this node as completed. // first add this node StringTemplate stateST; if (s.IsAcceptState) { stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate")); } else { stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state")); } stateST.SetAttribute("name", GetStateLabel(s)); dot.SetAttribute("states", stateST); if (s.IsAcceptState) { return; // don't go past end of rule node to the follow states } // special case: if decision point, then line up the alt start states // unless it's an end of block if (((NFAState)s).IsDecisionState) { GrammarAST n = ((NFAState)s).associatedASTNode; if (n != null && n.Type != ANTLRParser.EOB) { StringTemplate rankST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "decision-rank")); NFAState alt = (NFAState)s; while (alt != null) { rankST.SetAttribute("states", GetStateLabel(alt)); if (alt.transition[1] != null) { alt = (NFAState)alt.transition[1].target; } else { alt = null; } } dot.SetAttribute("decisionRanks", rankST); } } // make a DOT edge for each transition StringTemplate edgeST = null; for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = (Transition)s.GetTransition(i); if (edge is RuleClosureTransition) { RuleClosureTransition rr = ((RuleClosureTransition)edge); // don't jump to other rules, but display edge to follow node edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge")); if (rr.rule.grammar != grammar) { edgeST.SetAttribute("label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">"); } else { edgeST.SetAttribute("label", "<" + rr.rule.Name + ">"); } edgeST.SetAttribute("src", GetStateLabel(s)); edgeST.SetAttribute("target", GetStateLabel(rr.followState)); edgeST.SetAttribute("arrowhead", arrowhead); dot.SetAttribute("edges", edgeST); WalkRuleNFACreatingDOT(dot, rr.followState); continue; } if (edge.IsAction) { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "action-edge")); } else if (edge.IsEpsilon) { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "epsilon-edge")); } else { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge")); } edgeST.SetAttribute("label", GetEdgeLabel(edge)); edgeST.SetAttribute("src", GetStateLabel(s)); edgeST.SetAttribute("target", GetStateLabel(edge.target)); edgeST.SetAttribute("arrowhead", arrowhead); dot.SetAttribute("edges", edgeST); WalkRuleNFACreatingDOT(dot, edge.target); // keep walkin' } }
/** Generate a token vocab file with all the token names/types. For example: * ID=7 * FOR=8 * 'for'=8 * * This is independent of the target language; used by antlr internally */ protected virtual StringTemplate GenTokenVocabOutput() { StringTemplate vocabFileST = new StringTemplate( vocabFilePattern, typeof( AngleBracketTemplateLexer ) ); vocabFileST.Name = "vocab-file"; // make constants for the token names foreach ( string tokenID in grammar.TokenIDs ) { int tokenType = grammar.GetTokenType( tokenID ); if ( tokenType >= Label.MIN_TOKEN_TYPE ) { vocabFileST.SetAttribute( "tokens.{name,type}", tokenID, tokenType ); } } // now dump the strings foreach ( string literal in grammar.StringLiterals ) { int tokenType = grammar.GetTokenType( literal ); if ( tokenType >= Label.MIN_TOKEN_TYPE ) { vocabFileST.SetAttribute( "tokens.{name,type}", literal, tokenType ); } } return vocabFileST; }
/** Given the grammar to which we are attached, walk the AST associated * with that grammar to create NFAs. Then create the DFAs for all * decision points in the grammar by converting the NFAs to DFAs. * Finally, walk the AST again to generate code. * * Either 1 or 2 files are written: * * recognizer: the main parser/lexer/treewalker item * header file: language like C/C++ need extern definitions * * The target, such as JavaTarget, dictates which files get written. */ public virtual StringTemplate GenRecognizer() { //[email protected]("### generate "+grammar.name+" recognizer"); // LOAD OUTPUT TEMPLATES LoadTemplates( language ); if ( templates == null ) { return null; } // CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA if ( ErrorManager.DoNotAttemptAnalysis() ) { return null; } target.PerformGrammarAnalysis( this, grammar ); // some grammar analysis errors will not yield reliable DFA if ( ErrorManager.DoNotAttemptCodeGen() ) { return null; } // OPTIMIZE DFA DFAOptimizer optimizer = new DFAOptimizer( grammar ); optimizer.Optimize(); // OUTPUT FILE (contains recognizerST) outputFileST = templates.GetInstanceOf( "outputFile" ); // HEADER FILE if ( templates.IsDefined( "headerFile" ) ) { headerFileST = templates.GetInstanceOf( "headerFile" ); } else { // create a dummy to avoid null-checks all over code generator headerFileST = new StringTemplate( templates, "" ); headerFileST.Name = "dummy-header-file"; } bool filterMode = grammar.GetOption( "filter" ) != null && grammar.GetOption( "filter" ).Equals( "true" ); bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption || grammar.SyntacticPredicates != null || filterMode; // TODO: move this down further because generating the recognizer // alters the model with info on who uses predefined properties etc... // The actions here might refer to something. // The only two possible output files are available at this point. // Verify action scopes are ok for target and dump actions into output // Templates can say <actions.parser.header> for example. var actions = grammar.Actions; VerifyActionScopesOkForTarget( actions ); // translate $x::y references TranslateActionAttributeReferences( actions ); StringTemplate gateST = templates.GetInstanceOf( "actionGate" ); if ( filterMode ) { // if filtering, we need to set actions to execute at backtracking // level 1 not 0. gateST = templates.GetInstanceOf( "filteringActionGate" ); } grammar.SetSynPredGateIfNotAlready( gateST ); headerFileST.SetAttribute( "actions", actions ); outputFileST.SetAttribute( "actions", actions ); headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); headerFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); outputFileST.SetAttribute( "backtracking", canBacktrack ); headerFileST.SetAttribute( "backtracking", canBacktrack ); // turn on memoize attribute at grammar level so we can create ruleMemo. // each rule has memoize attr that hides this one, indicating whether // it needs to save results string memoize = (string)grammar.GetOption( "memoize" ); outputFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); headerFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); outputFileST.SetAttribute( "trace", trace ); headerFileST.SetAttribute( "trace", trace ); outputFileST.SetAttribute( "profile", profile ); headerFileST.SetAttribute( "profile", profile ); // RECOGNIZER if ( grammar.type == GrammarType.Lexer ) { recognizerST = templates.GetInstanceOf( "lexer" ); outputFileST.SetAttribute( "LEXER", true ); headerFileST.SetAttribute( "LEXER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } else if ( grammar.type == GrammarType.Parser || grammar.type == GrammarType.Combined ) { recognizerST = templates.GetInstanceOf( "parser" ); outputFileST.SetAttribute( "PARSER", true ); headerFileST.SetAttribute( "PARSER", true ); } else { recognizerST = templates.GetInstanceOf( "treeParser" ); outputFileST.SetAttribute( "TREE_PARSER", true ); headerFileST.SetAttribute( "TREE_PARSER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } outputFileST.SetAttribute( "recognizer", recognizerST ); headerFileST.SetAttribute( "recognizer", recognizerST ); outputFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); headerFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); string targetAppropriateFileNameString = target.GetTargetStringLiteralFromString( grammar.FileName ); outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); { // GENERATE RECOGNIZER // Walk the AST holding the input grammar, this time generating code // Decisions are generated by using the precomputed DFAs // Fill in the various templates with data CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) ); try { gen.grammar_( grammar, recognizerST, outputFileST, headerFileST ); } catch ( RecognitionException re ) { ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE, re ); } } GenTokenTypeConstants( recognizerST ); GenTokenTypeConstants( outputFileST ); GenTokenTypeConstants( headerFileST ); if ( grammar.type != GrammarType.Lexer ) { GenTokenTypeNames( recognizerST ); GenTokenTypeNames( outputFileST ); GenTokenTypeNames( headerFileST ); } // Now that we know what synpreds are used, we can set into template HashSet<string> synpredNames = null; if ( grammar.synPredNamesUsedInDFA.Count > 0 ) { synpredNames = grammar.synPredNamesUsedInDFA; } outputFileST.SetAttribute( "synpreds", synpredNames ); headerFileST.SetAttribute( "synpreds", synpredNames ); // all recognizers can see Grammar object recognizerST.SetAttribute( "grammar", grammar ); // WRITE FILES try { target.GenRecognizerFile( tool, this, grammar, outputFileST ); if ( templates.IsDefined( "headerFile" ) ) { StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" ); target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() ); } // write out the vocab interchange file; used by antlr, // does not change per target StringTemplate tokenVocabSerialization = GenTokenVocabOutput(); string vocabFileName = VocabFileName; if ( vocabFileName != null ) { Write( tokenVocabSerialization, vocabFileName ); } //[email protected](outputFileST.getDOTForDependencyGraph(false)); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE, VocabFileName, ioe ); } /* [email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs); [email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups); */ return outputFileST; }
/** Translate an action like [3,"foo",a[3]] and return a List of the * translated actions. Because actions are themselves translated to a list * of chunks, must cat together into a StringTemplate>. Don't translate * to strings early as we need to eval templates in context. */ public virtual List<StringTemplate> TranslateArgAction( string ruleName, GrammarAST actionTree ) { string actionText = actionTree.Token.Text; List<string> args = GetListOfArgumentsFromAction( actionText, ',' ); List<StringTemplate> translatedArgs = new List<StringTemplate>(); foreach ( string arg in args ) { if ( arg != null ) { IToken actionToken = new CommonToken( ANTLRParser.ACTION, arg ); ActionTranslator translator = new ActionTranslator( this, ruleName, actionToken, actionTree.outerAltNum ); IList chunks = translator.TranslateToChunks(); chunks = target.PostProcessAction( chunks, actionToken ); StringTemplate catST = new StringTemplate( templates, "<chunks>" ); catST.SetAttribute( "chunks", chunks ); templates.CreateStringTemplate(); translatedArgs.Add( catST ); } } if ( translatedArgs.Count == 0 ) { return null; } return translatedArgs; }
// T O K E N D E F I N I T I O N G E N E R A T I O N /** Set attributes tokens and literals attributes in the incoming * code template. This is not the token vocab interchange file, but * rather a list of token type ID needed by the recognizer. */ protected virtual void GenTokenTypeConstants( StringTemplate code ) { // make constants for the token types foreach ( string tokenID in grammar.TokenIDs ) { int tokenType = grammar.GetTokenType( tokenID ); if ( tokenType == Label.EOF || tokenType >= Label.MIN_TOKEN_TYPE ) { // don't do FAUX labels 'cept EOF code.SetAttribute( "tokens.{name,type}", tokenID, tokenType ); } } }
public void TestArguments() { string action = "$i; $i.x; $u; $u.x"; string expecting = "i; i.x; u; u.x"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "parser grammar t;\n" + "a[User u, int i]\n" + " : {" + action + "}\n" + " ;" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestEscapedLessThanInAction() { Grammar g = new Grammar(); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); string action = "i<3; '<xmltag>'"; ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 0 ); string expecting = action; string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, "<action>" ); actionST.SetAttribute( "action", rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); }
public void TestTokenRefTreeProperty() { string action = "$ID.tree;"; string expecting = "ID1_tree;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "a : ID {" + action + "} ;" + "ID : 'a';\n" ); AntlrTool antlr = newTool(); antlr.SetOutputDirectory( null ); // write to /dev/null CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); }
public void TestUnknownGlobalDynamicAttribute() { string action = "$Symbols::x"; string expecting = action; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "scope Symbols {\n" + " int n;\n" + "}\n" + "a : {'+action+'}\n" + " ;\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); int expectedMsgID = ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE_ATTRIBUTE; object expectedArg = "Symbols"; object expectedArg2 = "x"; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 ); checkError( equeue, expectedMessage ); }
public void TestTokenLabels() { string action = "$id; $f; $id.text; $id.getText(); $id.dork " + "$id.type; $id.line; $id.pos; " + "$id.channel; $id.index;"; string expecting = "id; f; (id!=null?id.getText():null); id.getText(); id.dork (id!=null?id.getType():0); (id!=null?id.getLine():0); (id!=null?id.getCharPositionInLine():0); (id!=null?id.getChannel():0); (id!=null?id.getTokenIndex():0);"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "parser grammar t;\n" + "a : id=ID f=FLOAT {" + action + "}\n" + " ;" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestTokenLabelFromMultipleAlts() { string action = "$ID.text;"; // must be qualified string action2 = "$INT.text;"; // must be qualified string expecting = "(ID1!=null?ID1.getText():null);"; string expecting2 = "(INT2!=null?INT2.getText():null);"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "a : ID {" + action + "}\n" + " | INT {" + action2 + "}\n" + " ;\n" + "ID : 'a';\n" + "INT : '0';\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action2 ), 2 ); rawTranslation = translator.Translate(); templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); actionST = new StringTemplate( templates, rawTranslation ); found = actionST.ToString(); assertEquals( expecting2, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestSimplePlusEqualLabel() { string action = "$ids.size();"; // must be qualified string expecting = "list_ids.size();"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "parser grammar t;\n" + "a : ids+=ID ( COMMA ids+=ID {" + action + "})* ;\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST) { /* * Below is an experimental attempt at providing a few named action blocks * that are printed in both lexer and parser files from combined grammars. * ANTLR appears to first generate a parser, then generate an independent lexer, * and then generate code from that. It keeps the combo/parser grammar object * and the lexer grammar object, as well as their respective code generator and * target instances, completely independent. So, while a bit hack-ish, this is * a solution that should work without having to modify Terrence Parr's * core tool code. * * - sharedActionBlocks is a class variable containing a hash map * - if this method is called with a combo grammar, and the action map * in the grammar contains an entry for the named scope "all", * add an entry to sharedActionBlocks mapping the grammar name to * the "all" action map. * - if this method is called with an `implicit lexer' * (one that's extracted from a combo grammar), check to see if * there's an entry in sharedActionBlocks for the lexer's grammar name. * - if there is an action map entry, place it in the lexer's action map * - the recognizerFile template has code to place the * "all" actions appropriately * * problems: * - This solution assumes that the parser will be generated * before the lexer. If that changes at some point, this will * not work. * - I have not investigated how this works with delegation yet * * Kyle Yetter - March 25, 2010 */ if (grammar.type == GrammarType.Combined) { IDictionary <string, object> all; if (grammar.Actions.TryGetValue("all", out all)) { sharedActionBlocks[grammar.name] = all; } } else if (grammar.implicitLexer) { IDictionary <string, object> shared; if (sharedActionBlocks.TryGetValue(grammar.name, out shared)) { grammar.Actions["all"] = shared; } } generator.Templates.RegisterRenderer(typeof(string), new RubyRenderer()); string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type); generator.Write(outputFileST, fileName); }
public void TestUnqualifiedRuleScopeAttribute() { string action = "$n;"; // must be qualified string expecting = "$n;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "a\n" + "scope {\n" + " int n;\n" + "} : b\n" + " ;\n" + "b : {'+action+'}\n" + " ;\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); ActionTranslator translator = new ActionTranslator( generator, "b", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); int expectedMsgID = ErrorManager.MSG_UNKNOWN_SIMPLE_ATTRIBUTE; object expectedArg = "n"; object expectedArg2 = null; GrammarSemanticsMessage expectedMessage = new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 ); checkError( equeue, expectedMessage ); }
public void TestSharedGlobalScope() { string action = "$Symbols::x;"; string expecting = "((Symbols_scope)Symbols_stack.peek()).x;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "scope Symbols {\n" + " String x;\n" + "}\n" + "a\n" + "scope { int y; }\n" + "scope Symbols;\n" + " : b {" + action + "}\n" + " ;\n" + "b : ID {$Symbols::x=$ID.text} ;\n" + "ID : 'a';\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestAssignToOwnRulenameAttr() { string action = "$rule.tree = null;"; string expecting = "retval.tree = null;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar a;\n" + "rule\n" + " : 'y' {" + action + "}\n" + " ;" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "rule", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestEscaped_InAction() { string action = "int \\$n; \"\\$in string\\$\""; string expecting = "int $n; \"$in string$\""; Grammar g = new Grammar( "parser grammar t;\n" + "@members {" + action + "}\n" + "a[User u, int i]\n" + " : {" + action + "}\n" + " ;" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 0 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); }
public void TestAssignToTreeNodeAttribute() { string action = "$tree.scope = localScope;"; string expecting = "(()retval.tree).scope = localScope;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar a;\n" + "options { output=AST; }" + "rule\n" + "@init {\n" + " Scope localScope=null;\n" + "}\n" + "@after {\n" + " $tree.scope = localScope;\n" + "}\n" + " : 'a' -> ^('a')\n" + ";" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "rule", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); assertEquals( expecting, found ); }
public virtual void Write( StringTemplate code, string fileName ) { Stopwatch watch = Stopwatch.StartNew(); TextWriter w = tool.GetOutputFile( grammar, fileName ); // Write the output to a StringWriter IStringTemplateWriter wr = templates.GetStringTemplateWriter( w ); wr.SetLineWidth( lineWidth ); code.Write( wr ); w.Close(); TimeSpan duration = watch.Elapsed; //[email protected]("render time for "+fileName+": "+(int)(stop-start)+"ms"); }
public void TestBasicGlobalScope() { string action = "$Symbols::names.add($id.text);"; string expecting = "((Symbols_scope)Symbols_stack.peek()).names.add((id!=null?id.getText():null));"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "scope Symbols {\n" + " int n;\n" + " List names;\n" + "}\n" + "a scope Symbols; : (id=ID ';' {" + action + "} )+\n" + " ;\n" + "ID : 'a';\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
/** Generate a token names table that maps token type to a printable * name: either the label like INT or the literal like "begin". */ protected virtual void GenTokenTypeNames( StringTemplate code ) { for ( int t = Label.MIN_TOKEN_TYPE; t <= grammar.MaxTokenType; t++ ) { string tokenName = grammar.GetTokenDisplayName( t ); if ( tokenName != null ) { tokenName = target.GetTargetStringLiteralFromString( tokenName, true ); code.SetAttribute( "tokenNames", tokenName ); } } }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. */ protected virtual void WalkCreatingDFADOT( StringTemplate dot, DFAState s ) { if ( markedStates.Contains( s.stateNumber ) ) { return; // already visited this node } markedStates.Add( s.stateNumber ); // mark this node as completed. // first add this node StringTemplate st; if ( s.IsAcceptState ) { st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) ); } else { st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) ); } st.SetAttribute( "name", GetStateLabel( s ) ); dot.SetAttribute( "states", st ); // make a DOT edge for each transition for ( int i = 0; i < s.NumberOfTransitions; i++ ) { Transition edge = (Transition)s.Transition( i ); //Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s" // + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions ); if ( StripNonreducedStates ) { if ( edge.target is DFAState && ( (DFAState)edge.target ).AcceptStateReachable != DFA.REACHABLE_YES ) { continue; // don't generate nodes for terminal states } } st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); st.SetAttribute( "label", GetEdgeLabel( edge ) ); st.SetAttribute( "src", GetStateLabel( s ) ); st.SetAttribute( "target", GetStateLabel( edge.target ) ); st.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", st ); WalkCreatingDFADOT( dot, (DFAState)edge.target ); // keep walkin' } }
// L O O K A H E A D D E C I S I O N G E N E R A T I O N /** Generate code that computes the predicted alt given a DFA. The * recognizerST can be either the main generated recognizerTemplate * for storage in the main parser file or a separate file. It's up to * the code that ultimately invokes the codegen.g grammar rule. * * Regardless, the output file and header file get a copy of the DFAs. */ public virtual StringTemplate GenLookaheadDecision( StringTemplate recognizerST, DFA dfa ) { StringTemplate decisionST; // If we are doing inline DFA and this one is acyclic and LL(*) // I have to check for is-non-LL(*) because if non-LL(*) the cyclic // check is not done by DFA.verify(); that is, verify() avoids // doesStateReachAcceptState() if non-LL(*) if ( dfa.CanInlineDecision ) { decisionST = acyclicDFAGenerator.GenFixedLookaheadDecision( Templates, dfa ); } else { // generate any kind of DFA here (cyclic or acyclic) dfa.CreateStateTables( this ); outputFileST.SetAttribute( "cyclicDFAs", dfa ); headerFileST.SetAttribute( "cyclicDFAs", dfa ); decisionST = templates.GetInstanceOf( "dfaDecision" ); string description = dfa.NFADecisionStartState.Description; description = target.GetTargetStringLiteralFromString( description ); if ( description != null ) { decisionST.SetAttribute( "description", description ); } decisionST.SetAttribute( "decisionNumber", dfa.DecisionNumber ); } return decisionST; }
/** From T.g return a list of File objects that * name files ANTLR will emit from T.g. */ public virtual IList <string> GetGeneratedFileList() { List <FileInfo> files = new List <FileInfo>(); System.IO.DirectoryInfo outputDir = tool.GetOutputDirectory(grammarFileName); if (outputDir.Name.Equals(".")) { outputDir = null; } else if (outputDir.Name.IndexOf(' ') >= 0) { // has spaces? string escSpaces = outputDir.ToString().Replace( " ", "\\ "); outputDir = new System.IO.DirectoryInfo(escSpaces); } // add generated recognizer; e.g., TParser.java string recognizer = generator.GetRecognizerFileName(grammar.name, grammar.type); files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, recognizer))); // add output vocab file; e.g., T.tokens. This is always generated to // the base output directory, which will be just . if there is no -o option // files.Add(new FileInfo(System.IO.Path.Combine(tool.OutputDirectory, generator.VocabFileName))); // are we generating a .h file? StringTemplate headerExtST = null; StringTemplate extST = generator.Templates.GetInstanceOf("codeFileExtension"); if (generator.Templates.IsDefined("headerFile")) { headerExtST = generator.Templates.GetInstanceOf("headerFileExtension"); string suffix = Grammar.grammarTypeToFileNameSuffix[(int)grammar.type]; string fileName = grammar.name + suffix + headerExtST.ToString(); files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, fileName))); } if (grammar.type == GrammarType.Combined) { // add autogenerated lexer; e.g., TLexer.java TLexer.h TLexer.tokens // don't add T__.g (just a temp file) string suffix = Grammar.grammarTypeToFileNameSuffix[(int)GrammarType.Lexer]; string lexer = grammar.name + suffix + extST.ToString(); files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, lexer))); // TLexer.h if (headerExtST != null) { string header = grammar.name + suffix + headerExtST.ToString(); files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, header))); } // for combined, don't generate TLexer.tokens } // handle generated files for imported grammars IList <Grammar> imports = grammar.composite.GetDelegates(grammar.composite.RootGrammar); foreach (Grammar g in imports) { outputDir = tool.GetOutputDirectory(g.FileName); string fname = GroomQualifiedFileName(outputDir.ToString(), g.GetRecognizerName() + extST.ToString()); files.Add(new FileInfo(fname)); } if (files.Count == 0) { return(null); } return(files.Select(info => info.FullName).ToArray()); }
/** For intervals such as [3..3, 30..35], generate an expression that * tests the lookahead similar to LA(1)==3 || (LA(1)>=30&&LA(1)<=35) */ public virtual StringTemplate GenSetExpr( StringTemplateGroup templates, IIntSet set, int k, bool partOfDFA ) { if ( !( set is IntervalSet ) ) { throw new ArgumentException( "unable to generate expressions for non IntervalSet objects" ); } IntervalSet iset = (IntervalSet)set; if ( iset.Intervals == null || iset.Intervals.Count == 0 ) { StringTemplate emptyST = new StringTemplate( templates, "" ); emptyST.Name = "empty-set-expr"; return emptyST; } string testSTName = "lookaheadTest"; string testRangeSTName = "lookaheadRangeTest"; if ( !partOfDFA ) { testSTName = "isolatedLookaheadTest"; testRangeSTName = "isolatedLookaheadRangeTest"; } StringTemplate setST = templates.GetInstanceOf( "setTest" ); int rangeNumber = 1; foreach ( Interval I in iset.GetIntervals() ) { int a = I.a; int b = I.b; StringTemplate eST; if ( a == b ) { eST = templates.GetInstanceOf( testSTName ); eST.SetAttribute( "atom", GetTokenTypeAsTargetLabel( a ) ); eST.SetAttribute( "atomAsInt", a ); //eST.setAttribute("k",Utils.integer(k)); } else { eST = templates.GetInstanceOf( testRangeSTName ); eST.SetAttribute( "lower", GetTokenTypeAsTargetLabel( a ) ); eST.SetAttribute( "lowerAsInt", a ); eST.SetAttribute( "upper", GetTokenTypeAsTargetLabel( b ) ); eST.SetAttribute( "upperAsInt", b ); eST.SetAttribute( "rangeNumber", rangeNumber ); } eST.SetAttribute( "k", k ); setST.SetAttribute( "ranges", eST ); rangeNumber++; } return setST; }
public void TestDoNotTranslateAttributeCompare() { string action = "$a.line == $b.line"; string expecting = "(a!=null?a.getLine():0) == (b!=null?b.getLine():0)"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "lexer grammar a;\n" + "RULE:\n" + " a=ID b=ID {" + action + "}" + " ;\n" + "ID : 'id';" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); ActionTranslator translator = new ActionTranslator( generator, "RULE", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); assertEquals( expecting, found ); }
protected virtual StringTemplate WalkFixedDFAGeneratingStateMachine( StringTemplateGroup templates, DFA dfa, DFAState s, int k) { //System.Console.Out.WriteLine( "walk " + s.stateNumber + " in dfa for decision " + dfa.decisionNumber ); if (s.IsAcceptState) { StringTemplate dfaST2 = templates.GetInstanceOf("dfaAcceptState"); dfaST2.SetAttribute("alt", s.GetUniquelyPredictedAlt()); return(dfaST2); } // the default templates for generating a state and its edges // can be an if-then-else structure or a switch string dfaStateName = "dfaState"; string dfaLoopbackStateName = "dfaLoopbackState"; string dfaOptionalBlockStateName = "dfaOptionalBlockState"; string dfaEdgeName = "dfaEdge"; if (parentGenerator.CanGenerateSwitch(s)) { dfaStateName = "dfaStateSwitch"; dfaLoopbackStateName = "dfaLoopbackStateSwitch"; dfaOptionalBlockStateName = "dfaOptionalBlockStateSwitch"; dfaEdgeName = "dfaEdgeSwitch"; } StringTemplate dfaST = templates.GetInstanceOf(dfaStateName); if (dfa.NFADecisionStartState.decisionStateType == NFAState.LOOPBACK) { dfaST = templates.GetInstanceOf(dfaLoopbackStateName); } else if (dfa.NFADecisionStartState.decisionStateType == NFAState.OPTIONAL_BLOCK_START) { dfaST = templates.GetInstanceOf(dfaOptionalBlockStateName); } dfaST.SetAttribute("k", k); dfaST.SetAttribute("stateNumber", s.stateNumber); dfaST.SetAttribute("semPredState", s.IsResolvedWithPredicates); /* * string description = dfa.getNFADecisionStartState().Description; * description = parentGenerator.target.getTargetStringLiteralFromString( description ); * //System.Console.Out.WriteLine( "DFA: " + description + " associated with AST " + dfa.getNFADecisionStartState() ); * if ( description != null ) * { * dfaST.SetAttribute( "description", description ); * } */ int EOTPredicts = NFA.INVALID_ALT_NUMBER; DFAState EOTTarget = null; //System.Console.Out.WriteLine( "DFA state " + s.stateNumber ); for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = (Transition)s.Transition(i); //System.Console.Out.WriteLine( "edge " + s.stateNumber + "-" + edge.label.ToString() + "->" + edge.target.stateNumber ); if (edge.label.Atom == Label.EOT) { // don't generate a real edge for EOT; track alt EOT predicts // generate that prediction in the else clause as default case EOTTarget = (DFAState)edge.target; EOTPredicts = EOTTarget.GetUniquelyPredictedAlt(); /* * System.Console.Out.WriteLine("DFA s"+s.stateNumber+" EOT goes to s"+ * edge.target.stateNumber+" predicates alt "+ * EOTPredicts); */ continue; } StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName); // If the template wants all the label values delineated, do that if (edgeST.GetFormalArgument("labels") != null) { List <string> labels = edge.Label.Set.Select(value => parentGenerator.GetTokenTypeAsTargetLabel(value)).ToList(); edgeST.SetAttribute("labels", labels); } else { // else create an expression to evaluate (the general case) edgeST.SetAttribute("labelExpr", parentGenerator.GenLabelExpr(templates, edge, k)); } // stick in any gated predicates for any edge if not already a pred if (!edge.label.IsSemanticPredicate) { DFAState target = (DFAState)edge.target; SemanticContext preds = target.GetGatedPredicatesInNFAConfigurations(); if (preds != null) { //System.Console.Out.WriteLine( "preds=" + target.getGatedPredicatesInNFAConfigurations() ); StringTemplate predST = preds.GenExpr(parentGenerator, parentGenerator.Templates, dfa); edgeST.SetAttribute("predicates", predST); } } StringTemplate targetST = WalkFixedDFAGeneratingStateMachine(templates, dfa, (DFAState)edge.target, k + 1); edgeST.SetAttribute("targetState", targetST); dfaST.SetAttribute("edges", edgeST); //System.Console.Out.WriteLine( "back to DFA " + dfa.decisionNumber + "." + s.stateNumber ); } // HANDLE EOT EDGE if (EOTPredicts != NFA.INVALID_ALT_NUMBER) { // EOT unique predicts an alt dfaST.SetAttribute("eotPredictsAlt", EOTPredicts); } else if (EOTTarget != null && EOTTarget.NumberOfTransitions > 0) { // EOT state has transitions so must split on predicates. // Generate predicate else-if clauses and then generate // NoViableAlt exception as else clause. // Note: these predicates emanate from the EOT target state // rather than the current DFAState s so the error message // might be slightly misleading if you are looking at the // state number. Predicates emanating from EOT targets are // hoisted up to the state that has the EOT edge. for (int i = 0; i < EOTTarget.NumberOfTransitions; i++) { Transition predEdge = (Transition)EOTTarget.Transition(i); StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName); edgeST.SetAttribute("labelExpr", parentGenerator.GenSemanticPredicateExpr(templates, predEdge)); // the target must be an accept state //System.Console.Out.WriteLine( "EOT edge" ); StringTemplate targetST = WalkFixedDFAGeneratingStateMachine(templates, dfa, (DFAState)predEdge.target, k + 1); edgeST.SetAttribute("targetState", targetST); dfaST.SetAttribute("edges", edgeST); } } return(dfaST); }
public void TestDoNotTranslateScopeAttributeCompare() { string action = "if ($rule::foo == \"foo\" || 1) { System.out.println(\"ouch\"); }"; string expecting = "if (((rule_scope)rule_stack.peek()).foo == \"foo\" || 1) { System.out.println(\"ouch\"); }"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar a;\n" + "rule\n" + "scope {\n" + " String foo;" + "} :\n" + " twoIDs" + " ;\n" + "twoIDs:\n" + " ID ID {" + action + "}\n" + " ;\n" + "ID : 'id';" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); ActionTranslator translator = new ActionTranslator( generator, "twoIDs", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); // check that we didn't use scopeSetAttributeRef int translation! bool foundScopeSetAttributeRef = false; for ( int i = 0; i < translator.chunks.Count; i++ ) { object chunk = translator.chunks[i]; if ( chunk is StringTemplate ) { if ( ( (StringTemplate)chunk ).Name.Equals( "scopeSetAttributeRef" ) ) { foundScopeSetAttributeRef = true; } } } assertFalse( "action translator used scopeSetAttributeRef template in comparison!", foundScopeSetAttributeRef ); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); assertEquals( expecting, found ); }
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST) { generator.Templates.RegisterRenderer(typeof(string), new StringRenderer(generator, this)); base.GenRecognizerFile(tool, generator, grammar, outputFileST); }
public void TestDynamicScopeRefOkEvenThoughRuleRefExists() { string action = "$b::n;"; string expecting = "((b_scope)b_stack.peek()).n;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "s : b ;\n" + "b\n" + "scope {\n" + " int n;\n" + "} : '(' b ')' {" + action + "}\n" + // refers to current invocation's n " ;\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "b", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public void TestBasicRuleScope() { string action = "$a::n;"; string expecting = "((a_scope)a_stack.peek()).n;"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); Grammar g = new Grammar( "grammar t;\n" + "a\n" + "scope {\n" + " int n;\n" + "} : {" + action + "}\n" + " ;\n" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); // Now fill template with information about problemState var labels = probe.GetSampleNonDeterministicInputSequence(problemState); string input = probe.GetInputSequenceDisplay(labels); st.SetAttribute("input", input); if (probe.dfa.IsTokensRuleDecision) { var disabledAlts = probe.GetDisabledAlternatives(problemState); foreach (int altI in disabledAlts) { string tokenName = probe.GetTokenNameForTokensRuleAlt((int)altI); // reset the line/col to the token definition (pick last one) NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName); line = ruleStart.associatedASTNode.Line; charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine; st.SetAttribute("disabled", tokenName); } } else { st.SetAttribute("disabled", probe.GetDisabledAlternatives(problemState)); } var nondetAlts = probe.GetNonDeterministicAltsForState(problemState); NFAState nfaStart = probe.dfa.NFADecisionStartState; // all state paths have to begin with same NFA state int firstAlt = 0; if (nondetAlts != null) { foreach (int displayAltI in nondetAlts) { if (DecisionProbe.verbose) { int tracePathAlt = nfaStart.TranslateDisplayAltToWalkAlt((int)displayAltI); if (firstAlt == 0) { firstAlt = tracePathAlt; } IList path = probe.GetNFAPathStatesForAlt(firstAlt, tracePathAlt, labels); st.SetAttribute("paths.{alt,states}", displayAltI, path); } else { if (probe.dfa.IsTokensRuleDecision) { // alts are token rules, convert to the names instead of numbers string tokenName = probe.GetTokenNameForTokensRuleAlt((int)displayAltI); st.SetAttribute("conflictingTokens", tokenName); } else { st.SetAttribute("conflictingAlts", displayAltI); } } } } st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction); return(base.ToString(st)); }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. We know this is an NFA * for a rule so don't traverse edges to other rules and * don't go past rule end state. */ protected virtual void WalkRuleNFACreatingDOT( StringTemplate dot, State s ) { if ( markedStates.Contains( s.stateNumber ) ) { return; // already visited this node } markedStates.Add( s.stateNumber ); // mark this node as completed. // first add this node StringTemplate stateST; if ( s.IsAcceptState ) { stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) ); } else { stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) ); } stateST.SetAttribute( "name", GetStateLabel( s ) ); dot.SetAttribute( "states", stateST ); if ( s.IsAcceptState ) { return; // don't go past end of rule node to the follow states } // special case: if decision point, then line up the alt start states // unless it's an end of block if ( ( (NFAState)s ).IsDecisionState ) { GrammarAST n = ( (NFAState)s ).associatedASTNode; if ( n != null && n.Type != ANTLRParser.EOB ) { StringTemplate rankST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "decision-rank" ) ); NFAState alt = (NFAState)s; while ( alt != null ) { rankST.SetAttribute( "states", GetStateLabel( alt ) ); if ( alt.transition[1] != null ) { alt = (NFAState)alt.transition[1].target; } else { alt = null; } } dot.SetAttribute( "decisionRanks", rankST ); } } // make a DOT edge for each transition StringTemplate edgeST = null; for ( int i = 0; i < s.NumberOfTransitions; i++ ) { Transition edge = (Transition)s.GetTransition( i ); if ( edge is RuleClosureTransition ) { RuleClosureTransition rr = ( (RuleClosureTransition)edge ); // don't jump to other rules, but display edge to follow node edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); if ( rr.rule.grammar != grammar ) { edgeST.SetAttribute( "label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">" ); } else { edgeST.SetAttribute( "label", "<" + rr.rule.Name + ">" ); } edgeST.SetAttribute( "src", GetStateLabel( s ) ); edgeST.SetAttribute( "target", GetStateLabel( rr.followState ) ); edgeST.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", edgeST ); WalkRuleNFACreatingDOT( dot, rr.followState ); continue; } if ( edge.IsAction ) { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "action-edge" ) ); } else if ( edge.IsEpsilon ) { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "epsilon-edge" ) ); } else { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); } edgeST.SetAttribute( "label", GetEdgeLabel( edge ) ); edgeST.SetAttribute( "src", GetStateLabel( s ) ); edgeST.SetAttribute( "target", GetStateLabel( edge.target ) ); edgeST.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", edgeST ); WalkRuleNFACreatingDOT( dot, edge.target ); // keep walkin' } }
public void TestComplicatedArgParsingWithTranslation() { string action = "x, $A.text+\"3242\", (*$A).foo(21,33), 3.2+1, '\\n', " + "\"a,oo\\nick\", {bl, \"fdkj\"eck}"; string expecting = "x, (A1!=null?A1.getText():null)+\"3242\", (*A1).foo(21,33), 3.2+1, '\\n', \"a,oo\\nick\", {bl, \"fdkj\"eck}"; ErrorQueue equeue = new ErrorQueue(); ErrorManager.SetErrorListener( equeue ); // now check in actual grammar. Grammar g = new Grammar( "parser grammar t;\n" + "a[User u, int i]\n" + " : A a[" + action + "] B\n" + " ;" ); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); g.CodeGenerator = generator; generator.GenRecognizer(); // forces load of templates ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 1 ); string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count ); }