public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); string ruleName = probe.dfa.NFADecisionStartState.enclosingRule.Name; st.SetAttribute("ruleName", ruleName); List <int> sortedAlts = new List <int>(); sortedAlts.addAll(altsWithRecursion); sortedAlts.Sort(); st.SetAttribute("alts", sortedAlts); return(base.ToString(st)); }
/** Return a String containing a DOT description that, when displayed, * will show the incoming state machine visually. All nodes reachable * from startState will be included. */ public virtual string GenerateGraph(State startState) { if (startState == null) { return(null); } // The output DOT graph for visualization StringTemplate dot = null; markedStates = new HashSet <int>(); if (startState is DFAState) { dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "dfa")); dot.SetAttribute("startState", startState.stateNumber); dot.SetAttribute("useBox", AntlrTool.internalOption_ShowNFAConfigsInDFA); WalkCreatingDFADOT(dot, (DFAState)startState); } else { dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "nfa")); dot.SetAttribute("startState", startState.stateNumber); WalkRuleNFACreatingDOT(dot, startState); } dot.SetAttribute("rankdir", rankdir); return(dot.ToString()); }
public override string ToString() { line = 0; charPositionInLine = 0; if (offendingToken != null) { line = offendingToken.Line; charPositionInLine = offendingToken.CharPositionInLine; } if (g != null) { file = g.FileName; } StringTemplate st = GetMessageTemplate(); if (arg != null) { st.SetAttribute("arg", arg); } if (arg2 != null) { st.SetAttribute("arg2", arg2); } return(base.ToString(st)); }
public override String ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; String fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); st.SetAttribute("targetRules", targetRules); st.SetAttribute("alt", alt); st.SetAttribute("callSiteStates", callSiteStates); var labels = probe.GetSampleNonDeterministicInputSequence(sampleBadState); String input = probe.GetInputSequenceDisplay(labels); st.SetAttribute("input", input); return(base.ToString(st)); }
public virtual StringTemplate GetDependencies() { LoadDependencyTemplates(); StringTemplate dependenciesST = templates.GetInstanceOf("dependencies"); dependenciesST.SetAttribute("in", GetDependenciesFileList()); dependenciesST.SetAttribute("out", GetGeneratedFileList()); dependenciesST.SetAttribute("grammarFileName", grammar.fileName); return(dependenciesST); }
public override String ToString() { StringTemplate st = GetMessageTemplate(); st.SetAttribute("listOfCycles", cycles); return(base.ToString(st)); }
protected StringTemplate GetRuleElementST(string name, string ruleTargetName, GrammarAST elementAST, GrammarAST ast_suffix, string label) { string suffix = GetSTSuffix(elementAST, ast_suffix, label); name += suffix; // if we're building trees and there is no label, gen a label // unless we're in a synpred rule. Rule r = grammar.GetRule(currentRuleName); if ((grammar.BuildAST || suffix.Length > 0) && label == null && (r == null || !r.isSynPred)) { // we will need a label to do the AST or tracking, make one label = generator.CreateUniqueLabel(ruleTargetName); CommonToken labelTok = new CommonToken(ANTLRParser.ID, label); grammar.DefineRuleRefLabel(currentRuleName, labelTok, elementAST); } StringTemplate elementST = templates.GetInstanceOf(name); if (label != null) { elementST.SetAttribute("label", label); } return(elementST); }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. */ protected virtual void WalkCreatingDFADOT(StringTemplate dot, DFAState s) { if (markedStates.Contains(s.stateNumber)) { return; // already visited this node } markedStates.Add(s.stateNumber); // mark this node as completed. // first add this node StringTemplate st; if (s.IsAcceptState) { st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate")); } else { st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state")); } st.SetAttribute("name", GetStateLabel(s)); dot.SetAttribute("states", st); // make a DOT edge for each transition for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = (Transition)s.Transition(i); //Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s" // + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions ); if (StripNonreducedStates) { if (edge.target is DFAState && ((DFAState)edge.target).AcceptStateReachable != DFA.REACHABLE_YES) { continue; // don't generate nodes for terminal states } } st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge")); st.SetAttribute("label", GetEdgeLabel(edge)); st.SetAttribute("src", GetStateLabel(s)); st.SetAttribute("target", GetStateLabel(edge.target)); st.SetAttribute("arrowhead", arrowhead); dot.SetAttribute("edges", st); WalkCreatingDFADOT(dot, (DFAState)edge.target); // keep walkin' } }
/** Return a String containing a DOT description that, when displayed, * will show the incoming state machine visually. All nodes reachable * from startState will be included. */ public string GetRuleNFADOT(State startState) { // The output DOT graph for visualization StringTemplate dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "nfa")); markedStates = new HashSet <object>(); dot.SetAttribute("startState", startState.stateNumber); walkRuleNFACreatingDOT(dot, startState); return(dot.ToString()); }
public override string ToString() { StringTemplate st = GetMessageTemplate(); if (arg != null) { st.SetAttribute("arg", arg); } if (arg2 != null) { st.SetAttribute("arg2", arg2); } if (e != null) { st.SetAttribute("exception", e); st.SetAttribute("stackTrace", e.getStackTrace()); } return(base.ToString(st)); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); // convert to string key to avoid 3.1 ST bug var altToLocationsWithStringKey = new SortedList <string, ICollection <IToken> >(StringComparer.Ordinal); List <int> alts = new List <int>(); alts.addAll(altToLocations.Keys); alts.Sort(); foreach (int altI in alts) { altToLocationsWithStringKey[altI.ToString()] = altToLocations.get(altI); //List<string> tokens = new List<string>(); //foreach ( IToken t in altToLocations.get( altI ) ) //{ // tokens.Add( t.ToString() ); //} //tokens.Sort(); //System.Console.Out.WriteLine( "tokens=\n" + tokens ); } st.SetAttribute("altToLocations", altToLocationsWithStringKey); var sampleInputLabels = problemState.dfa.probe.GetSampleNonDeterministicInputSequence(problemState); string input = problemState.dfa.probe.GetInputSequenceDisplay(sampleInputLabels); st.SetAttribute("upon", input); st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction); return(base.ToString(st)); }
protected FileResult FileExcel(Antlr3.ST.StringTemplate st, string fileName) { Response.AppendCookie(new HttpCookie("fileDownload", "true") { Path = "/" }); st.SetAttribute("FileTitle", fileName); var str = st.ToString(); byte[] fileContents = Encoding.UTF8.GetBytes(str); return(File(fileContents, "application/vnd.ms-excel", $"{fileName}.xls")); }
public string Convert(Model.TemplateSource.PropertyModel source) { string result = ""; string templateString = sourceStructure.GetPropertyTemplate(source.TypeName); Antlr3.ST.StringTemplate template = new Antlr3.ST.StringTemplate(); template.Template = templateString; template.SetAttribute("property", source); result = template.ToString(); return result; }
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST) { // Before we write this, and cause it to generate its string, // we need to add all the string literals that we are going to match // outputFileST.SetAttribute("literals", _strings); string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type); generator.Write(outputFileST, fileName); }
protected StringTemplate GetTokenElementST(string name, string elementName, GrammarAST elementAST, GrammarAST ast_suffix, string label) { bool tryUnchecked = false; if (name == "matchSet" && !string.IsNullOrEmpty(elementAST.enclosingRuleName) && Rule.GetRuleType(elementAST.enclosingRuleName) == RuleType.Lexer) { if ((elementAST.Parent.Type == ANTLRLexer.ALT && elementAST.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.ChildCount == 2) || (elementAST.Parent.Type == ANTLRLexer.NOT && elementAST.Parent.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.Parent.ChildCount == 2)) { // single alt at the start of the rule needs to be checked } else { tryUnchecked = true; } } string suffix = GetSTSuffix(elementAST, ast_suffix, label); // if we're building trees and there is no label, gen a label // unless we're in a synpred rule. Rule r = grammar.GetRule(currentRuleName); if ((grammar.BuildAST || suffix.Length > 0) && label == null && (r == null || !r.isSynPred)) { label = generator.CreateUniqueLabel(elementName); CommonToken labelTok = new CommonToken(ANTLRParser.ID, label); grammar.DefineTokenRefLabel(currentRuleName, labelTok, elementAST); } StringTemplate elementST = null; if (tryUnchecked && templates.IsDefined(name + "Unchecked" + suffix)) { elementST = templates.GetInstanceOf(name + "Unchecked" + suffix); } if (elementST == null) { elementST = templates.GetInstanceOf(name + suffix); } if (label != null) { elementST.SetAttribute("label", label); } return(elementST); }
public override String ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; String fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); if (probe.dfa.IsTokensRuleDecision) { // alts are token rules, convert to the names instead of numbers for (int i = 0; i < alts.Length; i++) { int altI = alts[i]; String tokenName = probe.GetTokenNameForTokensRuleAlt(altI); // reset the line/col to the token definition NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName); line = ruleStart.associatedASTNode.Line; charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine; st.SetAttribute("tokens", tokenName); } } else { // regular alt numbers, show the alts st.SetAttribute("alts", alts); } return(base.ToString(st)); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } var labels = probe.GetSampleNonDeterministicInputSequence(problemState); string input = probe.GetInputSequenceDisplay(labels); StringTemplate st = GetMessageTemplate(); List <int> alts = new List <int>(); alts.addAll(problemState.AltSet); alts.Sort(); st.SetAttribute("danglingAlts", alts); st.SetAttribute("input", input); return(base.ToString(st)); }
public virtual string ToString(StringTemplate messageST) { // setup the location locationST = ErrorManager.GetLocationFormat(); reportST = ErrorManager.GetReportFormat(); messageFormatST = ErrorManager.GetMessageFormat(); bool locationValid = false; if (line != -1) { locationST.SetAttribute("line", line); locationValid = true; } if (charPositionInLine != -1) { locationST.SetAttribute("column", charPositionInLine + 1); locationValid = true; } if (file != null) { locationST.SetAttribute("file", file); locationValid = true; } messageFormatST.SetAttribute("id", msgID); messageFormatST.SetAttribute("text", messageST); if (locationValid) { reportST.SetAttribute("location", locationST); } reportST.SetAttribute("message", messageFormatST); reportST.SetAttribute("type", ErrorManager.GetMessageType(msgID)); return(reportST.ToString()); }
public override StringTemplate GenExpr(CodeGenerator generator, StringTemplateGroup templates, DFA dfa) { StringTemplate eST = null; if (templates != null) { if (_synpred) { eST = templates.GetInstanceOf("evalSynPredicate"); } else { eST = templates.GetInstanceOf("evalPredicate"); generator.grammar.decisionsWhoseDFAsUsesSemPreds.Add(dfa); } string predEnclosingRuleName = predicateAST.enclosingRuleName; /* * String decisionEnclosingRuleName = * dfa.getNFADecisionStartState().getEnclosingRule(); * // if these rulenames are diff, then pred was hoisted out of rule * // Currently I don't warn you about this as it could be annoying. * // I do the translation anyway. */ //eST.setAttribute("pred", this.toString()); if (generator != null) { eST.SetAttribute("pred", generator.TranslateAction(predEnclosingRuleName, predicateAST)); } } else { eST = new StringTemplate("$pred$"); eST.SetAttribute("pred", this.ToString()); return(eST); } if (generator != null) { string description = generator.target.GetTargetStringLiteralFromString(this.ToString()); eST.SetAttribute("description", description); } return(eST); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); st.SetAttribute("enclosingRule", probe.dfa.NFADecisionStartState.enclosingRule.Name); return(base.ToString(st)); }
public override String ToString() { line = 0; charPositionInLine = 0; if (offendingToken != null) { line = offendingToken.Line; charPositionInLine = offendingToken.CharPositionInLine; } // TODO: actually set the right Grammar instance to get the filename // TODO: have to update all v2 grammar files for this. or use errormanager and tool to get the current grammar if (g != null) { file = g.FileName; } StringTemplate st = GetMessageTemplate(); if (arg != null) { st.SetAttribute("arg", arg); } return(base.ToString(st)); }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. We know this is an NFA * for a rule so don't traverse edges to other rules and * don't go past rule end state. */ protected virtual void WalkRuleNFACreatingDOT( StringTemplate dot, State s ) { if ( markedStates.Contains( s.stateNumber ) ) { return; // already visited this node } markedStates.Add( s.stateNumber ); // mark this node as completed. // first add this node StringTemplate stateST; if ( s.IsAcceptState ) { stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) ); } else { stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) ); } stateST.SetAttribute( "name", GetStateLabel( s ) ); dot.SetAttribute( "states", stateST ); if ( s.IsAcceptState ) { return; // don't go past end of rule node to the follow states } // special case: if decision point, then line up the alt start states // unless it's an end of block if ( ( (NFAState)s ).IsDecisionState ) { GrammarAST n = ( (NFAState)s ).associatedASTNode; if ( n != null && n.Type != ANTLRParser.EOB ) { StringTemplate rankST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "decision-rank" ) ); NFAState alt = (NFAState)s; while ( alt != null ) { rankST.SetAttribute( "states", GetStateLabel( alt ) ); if ( alt.transition[1] != null ) { alt = (NFAState)alt.transition[1].target; } else { alt = null; } } dot.SetAttribute( "decisionRanks", rankST ); } } // make a DOT edge for each transition StringTemplate edgeST = null; for ( int i = 0; i < s.NumberOfTransitions; i++ ) { Transition edge = (Transition)s.GetTransition( i ); if ( edge is RuleClosureTransition ) { RuleClosureTransition rr = ( (RuleClosureTransition)edge ); // don't jump to other rules, but display edge to follow node edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); if ( rr.rule.grammar != grammar ) { edgeST.SetAttribute( "label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">" ); } else { edgeST.SetAttribute( "label", "<" + rr.rule.Name + ">" ); } edgeST.SetAttribute( "src", GetStateLabel( s ) ); edgeST.SetAttribute( "target", GetStateLabel( rr.followState ) ); edgeST.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", edgeST ); WalkRuleNFACreatingDOT( dot, rr.followState ); continue; } if ( edge.IsAction ) { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "action-edge" ) ); } else if ( edge.IsEpsilon ) { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "epsilon-edge" ) ); } else { edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); } edgeST.SetAttribute( "label", GetEdgeLabel( edge ) ); edgeST.SetAttribute( "src", GetStateLabel( s ) ); edgeST.SetAttribute( "target", GetStateLabel( edge.target ) ); edgeST.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", edgeST ); WalkRuleNFACreatingDOT( dot, edge.target ); // keep walkin' } }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. */ protected virtual void WalkCreatingDFADOT( StringTemplate dot, DFAState s ) { if ( markedStates.Contains( s.stateNumber ) ) { return; // already visited this node } markedStates.Add( s.stateNumber ); // mark this node as completed. // first add this node StringTemplate st; if ( s.IsAcceptState ) { st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) ); } else { st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) ); } st.SetAttribute( "name", GetStateLabel( s ) ); dot.SetAttribute( "states", st ); // make a DOT edge for each transition for ( int i = 0; i < s.NumberOfTransitions; i++ ) { Transition edge = (Transition)s.Transition( i ); //Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s" // + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions ); if ( StripNonreducedStates ) { if ( edge.target is DFAState && ( (DFAState)edge.target ).AcceptStateReachable != DFA.REACHABLE_YES ) { continue; // don't generate nodes for terminal states } } st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) ); st.SetAttribute( "label", GetEdgeLabel( edge ) ); st.SetAttribute( "src", GetStateLabel( s ) ); st.SetAttribute( "target", GetStateLabel( edge.target ) ); st.SetAttribute( "arrowhead", arrowhead ); dot.SetAttribute( "edges", st ); WalkCreatingDFADOT( dot, (DFAState)edge.target ); // keep walkin' } }
public virtual string ToString( StringTemplate messageST ) { // setup the location locationST = ErrorManager.GetLocationFormat(); reportST = ErrorManager.GetReportFormat(); messageFormatST = ErrorManager.GetMessageFormat(); bool locationValid = false; if ( line != -1 ) { locationST.SetAttribute( "line", line ); locationValid = true; } if ( charPositionInLine != -1 ) { locationST.SetAttribute( "column", charPositionInLine + 1 ); locationValid = true; } if ( file != null ) { locationST.SetAttribute( "file", file ); locationValid = true; } messageFormatST.SetAttribute( "id", msgID ); messageFormatST.SetAttribute( "text", messageST ); if ( locationValid ) { reportST.SetAttribute( "location", locationST ); } reportST.SetAttribute( "message", messageFormatST ); reportST.SetAttribute( "type", ErrorManager.GetMessageType( msgID ) ); return reportST.ToString(); }
public override StringTemplate GenExpr( CodeGenerator generator, StringTemplateGroup templates, DFA dfa ) { StringTemplate eST = null; if ( templates != null ) { if ( _synpred ) { eST = templates.GetInstanceOf( "evalSynPredicate" ); } else { eST = templates.GetInstanceOf( "evalPredicate" ); generator.grammar.decisionsWhoseDFAsUsesSemPreds.Add( dfa ); } string predEnclosingRuleName = predicateAST.enclosingRuleName; /* String decisionEnclosingRuleName = dfa.getNFADecisionStartState().getEnclosingRule(); // if these rulenames are diff, then pred was hoisted out of rule // Currently I don't warn you about this as it could be annoying. // I do the translation anyway. */ //eST.setAttribute("pred", this.toString()); if ( generator != null ) { eST.SetAttribute( "pred", generator.TranslateAction( predEnclosingRuleName, predicateAST ) ); } } else { eST = new StringTemplate( "$pred$" ); eST.SetAttribute( "pred", this.ToString() ); return eST; } if ( generator != null ) { string description = generator.target.GetTargetStringLiteralFromString( this.ToString() ); eST.SetAttribute( "description", description ); } return eST; }
/** Given the grammar to which we are attached, walk the AST associated * with that grammar to create NFAs. Then create the DFAs for all * decision points in the grammar by converting the NFAs to DFAs. * Finally, walk the AST again to generate code. * * Either 1 or 2 files are written: * * recognizer: the main parser/lexer/treewalker item * header file: language like C/C++ need extern definitions * * The target, such as JavaTarget, dictates which files get written. */ public virtual StringTemplate GenRecognizer() { //[email protected]("### generate "+grammar.name+" recognizer"); // LOAD OUTPUT TEMPLATES LoadTemplates( language ); if ( templates == null ) { return null; } // CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA if ( ErrorManager.DoNotAttemptAnalysis() ) { return null; } target.PerformGrammarAnalysis( this, grammar ); // some grammar analysis errors will not yield reliable DFA if ( ErrorManager.DoNotAttemptCodeGen() ) { return null; } // OPTIMIZE DFA DFAOptimizer optimizer = new DFAOptimizer( grammar ); optimizer.Optimize(); // OUTPUT FILE (contains recognizerST) outputFileST = templates.GetInstanceOf( "outputFile" ); // HEADER FILE if ( templates.IsDefined( "headerFile" ) ) { headerFileST = templates.GetInstanceOf( "headerFile" ); } else { // create a dummy to avoid null-checks all over code generator headerFileST = new StringTemplate( templates, "" ); headerFileST.Name = "dummy-header-file"; } bool filterMode = grammar.GetOption( "filter" ) != null && grammar.GetOption( "filter" ).Equals( "true" ); bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption || grammar.SyntacticPredicates != null || filterMode; // TODO: move this down further because generating the recognizer // alters the model with info on who uses predefined properties etc... // The actions here might refer to something. // The only two possible output files are available at this point. // Verify action scopes are ok for target and dump actions into output // Templates can say <actions.parser.header> for example. var actions = grammar.Actions; VerifyActionScopesOkForTarget( actions ); // translate $x::y references TranslateActionAttributeReferences( actions ); StringTemplate gateST = templates.GetInstanceOf( "actionGate" ); if ( filterMode ) { // if filtering, we need to set actions to execute at backtracking // level 1 not 0. gateST = templates.GetInstanceOf( "filteringActionGate" ); } grammar.SetSynPredGateIfNotAlready( gateST ); headerFileST.SetAttribute( "actions", actions ); outputFileST.SetAttribute( "actions", actions ); headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate ); headerFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "buildAST", grammar.BuildAST ); outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode ); outputFileST.SetAttribute( "backtracking", canBacktrack ); headerFileST.SetAttribute( "backtracking", canBacktrack ); // turn on memoize attribute at grammar level so we can create ruleMemo. // each rule has memoize attr that hides this one, indicating whether // it needs to save results string memoize = (string)grammar.GetOption( "memoize" ); outputFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); headerFileST.SetAttribute( "memoize", ( grammar.atLeastOneRuleMemoizes || ( memoize != null && memoize.Equals( "true" ) ) && canBacktrack ) ); outputFileST.SetAttribute( "trace", trace ); headerFileST.SetAttribute( "trace", trace ); outputFileST.SetAttribute( "profile", profile ); headerFileST.SetAttribute( "profile", profile ); // RECOGNIZER if ( grammar.type == GrammarType.Lexer ) { recognizerST = templates.GetInstanceOf( "lexer" ); outputFileST.SetAttribute( "LEXER", true ); headerFileST.SetAttribute( "LEXER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } else if ( grammar.type == GrammarType.Parser || grammar.type == GrammarType.Combined ) { recognizerST = templates.GetInstanceOf( "parser" ); outputFileST.SetAttribute( "PARSER", true ); headerFileST.SetAttribute( "PARSER", true ); } else { recognizerST = templates.GetInstanceOf( "treeParser" ); outputFileST.SetAttribute( "TREE_PARSER", true ); headerFileST.SetAttribute( "TREE_PARSER", true ); recognizerST.SetAttribute( "filterMode", filterMode ); } outputFileST.SetAttribute( "recognizer", recognizerST ); headerFileST.SetAttribute( "recognizer", recognizerST ); outputFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); headerFileST.SetAttribute( "actionScope", grammar.GetDefaultActionScope( grammar.type ) ); string targetAppropriateFileNameString = target.GetTargetStringLiteralFromString( grammar.FileName ); outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString ); outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion ); outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() ); { // GENERATE RECOGNIZER // Walk the AST holding the input grammar, this time generating code // Decisions are generated by using the precomputed DFAs // Fill in the various templates with data CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) ); try { gen.grammar_( grammar, recognizerST, outputFileST, headerFileST ); } catch ( RecognitionException re ) { ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE, re ); } } GenTokenTypeConstants( recognizerST ); GenTokenTypeConstants( outputFileST ); GenTokenTypeConstants( headerFileST ); if ( grammar.type != GrammarType.Lexer ) { GenTokenTypeNames( recognizerST ); GenTokenTypeNames( outputFileST ); GenTokenTypeNames( headerFileST ); } // Now that we know what synpreds are used, we can set into template HashSet<string> synpredNames = null; if ( grammar.synPredNamesUsedInDFA.Count > 0 ) { synpredNames = grammar.synPredNamesUsedInDFA; } outputFileST.SetAttribute( "synpreds", synpredNames ); headerFileST.SetAttribute( "synpreds", synpredNames ); // all recognizers can see Grammar object recognizerST.SetAttribute( "grammar", grammar ); // WRITE FILES try { target.GenRecognizerFile( tool, this, grammar, outputFileST ); if ( templates.IsDefined( "headerFile" ) ) { StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" ); target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() ); } // write out the vocab interchange file; used by antlr, // does not change per target StringTemplate tokenVocabSerialization = GenTokenVocabOutput(); string vocabFileName = VocabFileName; if ( vocabFileName != null ) { Write( tokenVocabSerialization, vocabFileName ); } //[email protected](outputFileST.getDOTForDependencyGraph(false)); } catch ( IOException ioe ) { ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE, VocabFileName, ioe ); } /* [email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs); [email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups); */ return outputFileST; }
/** Generate a token vocab file with all the token names/types. For example: * ID=7 * FOR=8 * 'for'=8 * * This is independent of the target language; used by antlr internally */ protected virtual StringTemplate GenTokenVocabOutput() { StringTemplate vocabFileST = new StringTemplate( vocabFilePattern, typeof( AngleBracketTemplateLexer ) ); vocabFileST.Name = "vocab-file"; // make constants for the token names foreach ( string tokenID in grammar.TokenIDs ) { int tokenType = grammar.GetTokenType( tokenID ); if ( tokenType >= Label.MIN_TOKEN_TYPE ) { vocabFileST.SetAttribute( "tokens.{name,type}", tokenID, tokenType ); } } // now dump the strings foreach ( string literal in grammar.StringLiterals ) { int tokenType = grammar.GetTokenType( literal ); if ( tokenType >= Label.MIN_TOKEN_TYPE ) { vocabFileST.SetAttribute( "tokens.{name,type}", literal, tokenType ); } } return vocabFileST; }
/** Generate a token names table that maps token type to a printable * name: either the label like INT or the literal like "begin". */ protected virtual void GenTokenTypeNames( StringTemplate code ) { for ( int t = Label.MIN_TOKEN_TYPE; t <= grammar.MaxTokenType; t++ ) { string tokenName = grammar.GetTokenDisplayName( t ); if ( tokenName != null ) { tokenName = target.GetTargetStringLiteralFromString( tokenName, true ); code.SetAttribute( "tokenNames", tokenName ); } } }
// T O K E N D E F I N I T I O N G E N E R A T I O N /** Set attributes tokens and literals attributes in the incoming * code template. This is not the token vocab interchange file, but * rather a list of token type ID needed by the recognizer. */ protected virtual void GenTokenTypeConstants( StringTemplate code ) { // make constants for the token types foreach ( string tokenID in grammar.TokenIDs ) { int tokenType = grammar.GetTokenType( tokenID ); if ( tokenType == Label.EOF || tokenType >= Label.MIN_TOKEN_TYPE ) { // don't do FAUX labels 'cept EOF code.SetAttribute( "tokens.{name,type}", tokenID, tokenType ); } } }
/** Translate an action like [3,"foo",a[3]] and return a List of the * translated actions. Because actions are themselves translated to a list * of chunks, must cat together into a StringTemplate>. Don't translate * to strings early as we need to eval templates in context. */ public virtual List<StringTemplate> TranslateArgAction( string ruleName, GrammarAST actionTree ) { string actionText = actionTree.Token.Text; List<string> args = GetListOfArgumentsFromAction( actionText, ',' ); List<StringTemplate> translatedArgs = new List<StringTemplate>(); foreach ( string arg in args ) { if ( arg != null ) { IToken actionToken = new CommonToken( ANTLRParser.ACTION, arg ); ActionTranslator translator = new ActionTranslator( this, ruleName, actionToken, actionTree.outerAltNum ); IList chunks = translator.TranslateToChunks(); chunks = target.PostProcessAction( chunks, actionToken ); StringTemplate catST = new StringTemplate( templates, "<chunks>" ); catST.SetAttribute( "chunks", chunks ); templates.CreateStringTemplate(); translatedArgs.Add( catST ); } } if ( translatedArgs.Count == 0 ) { return null; } return translatedArgs; }
/** Do a depth-first walk of the state machine graph and * fill a DOT description template. Keep filling the * states and edges attributes. We know this is an NFA * for a rule so don't traverse edges to other rules and * don't go past rule end state. */ protected virtual void WalkRuleNFACreatingDOT(StringTemplate dot, State s) { if (markedStates.Contains(s.stateNumber)) { return; // already visited this node } markedStates.Add(s.stateNumber); // mark this node as completed. // first add this node StringTemplate stateST; if (s.IsAcceptState) { stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate")); } else { stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state")); } stateST.SetAttribute("name", GetStateLabel(s)); dot.SetAttribute("states", stateST); if (s.IsAcceptState) { return; // don't go past end of rule node to the follow states } // special case: if decision point, then line up the alt start states // unless it's an end of block if (((NFAState)s).IsDecisionState) { GrammarAST n = ((NFAState)s).associatedASTNode; if (n != null && n.Type != ANTLRParser.EOB) { StringTemplate rankST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "decision-rank")); NFAState alt = (NFAState)s; while (alt != null) { rankST.SetAttribute("states", GetStateLabel(alt)); if (alt.transition[1] != null) { alt = (NFAState)alt.transition[1].target; } else { alt = null; } } dot.SetAttribute("decisionRanks", rankST); } } // make a DOT edge for each transition StringTemplate edgeST = null; for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = (Transition)s.GetTransition(i); if (edge is RuleClosureTransition) { RuleClosureTransition rr = ((RuleClosureTransition)edge); // don't jump to other rules, but display edge to follow node edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge")); if (rr.rule.grammar != grammar) { edgeST.SetAttribute("label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">"); } else { edgeST.SetAttribute("label", "<" + rr.rule.Name + ">"); } edgeST.SetAttribute("src", GetStateLabel(s)); edgeST.SetAttribute("target", GetStateLabel(rr.followState)); edgeST.SetAttribute("arrowhead", arrowhead); dot.SetAttribute("edges", edgeST); WalkRuleNFACreatingDOT(dot, rr.followState); continue; } if (edge.IsAction) { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "action-edge")); } else if (edge.IsEpsilon) { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "epsilon-edge")); } else { edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge")); } edgeST.SetAttribute("label", GetEdgeLabel(edge)); edgeST.SetAttribute("src", GetStateLabel(s)); edgeST.SetAttribute("target", GetStateLabel(edge.target)); edgeST.SetAttribute("arrowhead", arrowhead); dot.SetAttribute("edges", edgeST); WalkRuleNFACreatingDOT(dot, edge.target); // keep walkin' } }
public void TestEscapedLessThanInAction() { Grammar g = new Grammar(); AntlrTool antlr = newTool(); CodeGenerator generator = new CodeGenerator( antlr, g, "Java" ); string action = "i<3; '<xmltag>'"; ActionTranslator translator = new ActionTranslator( generator, "a", new CommonToken( ANTLRParser.ACTION, action ), 0 ); string expecting = action; string rawTranslation = translator.Translate(); StringTemplateGroup templates = new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) ); StringTemplate actionST = new StringTemplate( templates, "<action>" ); actionST.SetAttribute( "action", rawTranslation ); string found = actionST.ToString(); assertEquals( expecting, found ); }
public override string ToString() { GrammarAST decisionASTNode = probe.dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.dfa.nfa.grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); // Now fill template with information about problemState var labels = probe.GetSampleNonDeterministicInputSequence(problemState); string input = probe.GetInputSequenceDisplay(labels); st.SetAttribute("input", input); if (probe.dfa.IsTokensRuleDecision) { var disabledAlts = probe.GetDisabledAlternatives(problemState); foreach (int altI in disabledAlts) { string tokenName = probe.GetTokenNameForTokensRuleAlt((int)altI); // reset the line/col to the token definition (pick last one) NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName); line = ruleStart.associatedASTNode.Line; charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine; st.SetAttribute("disabled", tokenName); } } else { st.SetAttribute("disabled", probe.GetDisabledAlternatives(problemState)); } var nondetAlts = probe.GetNonDeterministicAltsForState(problemState); NFAState nfaStart = probe.dfa.NFADecisionStartState; // all state paths have to begin with same NFA state int firstAlt = 0; if (nondetAlts != null) { foreach (int displayAltI in nondetAlts) { if (DecisionProbe.verbose) { int tracePathAlt = nfaStart.TranslateDisplayAltToWalkAlt((int)displayAltI); if (firstAlt == 0) { firstAlt = tracePathAlt; } IList path = probe.GetNFAPathStatesForAlt(firstAlt, tracePathAlt, labels); st.SetAttribute("paths.{alt,states}", displayAltI, path); } else { if (probe.dfa.IsTokensRuleDecision) { // alts are token rules, convert to the names instead of numbers string tokenName = probe.GetTokenNameForTokensRuleAlt((int)displayAltI); st.SetAttribute("conflictingTokens", tokenName); } else { st.SetAttribute("conflictingAlts", displayAltI); } } } } st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction); return(base.ToString(st)); }
protected virtual StringTemplate WalkFixedDFAGeneratingStateMachine( StringTemplateGroup templates, DFA dfa, DFAState s, int k) { //System.Console.Out.WriteLine( "walk " + s.stateNumber + " in dfa for decision " + dfa.decisionNumber ); if (s.IsAcceptState) { StringTemplate dfaST2 = templates.GetInstanceOf("dfaAcceptState"); dfaST2.SetAttribute("alt", s.GetUniquelyPredictedAlt()); return(dfaST2); } // the default templates for generating a state and its edges // can be an if-then-else structure or a switch string dfaStateName = "dfaState"; string dfaLoopbackStateName = "dfaLoopbackState"; string dfaOptionalBlockStateName = "dfaOptionalBlockState"; string dfaEdgeName = "dfaEdge"; if (parentGenerator.CanGenerateSwitch(s)) { dfaStateName = "dfaStateSwitch"; dfaLoopbackStateName = "dfaLoopbackStateSwitch"; dfaOptionalBlockStateName = "dfaOptionalBlockStateSwitch"; dfaEdgeName = "dfaEdgeSwitch"; } StringTemplate dfaST = templates.GetInstanceOf(dfaStateName); if (dfa.NFADecisionStartState.decisionStateType == NFAState.LOOPBACK) { dfaST = templates.GetInstanceOf(dfaLoopbackStateName); } else if (dfa.NFADecisionStartState.decisionStateType == NFAState.OPTIONAL_BLOCK_START) { dfaST = templates.GetInstanceOf(dfaOptionalBlockStateName); } dfaST.SetAttribute("k", k); dfaST.SetAttribute("stateNumber", s.stateNumber); dfaST.SetAttribute("semPredState", s.IsResolvedWithPredicates); /* * string description = dfa.getNFADecisionStartState().Description; * description = parentGenerator.target.getTargetStringLiteralFromString( description ); * //System.Console.Out.WriteLine( "DFA: " + description + " associated with AST " + dfa.getNFADecisionStartState() ); * if ( description != null ) * { * dfaST.SetAttribute( "description", description ); * } */ int EOTPredicts = NFA.INVALID_ALT_NUMBER; DFAState EOTTarget = null; //System.Console.Out.WriteLine( "DFA state " + s.stateNumber ); for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = (Transition)s.Transition(i); //System.Console.Out.WriteLine( "edge " + s.stateNumber + "-" + edge.label.ToString() + "->" + edge.target.stateNumber ); if (edge.label.Atom == Label.EOT) { // don't generate a real edge for EOT; track alt EOT predicts // generate that prediction in the else clause as default case EOTTarget = (DFAState)edge.target; EOTPredicts = EOTTarget.GetUniquelyPredictedAlt(); /* * System.Console.Out.WriteLine("DFA s"+s.stateNumber+" EOT goes to s"+ * edge.target.stateNumber+" predicates alt "+ * EOTPredicts); */ continue; } StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName); // If the template wants all the label values delineated, do that if (edgeST.GetFormalArgument("labels") != null) { List <string> labels = edge.Label.Set.Select(value => parentGenerator.GetTokenTypeAsTargetLabel(value)).ToList(); edgeST.SetAttribute("labels", labels); } else { // else create an expression to evaluate (the general case) edgeST.SetAttribute("labelExpr", parentGenerator.GenLabelExpr(templates, edge, k)); } // stick in any gated predicates for any edge if not already a pred if (!edge.label.IsSemanticPredicate) { DFAState target = (DFAState)edge.target; SemanticContext preds = target.GetGatedPredicatesInNFAConfigurations(); if (preds != null) { //System.Console.Out.WriteLine( "preds=" + target.getGatedPredicatesInNFAConfigurations() ); StringTemplate predST = preds.GenExpr(parentGenerator, parentGenerator.Templates, dfa); edgeST.SetAttribute("predicates", predST); } } StringTemplate targetST = WalkFixedDFAGeneratingStateMachine(templates, dfa, (DFAState)edge.target, k + 1); edgeST.SetAttribute("targetState", targetST); dfaST.SetAttribute("edges", edgeST); //System.Console.Out.WriteLine( "back to DFA " + dfa.decisionNumber + "." + s.stateNumber ); } // HANDLE EOT EDGE if (EOTPredicts != NFA.INVALID_ALT_NUMBER) { // EOT unique predicts an alt dfaST.SetAttribute("eotPredictsAlt", EOTPredicts); } else if (EOTTarget != null && EOTTarget.NumberOfTransitions > 0) { // EOT state has transitions so must split on predicates. // Generate predicate else-if clauses and then generate // NoViableAlt exception as else clause. // Note: these predicates emanate from the EOT target state // rather than the current DFAState s so the error message // might be slightly misleading if you are looking at the // state number. Predicates emanating from EOT targets are // hoisted up to the state that has the EOT edge. for (int i = 0; i < EOTTarget.NumberOfTransitions; i++) { Transition predEdge = (Transition)EOTTarget.Transition(i); StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName); edgeST.SetAttribute("labelExpr", parentGenerator.GenSemanticPredicateExpr(templates, predEdge)); // the target must be an accept state //System.Console.Out.WriteLine( "EOT edge" ); StringTemplate targetST = WalkFixedDFAGeneratingStateMachine(templates, dfa, (DFAState)predEdge.target, k + 1); edgeST.SetAttribute("targetState", targetST); dfaST.SetAttribute("edges", edgeST); } } return(dfaST); }