public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
string ruleName = probe.dfa.NFADecisionStartState.enclosingRule.Name;
st.SetAttribute("ruleName", ruleName);
List <int> sortedAlts = new List <int>();
sortedAlts.addAll(altsWithRecursion);
sortedAlts.Sort();
st.SetAttribute("alts", sortedAlts);
return(base.ToString(st));
}
/** Return a String containing a DOT description that, when displayed,
* will show the incoming state machine visually. All nodes reachable
* from startState will be included.
*/
public virtual string GenerateGraph(State startState)
{
if (startState == null)
{
return(null);
}
// The output DOT graph for visualization
StringTemplate dot = null;
markedStates = new HashSet <int>();
if (startState is DFAState)
{
dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "dfa"));
dot.SetAttribute("startState",
startState.stateNumber);
dot.SetAttribute("useBox",
AntlrTool.internalOption_ShowNFAConfigsInDFA);
WalkCreatingDFADOT(dot, (DFAState)startState);
}
else
{
dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "nfa"));
dot.SetAttribute("startState",
startState.stateNumber);
WalkRuleNFACreatingDOT(dot, startState);
}
dot.SetAttribute("rankdir", rankdir);
return(dot.ToString());
}
public override string ToString()
{
line = 0;
charPositionInLine = 0;
if (offendingToken != null)
{
line = offendingToken.Line;
charPositionInLine = offendingToken.CharPositionInLine;
}
if (g != null)
{
file = g.FileName;
}
StringTemplate st = GetMessageTemplate();
if (arg != null)
{
st.SetAttribute("arg", arg);
}
if (arg2 != null)
{
st.SetAttribute("arg2", arg2);
}
return(base.ToString(st));
}
public override String ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
String fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
st.SetAttribute("targetRules", targetRules);
st.SetAttribute("alt", alt);
st.SetAttribute("callSiteStates", callSiteStates);
var labels = probe.GetSampleNonDeterministicInputSequence(sampleBadState);
String input = probe.GetInputSequenceDisplay(labels);
st.SetAttribute("input", input);
return(base.ToString(st));
}
public virtual StringTemplate GetDependencies()
{
LoadDependencyTemplates();
StringTemplate dependenciesST = templates.GetInstanceOf("dependencies");
dependenciesST.SetAttribute("in", GetDependenciesFileList());
dependenciesST.SetAttribute("out", GetGeneratedFileList());
dependenciesST.SetAttribute("grammarFileName", grammar.fileName);
return(dependenciesST);
}
public override String ToString()
{
StringTemplate st = GetMessageTemplate();
st.SetAttribute("listOfCycles", cycles);
return(base.ToString(st));
}
protected StringTemplate GetRuleElementST(string name,
string ruleTargetName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label)
{
string suffix = GetSTSuffix(elementAST, ast_suffix, label);
name += suffix;
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule(currentRuleName);
if ((grammar.BuildAST || suffix.Length > 0) && label == null &&
(r == null || !r.isSynPred))
{
// we will need a label to do the AST or tracking, make one
label = generator.CreateUniqueLabel(ruleTargetName);
CommonToken labelTok = new CommonToken(ANTLRParser.ID, label);
grammar.DefineRuleRefLabel(currentRuleName, labelTok, elementAST);
}
StringTemplate elementST = templates.GetInstanceOf(name);
if (label != null)
{
elementST.SetAttribute("label", label);
}
return(elementST);
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes.
*/
protected virtual void WalkCreatingDFADOT(StringTemplate dot,
DFAState s)
{
if (markedStates.Contains(s.stateNumber))
{
return; // already visited this node
}
markedStates.Add(s.stateNumber); // mark this node as completed.
// first add this node
StringTemplate st;
if (s.IsAcceptState)
{
st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate"));
}
else
{
st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state"));
}
st.SetAttribute("name", GetStateLabel(s));
dot.SetAttribute("states", st);
// make a DOT edge for each transition
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = (Transition)s.Transition(i);
//Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s"
// + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions );
if (StripNonreducedStates)
{
if (edge.target is DFAState &&
((DFAState)edge.target).AcceptStateReachable != DFA.REACHABLE_YES)
{
continue; // don't generate nodes for terminal states
}
}
st = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge"));
st.SetAttribute("label", GetEdgeLabel(edge));
st.SetAttribute("src", GetStateLabel(s));
st.SetAttribute("target", GetStateLabel(edge.target));
st.SetAttribute("arrowhead", arrowhead);
dot.SetAttribute("edges", st);
WalkCreatingDFADOT(dot, (DFAState)edge.target); // keep walkin'
}
}
/** Return a String containing a DOT description that, when displayed,
* will show the incoming state machine visually. All nodes reachable
* from startState will be included.
*/
public string GetRuleNFADOT(State startState)
{
// The output DOT graph for visualization
StringTemplate dot = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "nfa"));
markedStates = new HashSet <object>();
dot.SetAttribute("startState", startState.stateNumber);
walkRuleNFACreatingDOT(dot, startState);
return(dot.ToString());
}
public override string ToString()
{
StringTemplate st = GetMessageTemplate();
if (arg != null)
{
st.SetAttribute("arg", arg);
}
if (arg2 != null)
{
st.SetAttribute("arg2", arg2);
}
if (e != null)
{
st.SetAttribute("exception", e);
st.SetAttribute("stackTrace", e.getStackTrace());
}
return(base.ToString(st));
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
// convert to string key to avoid 3.1 ST bug
var altToLocationsWithStringKey = new SortedList <string, ICollection <IToken> >(StringComparer.Ordinal);
List <int> alts = new List <int>();
alts.addAll(altToLocations.Keys);
alts.Sort();
foreach (int altI in alts)
{
altToLocationsWithStringKey[altI.ToString()] = altToLocations.get(altI);
//List<string> tokens = new List<string>();
//foreach ( IToken t in altToLocations.get( altI ) )
//{
// tokens.Add( t.ToString() );
//}
//tokens.Sort();
//System.Console.Out.WriteLine( "tokens=\n" + tokens );
}
st.SetAttribute("altToLocations", altToLocationsWithStringKey);
var sampleInputLabels = problemState.dfa.probe.GetSampleNonDeterministicInputSequence(problemState);
string input = problemState.dfa.probe.GetInputSequenceDisplay(sampleInputLabels);
st.SetAttribute("upon", input);
st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction);
return(base.ToString(st));
}
protected FileResult FileExcel(Antlr3.ST.StringTemplate st, string fileName)
{
Response.AppendCookie(new HttpCookie("fileDownload", "true")
{
Path = "/"
});
st.SetAttribute("FileTitle", fileName);
var str = st.ToString();
byte[] fileContents = Encoding.UTF8.GetBytes(str);
return(File(fileContents, "application/vnd.ms-excel", $"{fileName}.xls"));
}
public string Convert(Model.TemplateSource.PropertyModel source)
{
string result = "";
string templateString = sourceStructure.GetPropertyTemplate(source.TypeName);
Antlr3.ST.StringTemplate template = new Antlr3.ST.StringTemplate();
template.Template = templateString;
template.SetAttribute("property", source);
result = template.ToString();
return result;
}
protected override void GenRecognizerFile(AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate outputFileST)
{
// Before we write this, and cause it to generate its string,
// we need to add all the string literals that we are going to match
//
outputFileST.SetAttribute("literals", _strings);
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
protected StringTemplate GetTokenElementST(string name,
string elementName,
GrammarAST elementAST,
GrammarAST ast_suffix,
string label)
{
bool tryUnchecked = false;
if (name == "matchSet" && !string.IsNullOrEmpty(elementAST.enclosingRuleName) && Rule.GetRuleType(elementAST.enclosingRuleName) == RuleType.Lexer)
{
if ((elementAST.Parent.Type == ANTLRLexer.ALT && elementAST.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.ChildCount == 2) ||
(elementAST.Parent.Type == ANTLRLexer.NOT && elementAST.Parent.Parent.Parent.Parent.Type == RULE && elementAST.Parent.Parent.Parent.ChildCount == 2))
{
// single alt at the start of the rule needs to be checked
}
else
{
tryUnchecked = true;
}
}
string suffix = GetSTSuffix(elementAST, ast_suffix, label);
// if we're building trees and there is no label, gen a label
// unless we're in a synpred rule.
Rule r = grammar.GetRule(currentRuleName);
if ((grammar.BuildAST || suffix.Length > 0) && label == null &&
(r == null || !r.isSynPred))
{
label = generator.CreateUniqueLabel(elementName);
CommonToken labelTok = new CommonToken(ANTLRParser.ID, label);
grammar.DefineTokenRefLabel(currentRuleName, labelTok, elementAST);
}
StringTemplate elementST = null;
if (tryUnchecked && templates.IsDefined(name + "Unchecked" + suffix))
{
elementST = templates.GetInstanceOf(name + "Unchecked" + suffix);
}
if (elementST == null)
{
elementST = templates.GetInstanceOf(name + suffix);
}
if (label != null)
{
elementST.SetAttribute("label", label);
}
return(elementST);
}
public override String ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
String fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
if (probe.dfa.IsTokensRuleDecision)
{
// alts are token rules, convert to the names instead of numbers
for (int i = 0; i < alts.Length; i++)
{
int altI = alts[i];
String tokenName = probe.GetTokenNameForTokensRuleAlt(altI);
// reset the line/col to the token definition
NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName);
line = ruleStart.associatedASTNode.Line;
charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine;
st.SetAttribute("tokens", tokenName);
}
}
else
{
// regular alt numbers, show the alts
st.SetAttribute("alts", alts);
}
return(base.ToString(st));
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
var labels = probe.GetSampleNonDeterministicInputSequence(problemState);
string input = probe.GetInputSequenceDisplay(labels);
StringTemplate st = GetMessageTemplate();
List <int> alts = new List <int>();
alts.addAll(problemState.AltSet);
alts.Sort();
st.SetAttribute("danglingAlts", alts);
st.SetAttribute("input", input);
return(base.ToString(st));
}
public virtual string ToString(StringTemplate messageST)
{
// setup the location
locationST = ErrorManager.GetLocationFormat();
reportST = ErrorManager.GetReportFormat();
messageFormatST = ErrorManager.GetMessageFormat();
bool locationValid = false;
if (line != -1)
{
locationST.SetAttribute("line", line);
locationValid = true;
}
if (charPositionInLine != -1)
{
locationST.SetAttribute("column", charPositionInLine + 1);
locationValid = true;
}
if (file != null)
{
locationST.SetAttribute("file", file);
locationValid = true;
}
messageFormatST.SetAttribute("id", msgID);
messageFormatST.SetAttribute("text", messageST);
if (locationValid)
{
reportST.SetAttribute("location", locationST);
}
reportST.SetAttribute("message", messageFormatST);
reportST.SetAttribute("type", ErrorManager.GetMessageType(msgID));
return(reportST.ToString());
}
public override StringTemplate GenExpr(CodeGenerator generator,
StringTemplateGroup templates,
DFA dfa)
{
StringTemplate eST = null;
if (templates != null)
{
if (_synpred)
{
eST = templates.GetInstanceOf("evalSynPredicate");
}
else
{
eST = templates.GetInstanceOf("evalPredicate");
generator.grammar.decisionsWhoseDFAsUsesSemPreds.Add(dfa);
}
string predEnclosingRuleName = predicateAST.enclosingRuleName;
/*
* String decisionEnclosingRuleName =
* dfa.getNFADecisionStartState().getEnclosingRule();
* // if these rulenames are diff, then pred was hoisted out of rule
* // Currently I don't warn you about this as it could be annoying.
* // I do the translation anyway.
*/
//eST.setAttribute("pred", this.toString());
if (generator != null)
{
eST.SetAttribute("pred",
generator.TranslateAction(predEnclosingRuleName, predicateAST));
}
}
else
{
eST = new StringTemplate("$pred$");
eST.SetAttribute("pred", this.ToString());
return(eST);
}
if (generator != null)
{
string description =
generator.target.GetTargetStringLiteralFromString(this.ToString());
eST.SetAttribute("description", description);
}
return(eST);
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
st.SetAttribute("enclosingRule",
probe.dfa.NFADecisionStartState.enclosingRule.Name);
return(base.ToString(st));
}
public override String ToString()
{
line = 0;
charPositionInLine = 0;
if (offendingToken != null)
{
line = offendingToken.Line;
charPositionInLine = offendingToken.CharPositionInLine;
}
// TODO: actually set the right Grammar instance to get the filename
// TODO: have to update all v2 grammar files for this. or use errormanager and tool to get the current grammar
if (g != null)
{
file = g.FileName;
}
StringTemplate st = GetMessageTemplate();
if (arg != null)
{
st.SetAttribute("arg", arg);
}
return(base.ToString(st));
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes. We know this is an NFA
* for a rule so don't traverse edges to other rules and
* don't go past rule end state.
*/
protected virtual void WalkRuleNFACreatingDOT( StringTemplate dot,
State s )
{
if ( markedStates.Contains( s.stateNumber ) )
{
return; // already visited this node
}
markedStates.Add( s.stateNumber ); // mark this node as completed.
// first add this node
StringTemplate stateST;
if ( s.IsAcceptState )
{
stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) );
}
else
{
stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) );
}
stateST.SetAttribute( "name", GetStateLabel( s ) );
dot.SetAttribute( "states", stateST );
if ( s.IsAcceptState )
{
return; // don't go past end of rule node to the follow states
}
// special case: if decision point, then line up the alt start states
// unless it's an end of block
if ( ( (NFAState)s ).IsDecisionState )
{
GrammarAST n = ( (NFAState)s ).associatedASTNode;
if ( n != null && n.Type != ANTLRParser.EOB )
{
StringTemplate rankST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "decision-rank" ) );
NFAState alt = (NFAState)s;
while ( alt != null )
{
rankST.SetAttribute( "states", GetStateLabel( alt ) );
if ( alt.transition[1] != null )
{
alt = (NFAState)alt.transition[1].target;
}
else
{
alt = null;
}
}
dot.SetAttribute( "decisionRanks", rankST );
}
}
// make a DOT edge for each transition
StringTemplate edgeST = null;
for ( int i = 0; i < s.NumberOfTransitions; i++ )
{
Transition edge = (Transition)s.GetTransition( i );
if ( edge is RuleClosureTransition )
{
RuleClosureTransition rr = ( (RuleClosureTransition)edge );
// don't jump to other rules, but display edge to follow node
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
if ( rr.rule.grammar != grammar )
{
edgeST.SetAttribute( "label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">" );
}
else
{
edgeST.SetAttribute( "label", "<" + rr.rule.Name + ">" );
}
edgeST.SetAttribute( "src", GetStateLabel( s ) );
edgeST.SetAttribute( "target", GetStateLabel( rr.followState ) );
edgeST.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", edgeST );
WalkRuleNFACreatingDOT( dot, rr.followState );
continue;
}
if ( edge.IsAction )
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "action-edge" ) );
}
else if ( edge.IsEpsilon )
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "epsilon-edge" ) );
}
else
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
}
edgeST.SetAttribute( "label", GetEdgeLabel( edge ) );
edgeST.SetAttribute( "src", GetStateLabel( s ) );
edgeST.SetAttribute( "target", GetStateLabel( edge.target ) );
edgeST.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", edgeST );
WalkRuleNFACreatingDOT( dot, edge.target ); // keep walkin'
}
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes.
*/
protected virtual void WalkCreatingDFADOT( StringTemplate dot,
DFAState s )
{
if ( markedStates.Contains( s.stateNumber ) )
{
return; // already visited this node
}
markedStates.Add( s.stateNumber ); // mark this node as completed.
// first add this node
StringTemplate st;
if ( s.IsAcceptState )
{
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) );
}
else
{
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) );
}
st.SetAttribute( "name", GetStateLabel( s ) );
dot.SetAttribute( "states", st );
// make a DOT edge for each transition
for ( int i = 0; i < s.NumberOfTransitions; i++ )
{
Transition edge = (Transition)s.Transition( i );
//Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s"
// + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions );
if ( StripNonreducedStates )
{
if ( edge.target is DFAState &&
( (DFAState)edge.target ).AcceptStateReachable != DFA.REACHABLE_YES )
{
continue; // don't generate nodes for terminal states
}
}
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
st.SetAttribute( "label", GetEdgeLabel( edge ) );
st.SetAttribute( "src", GetStateLabel( s ) );
st.SetAttribute( "target", GetStateLabel( edge.target ) );
st.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", st );
WalkCreatingDFADOT( dot, (DFAState)edge.target ); // keep walkin'
}
}
public virtual string ToString( StringTemplate messageST )
{
// setup the location
locationST = ErrorManager.GetLocationFormat();
reportST = ErrorManager.GetReportFormat();
messageFormatST = ErrorManager.GetMessageFormat();
bool locationValid = false;
if ( line != -1 )
{
locationST.SetAttribute( "line", line );
locationValid = true;
}
if ( charPositionInLine != -1 )
{
locationST.SetAttribute( "column", charPositionInLine + 1 );
locationValid = true;
}
if ( file != null )
{
locationST.SetAttribute( "file", file );
locationValid = true;
}
messageFormatST.SetAttribute( "id", msgID );
messageFormatST.SetAttribute( "text", messageST );
if ( locationValid )
{
reportST.SetAttribute( "location", locationST );
}
reportST.SetAttribute( "message", messageFormatST );
reportST.SetAttribute( "type", ErrorManager.GetMessageType( msgID ) );
return reportST.ToString();
}
public override StringTemplate GenExpr( CodeGenerator generator,
StringTemplateGroup templates,
DFA dfa )
{
StringTemplate eST = null;
if ( templates != null )
{
if ( _synpred )
{
eST = templates.GetInstanceOf( "evalSynPredicate" );
}
else
{
eST = templates.GetInstanceOf( "evalPredicate" );
generator.grammar.decisionsWhoseDFAsUsesSemPreds.Add( dfa );
}
string predEnclosingRuleName = predicateAST.enclosingRuleName;
/*
String decisionEnclosingRuleName =
dfa.getNFADecisionStartState().getEnclosingRule();
// if these rulenames are diff, then pred was hoisted out of rule
// Currently I don't warn you about this as it could be annoying.
// I do the translation anyway.
*/
//eST.setAttribute("pred", this.toString());
if ( generator != null )
{
eST.SetAttribute( "pred",
generator.TranslateAction( predEnclosingRuleName, predicateAST ) );
}
}
else
{
eST = new StringTemplate( "$pred$" );
eST.SetAttribute( "pred", this.ToString() );
return eST;
}
if ( generator != null )
{
string description =
generator.target.GetTargetStringLiteralFromString( this.ToString() );
eST.SetAttribute( "description", description );
}
return eST;
}
/** Given the grammar to which we are attached, walk the AST associated
* with that grammar to create NFAs. Then create the DFAs for all
* decision points in the grammar by converting the NFAs to DFAs.
* Finally, walk the AST again to generate code.
*
* Either 1 or 2 files are written:
*
* recognizer: the main parser/lexer/treewalker item
* header file: language like C/C++ need extern definitions
*
* The target, such as JavaTarget, dictates which files get written.
*/
public virtual StringTemplate GenRecognizer()
{
//[email protected]("### generate "+grammar.name+" recognizer");
// LOAD OUTPUT TEMPLATES
LoadTemplates( language );
if ( templates == null )
{
return null;
}
// CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA
if ( ErrorManager.DoNotAttemptAnalysis() )
{
return null;
}
target.PerformGrammarAnalysis( this, grammar );
// some grammar analysis errors will not yield reliable DFA
if ( ErrorManager.DoNotAttemptCodeGen() )
{
return null;
}
// OPTIMIZE DFA
DFAOptimizer optimizer = new DFAOptimizer( grammar );
optimizer.Optimize();
// OUTPUT FILE (contains recognizerST)
outputFileST = templates.GetInstanceOf( "outputFile" );
// HEADER FILE
if ( templates.IsDefined( "headerFile" ) )
{
headerFileST = templates.GetInstanceOf( "headerFile" );
}
else
{
// create a dummy to avoid null-checks all over code generator
headerFileST = new StringTemplate( templates, "" );
headerFileST.Name = "dummy-header-file";
}
bool filterMode = grammar.GetOption( "filter" ) != null &&
grammar.GetOption( "filter" ).Equals( "true" );
bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption ||
grammar.SyntacticPredicates != null ||
filterMode;
// TODO: move this down further because generating the recognizer
// alters the model with info on who uses predefined properties etc...
// The actions here might refer to something.
// The only two possible output files are available at this point.
// Verify action scopes are ok for target and dump actions into output
// Templates can say <actions.parser.header> for example.
var actions = grammar.Actions;
VerifyActionScopesOkForTarget( actions );
// translate $x::y references
TranslateActionAttributeReferences( actions );
StringTemplate gateST = templates.GetInstanceOf( "actionGate" );
if ( filterMode )
{
// if filtering, we need to set actions to execute at backtracking
// level 1 not 0.
gateST = templates.GetInstanceOf( "filteringActionGate" );
}
grammar.SetSynPredGateIfNotAlready( gateST );
headerFileST.SetAttribute( "actions", actions );
outputFileST.SetAttribute( "actions", actions );
headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
headerFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
outputFileST.SetAttribute( "backtracking", canBacktrack );
headerFileST.SetAttribute( "backtracking", canBacktrack );
// turn on memoize attribute at grammar level so we can create ruleMemo.
// each rule has memoize attr that hides this one, indicating whether
// it needs to save results
string memoize = (string)grammar.GetOption( "memoize" );
outputFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
headerFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
outputFileST.SetAttribute( "trace", trace );
headerFileST.SetAttribute( "trace", trace );
outputFileST.SetAttribute( "profile", profile );
headerFileST.SetAttribute( "profile", profile );
// RECOGNIZER
if ( grammar.type == GrammarType.Lexer )
{
recognizerST = templates.GetInstanceOf( "lexer" );
outputFileST.SetAttribute( "LEXER", true );
headerFileST.SetAttribute( "LEXER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
else if ( grammar.type == GrammarType.Parser ||
grammar.type == GrammarType.Combined )
{
recognizerST = templates.GetInstanceOf( "parser" );
outputFileST.SetAttribute( "PARSER", true );
headerFileST.SetAttribute( "PARSER", true );
}
else
{
recognizerST = templates.GetInstanceOf( "treeParser" );
outputFileST.SetAttribute( "TREE_PARSER", true );
headerFileST.SetAttribute( "TREE_PARSER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
outputFileST.SetAttribute( "recognizer", recognizerST );
headerFileST.SetAttribute( "recognizer", recognizerST );
outputFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
headerFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
string targetAppropriateFileNameString =
target.GetTargetStringLiteralFromString( grammar.FileName );
outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
{
// GENERATE RECOGNIZER
// Walk the AST holding the input grammar, this time generating code
// Decisions are generated by using the precomputed DFAs
// Fill in the various templates with data
CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) );
try
{
gen.grammar_( grammar,
recognizerST,
outputFileST,
headerFileST );
}
catch ( RecognitionException re )
{
ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE,
re );
}
}
GenTokenTypeConstants( recognizerST );
GenTokenTypeConstants( outputFileST );
GenTokenTypeConstants( headerFileST );
if ( grammar.type != GrammarType.Lexer )
{
GenTokenTypeNames( recognizerST );
GenTokenTypeNames( outputFileST );
GenTokenTypeNames( headerFileST );
}
// Now that we know what synpreds are used, we can set into template
HashSet<string> synpredNames = null;
if ( grammar.synPredNamesUsedInDFA.Count > 0 )
{
synpredNames = grammar.synPredNamesUsedInDFA;
}
outputFileST.SetAttribute( "synpreds", synpredNames );
headerFileST.SetAttribute( "synpreds", synpredNames );
// all recognizers can see Grammar object
recognizerST.SetAttribute( "grammar", grammar );
// WRITE FILES
try
{
target.GenRecognizerFile( tool, this, grammar, outputFileST );
if ( templates.IsDefined( "headerFile" ) )
{
StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" );
target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() );
}
// write out the vocab interchange file; used by antlr,
// does not change per target
StringTemplate tokenVocabSerialization = GenTokenVocabOutput();
string vocabFileName = VocabFileName;
if ( vocabFileName != null )
{
Write( tokenVocabSerialization, vocabFileName );
}
//[email protected](outputFileST.getDOTForDependencyGraph(false));
}
catch ( IOException ioe )
{
ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE,
VocabFileName,
ioe );
}
/*
[email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs);
[email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups);
*/
return outputFileST;
}
/** Generate a token vocab file with all the token names/types. For example:
* ID=7
* FOR=8
* 'for'=8
*
* This is independent of the target language; used by antlr internally
*/
protected virtual StringTemplate GenTokenVocabOutput()
{
StringTemplate vocabFileST =
new StringTemplate( vocabFilePattern,
typeof( AngleBracketTemplateLexer ) );
vocabFileST.Name = "vocab-file";
// make constants for the token names
foreach ( string tokenID in grammar.TokenIDs )
{
int tokenType = grammar.GetTokenType( tokenID );
if ( tokenType >= Label.MIN_TOKEN_TYPE )
{
vocabFileST.SetAttribute( "tokens.{name,type}", tokenID, tokenType );
}
}
// now dump the strings
foreach ( string literal in grammar.StringLiterals )
{
int tokenType = grammar.GetTokenType( literal );
if ( tokenType >= Label.MIN_TOKEN_TYPE )
{
vocabFileST.SetAttribute( "tokens.{name,type}", literal, tokenType );
}
}
return vocabFileST;
}
/** Generate a token names table that maps token type to a printable
* name: either the label like INT or the literal like "begin".
*/
protected virtual void GenTokenTypeNames( StringTemplate code )
{
for ( int t = Label.MIN_TOKEN_TYPE; t <= grammar.MaxTokenType; t++ )
{
string tokenName = grammar.GetTokenDisplayName( t );
if ( tokenName != null )
{
tokenName = target.GetTargetStringLiteralFromString( tokenName, true );
code.SetAttribute( "tokenNames", tokenName );
}
}
}
// T O K E N D E F I N I T I O N G E N E R A T I O N
/** Set attributes tokens and literals attributes in the incoming
* code template. This is not the token vocab interchange file, but
* rather a list of token type ID needed by the recognizer.
*/
protected virtual void GenTokenTypeConstants( StringTemplate code )
{
// make constants for the token types
foreach ( string tokenID in grammar.TokenIDs )
{
int tokenType = grammar.GetTokenType( tokenID );
if ( tokenType == Label.EOF ||
tokenType >= Label.MIN_TOKEN_TYPE )
{
// don't do FAUX labels 'cept EOF
code.SetAttribute( "tokens.{name,type}", tokenID, tokenType );
}
}
}
/** Translate an action like [3,"foo",a[3]] and return a List of the
* translated actions. Because actions are themselves translated to a list
* of chunks, must cat together into a StringTemplate>. Don't translate
* to strings early as we need to eval templates in context.
*/
public virtual List<StringTemplate> TranslateArgAction( string ruleName,
GrammarAST actionTree )
{
string actionText = actionTree.Token.Text;
List<string> args = GetListOfArgumentsFromAction( actionText, ',' );
List<StringTemplate> translatedArgs = new List<StringTemplate>();
foreach ( string arg in args )
{
if ( arg != null )
{
IToken actionToken =
new CommonToken( ANTLRParser.ACTION, arg );
ActionTranslator translator =
new ActionTranslator( this, ruleName,
actionToken,
actionTree.outerAltNum );
IList chunks = translator.TranslateToChunks();
chunks = target.PostProcessAction( chunks, actionToken );
StringTemplate catST = new StringTemplate( templates, "<chunks>" );
catST.SetAttribute( "chunks", chunks );
templates.CreateStringTemplate();
translatedArgs.Add( catST );
}
}
if ( translatedArgs.Count == 0 )
{
return null;
}
return translatedArgs;
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes. We know this is an NFA
* for a rule so don't traverse edges to other rules and
* don't go past rule end state.
*/
protected virtual void WalkRuleNFACreatingDOT(StringTemplate dot,
State s)
{
if (markedStates.Contains(s.stateNumber))
{
return; // already visited this node
}
markedStates.Add(s.stateNumber); // mark this node as completed.
// first add this node
StringTemplate stateST;
if (s.IsAcceptState)
{
stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate"));
}
else
{
stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state"));
}
stateST.SetAttribute("name", GetStateLabel(s));
dot.SetAttribute("states", stateST);
if (s.IsAcceptState)
{
return; // don't go past end of rule node to the follow states
}
// special case: if decision point, then line up the alt start states
// unless it's an end of block
if (((NFAState)s).IsDecisionState)
{
GrammarAST n = ((NFAState)s).associatedASTNode;
if (n != null && n.Type != ANTLRParser.EOB)
{
StringTemplate rankST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "decision-rank"));
NFAState alt = (NFAState)s;
while (alt != null)
{
rankST.SetAttribute("states", GetStateLabel(alt));
if (alt.transition[1] != null)
{
alt = (NFAState)alt.transition[1].target;
}
else
{
alt = null;
}
}
dot.SetAttribute("decisionRanks", rankST);
}
}
// make a DOT edge for each transition
StringTemplate edgeST = null;
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = (Transition)s.GetTransition(i);
if (edge is RuleClosureTransition)
{
RuleClosureTransition rr = ((RuleClosureTransition)edge);
// don't jump to other rules, but display edge to follow node
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge"));
if (rr.rule.grammar != grammar)
{
edgeST.SetAttribute("label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">");
}
else
{
edgeST.SetAttribute("label", "<" + rr.rule.Name + ">");
}
edgeST.SetAttribute("src", GetStateLabel(s));
edgeST.SetAttribute("target", GetStateLabel(rr.followState));
edgeST.SetAttribute("arrowhead", arrowhead);
dot.SetAttribute("edges", edgeST);
WalkRuleNFACreatingDOT(dot, rr.followState);
continue;
}
if (edge.IsAction)
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "action-edge"));
}
else if (edge.IsEpsilon)
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "epsilon-edge"));
}
else
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge"));
}
edgeST.SetAttribute("label", GetEdgeLabel(edge));
edgeST.SetAttribute("src", GetStateLabel(s));
edgeST.SetAttribute("target", GetStateLabel(edge.target));
edgeST.SetAttribute("arrowhead", arrowhead);
dot.SetAttribute("edges", edgeST);
WalkRuleNFACreatingDOT(dot, edge.target); // keep walkin'
}
}
public void TestEscapedLessThanInAction()
{
Grammar g = new Grammar();
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
string action = "i<3; '<xmltag>'";
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 0 );
string expecting = action;
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, "<action>" );
actionST.SetAttribute( "action", rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
// Now fill template with information about problemState
var labels = probe.GetSampleNonDeterministicInputSequence(problemState);
string input = probe.GetInputSequenceDisplay(labels);
st.SetAttribute("input", input);
if (probe.dfa.IsTokensRuleDecision)
{
var disabledAlts = probe.GetDisabledAlternatives(problemState);
foreach (int altI in disabledAlts)
{
string tokenName =
probe.GetTokenNameForTokensRuleAlt((int)altI);
// reset the line/col to the token definition (pick last one)
NFAState ruleStart =
probe.dfa.nfa.grammar.GetRuleStartState(tokenName);
line = ruleStart.associatedASTNode.Line;
charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine;
st.SetAttribute("disabled", tokenName);
}
}
else
{
st.SetAttribute("disabled", probe.GetDisabledAlternatives(problemState));
}
var nondetAlts = probe.GetNonDeterministicAltsForState(problemState);
NFAState nfaStart = probe.dfa.NFADecisionStartState;
// all state paths have to begin with same NFA state
int firstAlt = 0;
if (nondetAlts != null)
{
foreach (int displayAltI in nondetAlts)
{
if (DecisionProbe.verbose)
{
int tracePathAlt =
nfaStart.TranslateDisplayAltToWalkAlt((int)displayAltI);
if (firstAlt == 0)
{
firstAlt = tracePathAlt;
}
IList path =
probe.GetNFAPathStatesForAlt(firstAlt,
tracePathAlt,
labels);
st.SetAttribute("paths.{alt,states}",
displayAltI, path);
}
else
{
if (probe.dfa.IsTokensRuleDecision)
{
// alts are token rules, convert to the names instead of numbers
string tokenName =
probe.GetTokenNameForTokensRuleAlt((int)displayAltI);
st.SetAttribute("conflictingTokens", tokenName);
}
else
{
st.SetAttribute("conflictingAlts", displayAltI);
}
}
}
}
st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction);
return(base.ToString(st));
}
protected virtual StringTemplate WalkFixedDFAGeneratingStateMachine(
StringTemplateGroup templates,
DFA dfa,
DFAState s,
int k)
{
//System.Console.Out.WriteLine( "walk " + s.stateNumber + " in dfa for decision " + dfa.decisionNumber );
if (s.IsAcceptState)
{
StringTemplate dfaST2 = templates.GetInstanceOf("dfaAcceptState");
dfaST2.SetAttribute("alt", s.GetUniquelyPredictedAlt());
return(dfaST2);
}
// the default templates for generating a state and its edges
// can be an if-then-else structure or a switch
string dfaStateName = "dfaState";
string dfaLoopbackStateName = "dfaLoopbackState";
string dfaOptionalBlockStateName = "dfaOptionalBlockState";
string dfaEdgeName = "dfaEdge";
if (parentGenerator.CanGenerateSwitch(s))
{
dfaStateName = "dfaStateSwitch";
dfaLoopbackStateName = "dfaLoopbackStateSwitch";
dfaOptionalBlockStateName = "dfaOptionalBlockStateSwitch";
dfaEdgeName = "dfaEdgeSwitch";
}
StringTemplate dfaST = templates.GetInstanceOf(dfaStateName);
if (dfa.NFADecisionStartState.decisionStateType == NFAState.LOOPBACK)
{
dfaST = templates.GetInstanceOf(dfaLoopbackStateName);
}
else if (dfa.NFADecisionStartState.decisionStateType == NFAState.OPTIONAL_BLOCK_START)
{
dfaST = templates.GetInstanceOf(dfaOptionalBlockStateName);
}
dfaST.SetAttribute("k", k);
dfaST.SetAttribute("stateNumber", s.stateNumber);
dfaST.SetAttribute("semPredState",
s.IsResolvedWithPredicates);
/*
* string description = dfa.getNFADecisionStartState().Description;
* description = parentGenerator.target.getTargetStringLiteralFromString( description );
* //System.Console.Out.WriteLine( "DFA: " + description + " associated with AST " + dfa.getNFADecisionStartState() );
* if ( description != null )
* {
* dfaST.SetAttribute( "description", description );
* }
*/
int EOTPredicts = NFA.INVALID_ALT_NUMBER;
DFAState EOTTarget = null;
//System.Console.Out.WriteLine( "DFA state " + s.stateNumber );
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = (Transition)s.Transition(i);
//System.Console.Out.WriteLine( "edge " + s.stateNumber + "-" + edge.label.ToString() + "->" + edge.target.stateNumber );
if (edge.label.Atom == Label.EOT)
{
// don't generate a real edge for EOT; track alt EOT predicts
// generate that prediction in the else clause as default case
EOTTarget = (DFAState)edge.target;
EOTPredicts = EOTTarget.GetUniquelyPredictedAlt();
/*
* System.Console.Out.WriteLine("DFA s"+s.stateNumber+" EOT goes to s"+
* edge.target.stateNumber+" predicates alt "+
* EOTPredicts);
*/
continue;
}
StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName);
// If the template wants all the label values delineated, do that
if (edgeST.GetFormalArgument("labels") != null)
{
List <string> labels = edge.Label.Set.Select(value => parentGenerator.GetTokenTypeAsTargetLabel(value)).ToList();
edgeST.SetAttribute("labels", labels);
}
else
{ // else create an expression to evaluate (the general case)
edgeST.SetAttribute("labelExpr",
parentGenerator.GenLabelExpr(templates, edge, k));
}
// stick in any gated predicates for any edge if not already a pred
if (!edge.label.IsSemanticPredicate)
{
DFAState target = (DFAState)edge.target;
SemanticContext preds =
target.GetGatedPredicatesInNFAConfigurations();
if (preds != null)
{
//System.Console.Out.WriteLine( "preds=" + target.getGatedPredicatesInNFAConfigurations() );
StringTemplate predST = preds.GenExpr(parentGenerator,
parentGenerator.Templates,
dfa);
edgeST.SetAttribute("predicates", predST);
}
}
StringTemplate targetST =
WalkFixedDFAGeneratingStateMachine(templates,
dfa,
(DFAState)edge.target,
k + 1);
edgeST.SetAttribute("targetState", targetST);
dfaST.SetAttribute("edges", edgeST);
//System.Console.Out.WriteLine( "back to DFA " + dfa.decisionNumber + "." + s.stateNumber );
}
// HANDLE EOT EDGE
if (EOTPredicts != NFA.INVALID_ALT_NUMBER)
{
// EOT unique predicts an alt
dfaST.SetAttribute("eotPredictsAlt", EOTPredicts);
}
else if (EOTTarget != null && EOTTarget.NumberOfTransitions > 0)
{
// EOT state has transitions so must split on predicates.
// Generate predicate else-if clauses and then generate
// NoViableAlt exception as else clause.
// Note: these predicates emanate from the EOT target state
// rather than the current DFAState s so the error message
// might be slightly misleading if you are looking at the
// state number. Predicates emanating from EOT targets are
// hoisted up to the state that has the EOT edge.
for (int i = 0; i < EOTTarget.NumberOfTransitions; i++)
{
Transition predEdge = (Transition)EOTTarget.Transition(i);
StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName);
edgeST.SetAttribute("labelExpr",
parentGenerator.GenSemanticPredicateExpr(templates, predEdge));
// the target must be an accept state
//System.Console.Out.WriteLine( "EOT edge" );
StringTemplate targetST =
WalkFixedDFAGeneratingStateMachine(templates,
dfa,
(DFAState)predEdge.target,
k + 1);
edgeST.SetAttribute("targetState", targetST);
dfaST.SetAttribute("edges", edgeST);
}
}
return(dfaST);
}
