public string Convert(Model.TemplateSource.PropertyModel source)
{
string result = "";
string templateString = sourceStructure.GetPropertyTemplate(source.TypeName);
Antlr3.ST.StringTemplate template = new Antlr3.ST.StringTemplate();
template.Template = templateString;
template.SetAttribute("property", source);
result = template.ToString();
return result;
}
public void TestNewlineLiterals() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar T;\n" +
"A : '\\n\\n' ;\n" // ANTLR sees '\n\n'
);
string expecting = "match(\"\\n\\n\")";
AntlrTool antlr = newTool();
antlr.SetOutputDirectory(null); // write to /dev/null
CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
g.CodeGenerator = generator;
generator.GenRecognizer(); // codegen phase sets some vars we need
StringTemplate codeST = generator.RecognizerST;
string code = codeST.ToString();
int m = code.IndexOf("match(\"");
string found = code.Substring(m, expecting.Length);
assertEquals(expecting, found);
}
public void TestIndirectTemplateConstructor() /*throws Exception*/
{
string action = "x = %({\"foo\"})(name={$ID.text});";
string expecting = "x = templateLib.getInstanceOf(\"foo\"," +
LINE_SEP + " new STAttrMap().put(\"name\", (ID1!=null?ID1.getText():null)));";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
Grammar g = new Grammar(
"grammar t;\n" +
"options {\n" +
" output=template;\n" +
"}\n" +
"\n" +
"a : ID {" + action + "}\n" +
" ;\n" +
"\n" +
"ID : 'a';\n");
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator =
new ActionTranslator(generator,
"a",
new CommonToken(ANTLRParser.ACTION, action), 1);
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup(".", typeof(AngleBracketTemplateLexer));
StringTemplate actionST = new StringTemplate(templates, rawTranslation);
string found = actionST.ToString();
assertNoErrors(equeue);
assertEquals(expecting, found);
}
public void TestSetAttr() /*throws Exception*/
{
string action = "%x.y = z;";
string expecting = "(x).setAttribute(\"y\", z);";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
Grammar g = new Grammar(
"grammar t;\n" +
"options {\n" +
" output=template;\n" +
"}\n" +
"\n" +
"a : ID {" + action + "}\n" +
" ;\n" +
"\n" +
"ID : 'a';\n");
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator =
new ActionTranslator(generator,
"a",
new CommonToken(ANTLRParser.ACTION, action), 1);
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup(".", typeof(AngleBracketTemplateLexer));
StringTemplate actionST = new StringTemplate(templates, rawTranslation);
string found = actionST.ToString();
assertNoErrors(equeue);
assertEquals(expecting, found);
}
public override String ToString()
{
line = 0;
charPositionInLine = 0;
if (offendingToken != null)
{
line = offendingToken.Line;
charPositionInLine = offendingToken.CharPositionInLine;
}
// TODO: actually set the right Grammar instance to get the filename
// TODO: have to update all v2 grammar files for this. or use errormanager and tool to get the current grammar
if (g != null)
{
file = g.FileName;
}
StringTemplate st = GetMessageTemplate();
if (arg != null)
{
st.SetAttribute("arg", arg);
}
return(base.ToString(st));
}
public override String ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
String fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
if (probe.dfa.IsTokensRuleDecision)
{
// alts are token rules, convert to the names instead of numbers
for (int i = 0; i < alts.Length; i++)
{
int altI = alts[i];
String tokenName = probe.GetTokenNameForTokensRuleAlt(altI);
// reset the line/col to the token definition
NFAState ruleStart = probe.dfa.nfa.grammar.GetRuleStartState(tokenName);
line = ruleStart.associatedASTNode.Line;
charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine;
st.SetAttribute("tokens", tokenName);
}
}
else
{
// regular alt numbers, show the alts
st.SetAttribute("alts", alts);
}
return(base.ToString(st));
}
public void TestRefToRuleWithNoReturnValue() /*throws Exception*/
{
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
string grammarStr =
"grammar P;\n" +
"a : x=b ;\n" +
"b : B ;\n" +
"B : 'b' ;\n";
Grammar g = new Grammar(grammarStr);
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
g.CodeGenerator = generator;
StringTemplate recogST = generator.GenRecognizer();
string code = recogST.ToString();
assertTrue("not expecting label", code.IndexOf("x=b();") < 0);
assertEquals("unexpected errors: " + equeue, 0, equeue.errors.Count);
}
public virtual string ToString(StringTemplate messageST)
{
// setup the location
locationST = ErrorManager.GetLocationFormat();
reportST = ErrorManager.GetReportFormat();
messageFormatST = ErrorManager.GetMessageFormat();
bool locationValid = false;
if (line != -1)
{
locationST.SetAttribute("line", line);
locationValid = true;
}
if (charPositionInLine != -1)
{
locationST.SetAttribute("column", charPositionInLine + 1);
locationValid = true;
}
if (file != null)
{
locationST.SetAttribute("file", file);
locationValid = true;
}
messageFormatST.SetAttribute("id", msgID);
messageFormatST.SetAttribute("text", messageST);
if (locationValid)
{
reportST.SetAttribute("location", locationST);
}
reportST.SetAttribute("message", messageFormatST);
reportST.SetAttribute("type", ErrorManager.GetMessageType(msgID));
return(reportST.ToString());
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
var labels = probe.GetSampleNonDeterministicInputSequence(problemState);
string input = probe.GetInputSequenceDisplay(labels);
StringTemplate st = GetMessageTemplate();
List <int> alts = new List <int>();
alts.addAll(problemState.AltSet);
alts.Sort();
st.SetAttribute("danglingAlts", alts);
st.SetAttribute("input", input);
return(base.ToString(st));
}
protected override void GenRecognizerHeaderFile( AntlrTool tool,
CodeGenerator generator,
Grammar grammar,
StringTemplate headerFileST,
string extName )
{
StringTemplateGroup templates = generator.Templates;
generator.Write( headerFileST, grammar.name + extName );
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes. We know this is an NFA
* for a rule so don't traverse edges to other rules and
* don't go past rule end state.
*/
protected virtual void WalkRuleNFACreatingDOT(StringTemplate dot,
State s)
{
if (markedStates.Contains(s.stateNumber))
{
return; // already visited this node
}
markedStates.Add(s.stateNumber); // mark this node as completed.
// first add this node
StringTemplate stateST;
if (s.IsAcceptState)
{
stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "stopstate"));
}
else
{
stateST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "state"));
}
stateST.SetAttribute("name", GetStateLabel(s));
dot.SetAttribute("states", stateST);
if (s.IsAcceptState)
{
return; // don't go past end of rule node to the follow states
}
// special case: if decision point, then line up the alt start states
// unless it's an end of block
if (((NFAState)s).IsDecisionState)
{
GrammarAST n = ((NFAState)s).associatedASTNode;
if (n != null && n.Type != ANTLRParser.EOB)
{
StringTemplate rankST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "decision-rank"));
NFAState alt = (NFAState)s;
while (alt != null)
{
rankST.SetAttribute("states", GetStateLabel(alt));
if (alt.transition[1] != null)
{
alt = (NFAState)alt.transition[1].target;
}
else
{
alt = null;
}
}
dot.SetAttribute("decisionRanks", rankST);
}
}
// make a DOT edge for each transition
StringTemplate edgeST = null;
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = (Transition)s.GetTransition(i);
if (edge is RuleClosureTransition)
{
RuleClosureTransition rr = ((RuleClosureTransition)edge);
// don't jump to other rules, but display edge to follow node
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge"));
if (rr.rule.grammar != grammar)
{
edgeST.SetAttribute("label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">");
}
else
{
edgeST.SetAttribute("label", "<" + rr.rule.Name + ">");
}
edgeST.SetAttribute("src", GetStateLabel(s));
edgeST.SetAttribute("target", GetStateLabel(rr.followState));
edgeST.SetAttribute("arrowhead", arrowhead);
dot.SetAttribute("edges", edgeST);
WalkRuleNFACreatingDOT(dot, rr.followState);
continue;
}
if (edge.IsAction)
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "action-edge"));
}
else if (edge.IsEpsilon)
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "epsilon-edge"));
}
else
{
edgeST = stlib.GetInstanceOf(Path.Combine(dfaTemplateDirectoryName, "edge"));
}
edgeST.SetAttribute("label", GetEdgeLabel(edge));
edgeST.SetAttribute("src", GetStateLabel(s));
edgeST.SetAttribute("target", GetStateLabel(edge.target));
edgeST.SetAttribute("arrowhead", arrowhead);
dot.SetAttribute("edges", edgeST);
WalkRuleNFACreatingDOT(dot, edge.target); // keep walkin'
}
}
/** Generate a token vocab file with all the token names/types. For example:
* ID=7
* FOR=8
* 'for'=8
*
* This is independent of the target language; used by antlr internally
*/
protected virtual StringTemplate GenTokenVocabOutput()
{
StringTemplate vocabFileST =
new StringTemplate( vocabFilePattern,
typeof( AngleBracketTemplateLexer ) );
vocabFileST.Name = "vocab-file";
// make constants for the token names
foreach ( string tokenID in grammar.TokenIDs )
{
int tokenType = grammar.GetTokenType( tokenID );
if ( tokenType >= Label.MIN_TOKEN_TYPE )
{
vocabFileST.SetAttribute( "tokens.{name,type}", tokenID, tokenType );
}
}
// now dump the strings
foreach ( string literal in grammar.StringLiterals )
{
int tokenType = grammar.GetTokenType( literal );
if ( tokenType >= Label.MIN_TOKEN_TYPE )
{
vocabFileST.SetAttribute( "tokens.{name,type}", literal, tokenType );
}
}
return vocabFileST;
}
/** Given the grammar to which we are attached, walk the AST associated
* with that grammar to create NFAs. Then create the DFAs for all
* decision points in the grammar by converting the NFAs to DFAs.
* Finally, walk the AST again to generate code.
*
* Either 1 or 2 files are written:
*
* recognizer: the main parser/lexer/treewalker item
* header file: language like C/C++ need extern definitions
*
* The target, such as JavaTarget, dictates which files get written.
*/
public virtual StringTemplate GenRecognizer()
{
//[email protected]("### generate "+grammar.name+" recognizer");
// LOAD OUTPUT TEMPLATES
LoadTemplates( language );
if ( templates == null )
{
return null;
}
// CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA
if ( ErrorManager.DoNotAttemptAnalysis() )
{
return null;
}
target.PerformGrammarAnalysis( this, grammar );
// some grammar analysis errors will not yield reliable DFA
if ( ErrorManager.DoNotAttemptCodeGen() )
{
return null;
}
// OPTIMIZE DFA
DFAOptimizer optimizer = new DFAOptimizer( grammar );
optimizer.Optimize();
// OUTPUT FILE (contains recognizerST)
outputFileST = templates.GetInstanceOf( "outputFile" );
// HEADER FILE
if ( templates.IsDefined( "headerFile" ) )
{
headerFileST = templates.GetInstanceOf( "headerFile" );
}
else
{
// create a dummy to avoid null-checks all over code generator
headerFileST = new StringTemplate( templates, "" );
headerFileST.Name = "dummy-header-file";
}
bool filterMode = grammar.GetOption( "filter" ) != null &&
grammar.GetOption( "filter" ).Equals( "true" );
bool canBacktrack = grammar.composite.GetRootGrammar().atLeastOneBacktrackOption ||
grammar.SyntacticPredicates != null ||
filterMode;
// TODO: move this down further because generating the recognizer
// alters the model with info on who uses predefined properties etc...
// The actions here might refer to something.
// The only two possible output files are available at this point.
// Verify action scopes are ok for target and dump actions into output
// Templates can say <actions.parser.header> for example.
var actions = grammar.Actions;
VerifyActionScopesOkForTarget( actions );
// translate $x::y references
TranslateActionAttributeReferences( actions );
StringTemplate gateST = templates.GetInstanceOf( "actionGate" );
if ( filterMode )
{
// if filtering, we need to set actions to execute at backtracking
// level 1 not 0.
gateST = templates.GetInstanceOf( "filteringActionGate" );
}
grammar.SetSynPredGateIfNotAlready( gateST );
headerFileST.SetAttribute( "actions", actions );
outputFileST.SetAttribute( "actions", actions );
headerFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
outputFileST.SetAttribute( "buildTemplate", grammar.BuildTemplate );
headerFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "buildAST", grammar.BuildAST );
outputFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
headerFileST.SetAttribute( "rewriteMode", grammar.RewriteMode );
outputFileST.SetAttribute( "backtracking", canBacktrack );
headerFileST.SetAttribute( "backtracking", canBacktrack );
// turn on memoize attribute at grammar level so we can create ruleMemo.
// each rule has memoize attr that hides this one, indicating whether
// it needs to save results
string memoize = (string)grammar.GetOption( "memoize" );
outputFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
headerFileST.SetAttribute( "memoize",
( grammar.atLeastOneRuleMemoizes ||
( memoize != null && memoize.Equals( "true" ) ) &&
canBacktrack ) );
outputFileST.SetAttribute( "trace", trace );
headerFileST.SetAttribute( "trace", trace );
outputFileST.SetAttribute( "profile", profile );
headerFileST.SetAttribute( "profile", profile );
// RECOGNIZER
if ( grammar.type == GrammarType.Lexer )
{
recognizerST = templates.GetInstanceOf( "lexer" );
outputFileST.SetAttribute( "LEXER", true );
headerFileST.SetAttribute( "LEXER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
else if ( grammar.type == GrammarType.Parser ||
grammar.type == GrammarType.Combined )
{
recognizerST = templates.GetInstanceOf( "parser" );
outputFileST.SetAttribute( "PARSER", true );
headerFileST.SetAttribute( "PARSER", true );
}
else
{
recognizerST = templates.GetInstanceOf( "treeParser" );
outputFileST.SetAttribute( "TREE_PARSER", true );
headerFileST.SetAttribute( "TREE_PARSER", true );
recognizerST.SetAttribute( "filterMode", filterMode );
}
outputFileST.SetAttribute( "recognizer", recognizerST );
headerFileST.SetAttribute( "recognizer", recognizerST );
outputFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
headerFileST.SetAttribute( "actionScope",
grammar.GetDefaultActionScope( grammar.type ) );
string targetAppropriateFileNameString =
target.GetTargetStringLiteralFromString( grammar.FileName );
outputFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
headerFileST.SetAttribute( "fileName", targetAppropriateFileNameString );
outputFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
headerFileST.SetAttribute( "ANTLRVersion", AntlrTool.AssemblyVersion );
outputFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
headerFileST.SetAttribute( "generatedTimestamp", AntlrTool.GetCurrentTimeStamp() );
{
// GENERATE RECOGNIZER
// Walk the AST holding the input grammar, this time generating code
// Decisions are generated by using the precomputed DFAs
// Fill in the various templates with data
CodeGenTreeWalker gen = new CodeGenTreeWalker( new Antlr.Runtime.Tree.CommonTreeNodeStream( grammar.Tree ) );
try
{
gen.grammar_( grammar,
recognizerST,
outputFileST,
headerFileST );
}
catch ( RecognitionException re )
{
ErrorManager.Error( ErrorManager.MSG_BAD_AST_STRUCTURE,
re );
}
}
GenTokenTypeConstants( recognizerST );
GenTokenTypeConstants( outputFileST );
GenTokenTypeConstants( headerFileST );
if ( grammar.type != GrammarType.Lexer )
{
GenTokenTypeNames( recognizerST );
GenTokenTypeNames( outputFileST );
GenTokenTypeNames( headerFileST );
}
// Now that we know what synpreds are used, we can set into template
HashSet<string> synpredNames = null;
if ( grammar.synPredNamesUsedInDFA.Count > 0 )
{
synpredNames = grammar.synPredNamesUsedInDFA;
}
outputFileST.SetAttribute( "synpreds", synpredNames );
headerFileST.SetAttribute( "synpreds", synpredNames );
// all recognizers can see Grammar object
recognizerST.SetAttribute( "grammar", grammar );
// WRITE FILES
try
{
target.GenRecognizerFile( tool, this, grammar, outputFileST );
if ( templates.IsDefined( "headerFile" ) )
{
StringTemplate extST = templates.GetInstanceOf( "headerFileExtension" );
target.GenRecognizerHeaderFile( tool, this, grammar, headerFileST, extST.ToString() );
}
// write out the vocab interchange file; used by antlr,
// does not change per target
StringTemplate tokenVocabSerialization = GenTokenVocabOutput();
string vocabFileName = VocabFileName;
if ( vocabFileName != null )
{
Write( tokenVocabSerialization, vocabFileName );
}
//[email protected](outputFileST.getDOTForDependencyGraph(false));
}
catch ( IOException ioe )
{
ErrorManager.Error( ErrorManager.MSG_CANNOT_WRITE_FILE,
VocabFileName,
ioe );
}
/*
[email protected]("num obj.prop refs: "+ ASTExpr.totalObjPropRefs);
[email protected]("num reflection lookups: "+ ASTExpr.totalReflectionLookups);
*/
return outputFileST;
}
/** Translate an action like [3,"foo",a[3]] and return a List of the
* translated actions. Because actions are themselves translated to a list
* of chunks, must cat together into a StringTemplate>. Don't translate
* to strings early as we need to eval templates in context.
*/
public virtual List<StringTemplate> TranslateArgAction( string ruleName,
GrammarAST actionTree )
{
string actionText = actionTree.Token.Text;
List<string> args = GetListOfArgumentsFromAction( actionText, ',' );
List<StringTemplate> translatedArgs = new List<StringTemplate>();
foreach ( string arg in args )
{
if ( arg != null )
{
IToken actionToken =
new CommonToken( ANTLRParser.ACTION, arg );
ActionTranslator translator =
new ActionTranslator( this, ruleName,
actionToken,
actionTree.outerAltNum );
IList chunks = translator.TranslateToChunks();
chunks = target.PostProcessAction( chunks, actionToken );
StringTemplate catST = new StringTemplate( templates, "<chunks>" );
catST.SetAttribute( "chunks", chunks );
templates.CreateStringTemplate();
translatedArgs.Add( catST );
}
}
if ( translatedArgs.Count == 0 )
{
return null;
}
return translatedArgs;
}
// T O K E N D E F I N I T I O N G E N E R A T I O N
/** Set attributes tokens and literals attributes in the incoming
* code template. This is not the token vocab interchange file, but
* rather a list of token type ID needed by the recognizer.
*/
protected virtual void GenTokenTypeConstants( StringTemplate code )
{
// make constants for the token types
foreach ( string tokenID in grammar.TokenIDs )
{
int tokenType = grammar.GetTokenType( tokenID );
if ( tokenType == Label.EOF ||
tokenType >= Label.MIN_TOKEN_TYPE )
{
// don't do FAUX labels 'cept EOF
code.SetAttribute( "tokens.{name,type}", tokenID, tokenType );
}
}
}
public void TestArguments()
{
string action = "$i; $i.x; $u; $u.x";
string expecting = "i; i.x; u; u.x";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"parser grammar t;\n" +
"a[User u, int i]\n" +
" : {" + action + "}\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestEscapedLessThanInAction()
{
Grammar g = new Grammar();
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
string action = "i<3; '<xmltag>'";
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 0 );
string expecting = action;
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, "<action>" );
actionST.SetAttribute( "action", rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
}
public void TestTokenRefTreeProperty()
{
string action = "$ID.tree;";
string expecting = "ID1_tree;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"a : ID {" + action + "} ;" +
"ID : 'a';\n" );
AntlrTool antlr = newTool();
antlr.SetOutputDirectory( null ); // write to /dev/null
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer();
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
}
public void TestUnknownGlobalDynamicAttribute()
{
string action = "$Symbols::x";
string expecting = action;
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"scope Symbols {\n" +
" int n;\n" +
"}\n" +
"a : {'+action+'}\n" +
" ;\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator =
new ActionTranslator( generator,
"a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
int expectedMsgID = ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE_ATTRIBUTE;
object expectedArg = "Symbols";
object expectedArg2 = "x";
GrammarSemanticsMessage expectedMessage =
new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 );
checkError( equeue, expectedMessage );
}
public void TestTokenLabels()
{
string action = "$id; $f; $id.text; $id.getText(); $id.dork " +
"$id.type; $id.line; $id.pos; " +
"$id.channel; $id.index;";
string expecting = "id; f; (id!=null?id.getText():null); id.getText(); id.dork (id!=null?id.getType():0); (id!=null?id.getLine():0); (id!=null?id.getCharPositionInLine():0); (id!=null?id.getChannel():0); (id!=null?id.getTokenIndex():0);";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"parser grammar t;\n" +
"a : id=ID f=FLOAT {" + action + "}\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestTokenLabelFromMultipleAlts()
{
string action = "$ID.text;"; // must be qualified
string action2 = "$INT.text;"; // must be qualified
string expecting = "(ID1!=null?ID1.getText():null);";
string expecting2 = "(INT2!=null?INT2.getText():null);";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"a : ID {" + action + "}\n" +
" | INT {" + action2 + "}\n" +
" ;\n" +
"ID : 'a';\n" +
"INT : '0';\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
translator = new ActionTranslator( generator,
"a",
new CommonToken( ANTLRParser.ACTION, action2 ), 2 );
rawTranslation =
translator.Translate();
templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
actionST = new StringTemplate( templates, rawTranslation );
found = actionST.ToString();
assertEquals( expecting2, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestSimplePlusEqualLabel()
{
string action = "$ids.size();"; // must be qualified
string expecting = "list_ids.size();";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"parser grammar t;\n" +
"a : ids+=ID ( COMMA ids+=ID {" + action + "})* ;\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator =
new ActionTranslator( generator,
"a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST)
{
/*
* Below is an experimental attempt at providing a few named action blocks
* that are printed in both lexer and parser files from combined grammars.
* ANTLR appears to first generate a parser, then generate an independent lexer,
* and then generate code from that. It keeps the combo/parser grammar object
* and the lexer grammar object, as well as their respective code generator and
* target instances, completely independent. So, while a bit hack-ish, this is
* a solution that should work without having to modify Terrence Parr's
* core tool code.
*
* - sharedActionBlocks is a class variable containing a hash map
* - if this method is called with a combo grammar, and the action map
* in the grammar contains an entry for the named scope "all",
* add an entry to sharedActionBlocks mapping the grammar name to
* the "all" action map.
* - if this method is called with an `implicit lexer'
* (one that's extracted from a combo grammar), check to see if
* there's an entry in sharedActionBlocks for the lexer's grammar name.
* - if there is an action map entry, place it in the lexer's action map
* - the recognizerFile template has code to place the
* "all" actions appropriately
*
* problems:
* - This solution assumes that the parser will be generated
* before the lexer. If that changes at some point, this will
* not work.
* - I have not investigated how this works with delegation yet
*
* Kyle Yetter - March 25, 2010
*/
if (grammar.type == GrammarType.Combined)
{
IDictionary <string, object> all;
if (grammar.Actions.TryGetValue("all", out all))
{
sharedActionBlocks[grammar.name] = all;
}
}
else if (grammar.implicitLexer)
{
IDictionary <string, object> shared;
if (sharedActionBlocks.TryGetValue(grammar.name, out shared))
{
grammar.Actions["all"] = shared;
}
}
generator.Templates.RegisterRenderer(typeof(string), new RubyRenderer());
string fileName = generator.GetRecognizerFileName(grammar.name, grammar.type);
generator.Write(outputFileST, fileName);
}
public void TestUnqualifiedRuleScopeAttribute()
{
string action = "$n;"; // must be qualified
string expecting = "$n;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"a\n" +
"scope {\n" +
" int n;\n" +
"} : b\n" +
" ;\n" +
"b : {'+action+'}\n" +
" ;\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
ActionTranslator translator =
new ActionTranslator( generator,
"b",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
int expectedMsgID = ErrorManager.MSG_UNKNOWN_SIMPLE_ATTRIBUTE;
object expectedArg = "n";
object expectedArg2 = null;
GrammarSemanticsMessage expectedMessage =
new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 );
checkError( equeue, expectedMessage );
}
public void TestSharedGlobalScope()
{
string action = "$Symbols::x;";
string expecting = "((Symbols_scope)Symbols_stack.peek()).x;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"scope Symbols {\n" +
" String x;\n" +
"}\n" +
"a\n" +
"scope { int y; }\n" +
"scope Symbols;\n" +
" : b {" + action + "}\n" +
" ;\n" +
"b : ID {$Symbols::x=$ID.text} ;\n" +
"ID : 'a';\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestAssignToOwnRulenameAttr()
{
string action = "$rule.tree = null;";
string expecting = "retval.tree = null;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar a;\n" +
"rule\n" +
" : 'y' {" + action + "}\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator,
"rule",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestEscaped_InAction()
{
string action = "int \\$n; \"\\$in string\\$\"";
string expecting = "int $n; \"$in string$\"";
Grammar g = new Grammar(
"parser grammar t;\n" +
"@members {" + action + "}\n" +
"a[User u, int i]\n" +
" : {" + action + "}\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator =
new ActionTranslator( generator,
"a",
new CommonToken( ANTLRParser.ACTION, action ), 0 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
}
public void TestAssignToTreeNodeAttribute()
{
string action = "$tree.scope = localScope;";
string expecting = "(()retval.tree).scope = localScope;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar a;\n" +
"options { output=AST; }" +
"rule\n" +
"@init {\n" +
" Scope localScope=null;\n" +
"}\n" +
"@after {\n" +
" $tree.scope = localScope;\n" +
"}\n" +
" : 'a' -> ^('a')\n" +
";" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator,
"rule",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
assertEquals( expecting, found );
}
public virtual void Write( StringTemplate code, string fileName )
{
Stopwatch watch = Stopwatch.StartNew();
TextWriter w = tool.GetOutputFile( grammar, fileName );
// Write the output to a StringWriter
IStringTemplateWriter wr = templates.GetStringTemplateWriter( w );
wr.SetLineWidth( lineWidth );
code.Write( wr );
w.Close();
TimeSpan duration = watch.Elapsed;
//[email protected]("render time for "+fileName+": "+(int)(stop-start)+"ms");
}
public void TestBasicGlobalScope()
{
string action = "$Symbols::names.add($id.text);";
string expecting = "((Symbols_scope)Symbols_stack.peek()).names.add((id!=null?id.getText():null));";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"scope Symbols {\n" +
" int n;\n" +
" List names;\n" +
"}\n" +
"a scope Symbols; : (id=ID ';' {" + action + "} )+\n" +
" ;\n" +
"ID : 'a';\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
/** Generate a token names table that maps token type to a printable
* name: either the label like INT or the literal like "begin".
*/
protected virtual void GenTokenTypeNames( StringTemplate code )
{
for ( int t = Label.MIN_TOKEN_TYPE; t <= grammar.MaxTokenType; t++ )
{
string tokenName = grammar.GetTokenDisplayName( t );
if ( tokenName != null )
{
tokenName = target.GetTargetStringLiteralFromString( tokenName, true );
code.SetAttribute( "tokenNames", tokenName );
}
}
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes.
*/
protected virtual void WalkCreatingDFADOT( StringTemplate dot,
DFAState s )
{
if ( markedStates.Contains( s.stateNumber ) )
{
return; // already visited this node
}
markedStates.Add( s.stateNumber ); // mark this node as completed.
// first add this node
StringTemplate st;
if ( s.IsAcceptState )
{
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) );
}
else
{
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) );
}
st.SetAttribute( "name", GetStateLabel( s ) );
dot.SetAttribute( "states", st );
// make a DOT edge for each transition
for ( int i = 0; i < s.NumberOfTransitions; i++ )
{
Transition edge = (Transition)s.Transition( i );
//Console.Out.WriteLine( "dfa " + s.dfa.decisionNumber + " edge from s"
// + s.stateNumber + " [" + i + "] of " + s.NumberOfTransitions );
if ( StripNonreducedStates )
{
if ( edge.target is DFAState &&
( (DFAState)edge.target ).AcceptStateReachable != DFA.REACHABLE_YES )
{
continue; // don't generate nodes for terminal states
}
}
st = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
st.SetAttribute( "label", GetEdgeLabel( edge ) );
st.SetAttribute( "src", GetStateLabel( s ) );
st.SetAttribute( "target", GetStateLabel( edge.target ) );
st.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", st );
WalkCreatingDFADOT( dot, (DFAState)edge.target ); // keep walkin'
}
}
// L O O K A H E A D D E C I S I O N G E N E R A T I O N
/** Generate code that computes the predicted alt given a DFA. The
* recognizerST can be either the main generated recognizerTemplate
* for storage in the main parser file or a separate file. It's up to
* the code that ultimately invokes the codegen.g grammar rule.
*
* Regardless, the output file and header file get a copy of the DFAs.
*/
public virtual StringTemplate GenLookaheadDecision( StringTemplate recognizerST,
DFA dfa )
{
StringTemplate decisionST;
// If we are doing inline DFA and this one is acyclic and LL(*)
// I have to check for is-non-LL(*) because if non-LL(*) the cyclic
// check is not done by DFA.verify(); that is, verify() avoids
// doesStateReachAcceptState() if non-LL(*)
if ( dfa.CanInlineDecision )
{
decisionST =
acyclicDFAGenerator.GenFixedLookaheadDecision( Templates, dfa );
}
else
{
// generate any kind of DFA here (cyclic or acyclic)
dfa.CreateStateTables( this );
outputFileST.SetAttribute( "cyclicDFAs", dfa );
headerFileST.SetAttribute( "cyclicDFAs", dfa );
decisionST = templates.GetInstanceOf( "dfaDecision" );
string description = dfa.NFADecisionStartState.Description;
description = target.GetTargetStringLiteralFromString( description );
if ( description != null )
{
decisionST.SetAttribute( "description", description );
}
decisionST.SetAttribute( "decisionNumber",
dfa.DecisionNumber );
}
return decisionST;
}
/** From T.g return a list of File objects that
* name files ANTLR will emit from T.g.
*/
public virtual IList <string> GetGeneratedFileList()
{
List <FileInfo> files = new List <FileInfo>();
System.IO.DirectoryInfo outputDir = tool.GetOutputDirectory(grammarFileName);
if (outputDir.Name.Equals("."))
{
outputDir = null;
}
else if (outputDir.Name.IndexOf(' ') >= 0)
{ // has spaces?
string escSpaces = outputDir.ToString().Replace(
" ",
"\\ ");
outputDir = new System.IO.DirectoryInfo(escSpaces);
}
// add generated recognizer; e.g., TParser.java
string recognizer =
generator.GetRecognizerFileName(grammar.name, grammar.type);
files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, recognizer)));
// add output vocab file; e.g., T.tokens. This is always generated to
// the base output directory, which will be just . if there is no -o option
//
files.Add(new FileInfo(System.IO.Path.Combine(tool.OutputDirectory, generator.VocabFileName)));
// are we generating a .h file?
StringTemplate headerExtST = null;
StringTemplate extST = generator.Templates.GetInstanceOf("codeFileExtension");
if (generator.Templates.IsDefined("headerFile"))
{
headerExtST = generator.Templates.GetInstanceOf("headerFileExtension");
string suffix = Grammar.grammarTypeToFileNameSuffix[(int)grammar.type];
string fileName = grammar.name + suffix + headerExtST.ToString();
files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, fileName)));
}
if (grammar.type == GrammarType.Combined)
{
// add autogenerated lexer; e.g., TLexer.java TLexer.h TLexer.tokens
// don't add T__.g (just a temp file)
string suffix = Grammar.grammarTypeToFileNameSuffix[(int)GrammarType.Lexer];
string lexer = grammar.name + suffix + extST.ToString();
files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, lexer)));
// TLexer.h
if (headerExtST != null)
{
string header = grammar.name + suffix + headerExtST.ToString();
files.Add(new FileInfo(System.IO.Path.Combine(outputDir.FullName, header)));
}
// for combined, don't generate TLexer.tokens
}
// handle generated files for imported grammars
IList <Grammar> imports =
grammar.composite.GetDelegates(grammar.composite.RootGrammar);
foreach (Grammar g in imports)
{
outputDir = tool.GetOutputDirectory(g.FileName);
string fname = GroomQualifiedFileName(outputDir.ToString(), g.GetRecognizerName() + extST.ToString());
files.Add(new FileInfo(fname));
}
if (files.Count == 0)
{
return(null);
}
return(files.Select(info => info.FullName).ToArray());
}
/** For intervals such as [3..3, 30..35], generate an expression that
* tests the lookahead similar to LA(1)==3 || (LA(1)>=30&&LA(1)<=35)
*/
public virtual StringTemplate GenSetExpr( StringTemplateGroup templates,
IIntSet set,
int k,
bool partOfDFA )
{
if ( !( set is IntervalSet ) )
{
throw new ArgumentException( "unable to generate expressions for non IntervalSet objects" );
}
IntervalSet iset = (IntervalSet)set;
if ( iset.Intervals == null || iset.Intervals.Count == 0 )
{
StringTemplate emptyST = new StringTemplate( templates, "" );
emptyST.Name = "empty-set-expr";
return emptyST;
}
string testSTName = "lookaheadTest";
string testRangeSTName = "lookaheadRangeTest";
if ( !partOfDFA )
{
testSTName = "isolatedLookaheadTest";
testRangeSTName = "isolatedLookaheadRangeTest";
}
StringTemplate setST = templates.GetInstanceOf( "setTest" );
int rangeNumber = 1;
foreach ( Interval I in iset.GetIntervals() )
{
int a = I.a;
int b = I.b;
StringTemplate eST;
if ( a == b )
{
eST = templates.GetInstanceOf( testSTName );
eST.SetAttribute( "atom", GetTokenTypeAsTargetLabel( a ) );
eST.SetAttribute( "atomAsInt", a );
//eST.setAttribute("k",Utils.integer(k));
}
else
{
eST = templates.GetInstanceOf( testRangeSTName );
eST.SetAttribute( "lower", GetTokenTypeAsTargetLabel( a ) );
eST.SetAttribute( "lowerAsInt", a );
eST.SetAttribute( "upper", GetTokenTypeAsTargetLabel( b ) );
eST.SetAttribute( "upperAsInt", b );
eST.SetAttribute( "rangeNumber", rangeNumber );
}
eST.SetAttribute( "k", k );
setST.SetAttribute( "ranges", eST );
rangeNumber++;
}
return setST;
}
public void TestDoNotTranslateAttributeCompare()
{
string action = "$a.line == $b.line";
string expecting = "(a!=null?a.getLine():0) == (b!=null?b.getLine():0)";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"lexer grammar a;\n" +
"RULE:\n" +
" a=ID b=ID {" + action + "}" +
" ;\n" +
"ID : 'id';"
);
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer();
ActionTranslator translator = new ActionTranslator( generator,
"RULE",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
assertEquals( expecting, found );
}
protected virtual StringTemplate WalkFixedDFAGeneratingStateMachine(
StringTemplateGroup templates,
DFA dfa,
DFAState s,
int k)
{
//System.Console.Out.WriteLine( "walk " + s.stateNumber + " in dfa for decision " + dfa.decisionNumber );
if (s.IsAcceptState)
{
StringTemplate dfaST2 = templates.GetInstanceOf("dfaAcceptState");
dfaST2.SetAttribute("alt", s.GetUniquelyPredictedAlt());
return(dfaST2);
}
// the default templates for generating a state and its edges
// can be an if-then-else structure or a switch
string dfaStateName = "dfaState";
string dfaLoopbackStateName = "dfaLoopbackState";
string dfaOptionalBlockStateName = "dfaOptionalBlockState";
string dfaEdgeName = "dfaEdge";
if (parentGenerator.CanGenerateSwitch(s))
{
dfaStateName = "dfaStateSwitch";
dfaLoopbackStateName = "dfaLoopbackStateSwitch";
dfaOptionalBlockStateName = "dfaOptionalBlockStateSwitch";
dfaEdgeName = "dfaEdgeSwitch";
}
StringTemplate dfaST = templates.GetInstanceOf(dfaStateName);
if (dfa.NFADecisionStartState.decisionStateType == NFAState.LOOPBACK)
{
dfaST = templates.GetInstanceOf(dfaLoopbackStateName);
}
else if (dfa.NFADecisionStartState.decisionStateType == NFAState.OPTIONAL_BLOCK_START)
{
dfaST = templates.GetInstanceOf(dfaOptionalBlockStateName);
}
dfaST.SetAttribute("k", k);
dfaST.SetAttribute("stateNumber", s.stateNumber);
dfaST.SetAttribute("semPredState",
s.IsResolvedWithPredicates);
/*
* string description = dfa.getNFADecisionStartState().Description;
* description = parentGenerator.target.getTargetStringLiteralFromString( description );
* //System.Console.Out.WriteLine( "DFA: " + description + " associated with AST " + dfa.getNFADecisionStartState() );
* if ( description != null )
* {
* dfaST.SetAttribute( "description", description );
* }
*/
int EOTPredicts = NFA.INVALID_ALT_NUMBER;
DFAState EOTTarget = null;
//System.Console.Out.WriteLine( "DFA state " + s.stateNumber );
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = (Transition)s.Transition(i);
//System.Console.Out.WriteLine( "edge " + s.stateNumber + "-" + edge.label.ToString() + "->" + edge.target.stateNumber );
if (edge.label.Atom == Label.EOT)
{
// don't generate a real edge for EOT; track alt EOT predicts
// generate that prediction in the else clause as default case
EOTTarget = (DFAState)edge.target;
EOTPredicts = EOTTarget.GetUniquelyPredictedAlt();
/*
* System.Console.Out.WriteLine("DFA s"+s.stateNumber+" EOT goes to s"+
* edge.target.stateNumber+" predicates alt "+
* EOTPredicts);
*/
continue;
}
StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName);
// If the template wants all the label values delineated, do that
if (edgeST.GetFormalArgument("labels") != null)
{
List <string> labels = edge.Label.Set.Select(value => parentGenerator.GetTokenTypeAsTargetLabel(value)).ToList();
edgeST.SetAttribute("labels", labels);
}
else
{ // else create an expression to evaluate (the general case)
edgeST.SetAttribute("labelExpr",
parentGenerator.GenLabelExpr(templates, edge, k));
}
// stick in any gated predicates for any edge if not already a pred
if (!edge.label.IsSemanticPredicate)
{
DFAState target = (DFAState)edge.target;
SemanticContext preds =
target.GetGatedPredicatesInNFAConfigurations();
if (preds != null)
{
//System.Console.Out.WriteLine( "preds=" + target.getGatedPredicatesInNFAConfigurations() );
StringTemplate predST = preds.GenExpr(parentGenerator,
parentGenerator.Templates,
dfa);
edgeST.SetAttribute("predicates", predST);
}
}
StringTemplate targetST =
WalkFixedDFAGeneratingStateMachine(templates,
dfa,
(DFAState)edge.target,
k + 1);
edgeST.SetAttribute("targetState", targetST);
dfaST.SetAttribute("edges", edgeST);
//System.Console.Out.WriteLine( "back to DFA " + dfa.decisionNumber + "." + s.stateNumber );
}
// HANDLE EOT EDGE
if (EOTPredicts != NFA.INVALID_ALT_NUMBER)
{
// EOT unique predicts an alt
dfaST.SetAttribute("eotPredictsAlt", EOTPredicts);
}
else if (EOTTarget != null && EOTTarget.NumberOfTransitions > 0)
{
// EOT state has transitions so must split on predicates.
// Generate predicate else-if clauses and then generate
// NoViableAlt exception as else clause.
// Note: these predicates emanate from the EOT target state
// rather than the current DFAState s so the error message
// might be slightly misleading if you are looking at the
// state number. Predicates emanating from EOT targets are
// hoisted up to the state that has the EOT edge.
for (int i = 0; i < EOTTarget.NumberOfTransitions; i++)
{
Transition predEdge = (Transition)EOTTarget.Transition(i);
StringTemplate edgeST = templates.GetInstanceOf(dfaEdgeName);
edgeST.SetAttribute("labelExpr",
parentGenerator.GenSemanticPredicateExpr(templates, predEdge));
// the target must be an accept state
//System.Console.Out.WriteLine( "EOT edge" );
StringTemplate targetST =
WalkFixedDFAGeneratingStateMachine(templates,
dfa,
(DFAState)predEdge.target,
k + 1);
edgeST.SetAttribute("targetState", targetST);
dfaST.SetAttribute("edges", edgeST);
}
}
return(dfaST);
}
public void TestDoNotTranslateScopeAttributeCompare()
{
string action = "if ($rule::foo == \"foo\" || 1) { System.out.println(\"ouch\"); }";
string expecting = "if (((rule_scope)rule_stack.peek()).foo == \"foo\" || 1) { System.out.println(\"ouch\"); }";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar a;\n" +
"rule\n" +
"scope {\n" +
" String foo;" +
"} :\n" +
" twoIDs" +
" ;\n" +
"twoIDs:\n" +
" ID ID {" + action + "}\n" +
" ;\n" +
"ID : 'id';"
);
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer();
ActionTranslator translator = new ActionTranslator( generator,
"twoIDs",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
// check that we didn't use scopeSetAttributeRef int translation!
bool foundScopeSetAttributeRef = false;
for ( int i = 0; i < translator.chunks.Count; i++ )
{
object chunk = translator.chunks[i];
if ( chunk is StringTemplate )
{
if ( ( (StringTemplate)chunk ).Name.Equals( "scopeSetAttributeRef" ) )
{
foundScopeSetAttributeRef = true;
}
}
}
assertFalse( "action translator used scopeSetAttributeRef template in comparison!", foundScopeSetAttributeRef );
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
assertEquals( expecting, found );
}
protected override void GenRecognizerFile(AntlrTool tool, CodeGenerator generator, Grammar grammar, StringTemplate outputFileST)
{
generator.Templates.RegisterRenderer(typeof(string), new StringRenderer(generator, this));
base.GenRecognizerFile(tool, generator, grammar, outputFileST);
}
public void TestDynamicScopeRefOkEvenThoughRuleRefExists()
{
string action = "$b::n;";
string expecting = "((b_scope)b_stack.peek()).n;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"s : b ;\n" +
"b\n" +
"scope {\n" +
" int n;\n" +
"} : '(' b ')' {" + action + "}\n" + // refers to current invocation's n
" ;\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "b",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public void TestBasicRuleScope()
{
string action = "$a::n;";
string expecting = "((a_scope)a_stack.peek()).n;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"a\n" +
"scope {\n" +
" int n;\n" +
"} : {" + action + "}\n" +
" ;\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
public override string ToString()
{
GrammarAST decisionASTNode = probe.dfa.DecisionASTNode;
line = decisionASTNode.Line;
charPositionInLine = decisionASTNode.CharPositionInLine;
string fileName = probe.dfa.nfa.grammar.FileName;
if (fileName != null)
{
file = fileName;
}
StringTemplate st = GetMessageTemplate();
// Now fill template with information about problemState
var labels = probe.GetSampleNonDeterministicInputSequence(problemState);
string input = probe.GetInputSequenceDisplay(labels);
st.SetAttribute("input", input);
if (probe.dfa.IsTokensRuleDecision)
{
var disabledAlts = probe.GetDisabledAlternatives(problemState);
foreach (int altI in disabledAlts)
{
string tokenName =
probe.GetTokenNameForTokensRuleAlt((int)altI);
// reset the line/col to the token definition (pick last one)
NFAState ruleStart =
probe.dfa.nfa.grammar.GetRuleStartState(tokenName);
line = ruleStart.associatedASTNode.Line;
charPositionInLine = ruleStart.associatedASTNode.CharPositionInLine;
st.SetAttribute("disabled", tokenName);
}
}
else
{
st.SetAttribute("disabled", probe.GetDisabledAlternatives(problemState));
}
var nondetAlts = probe.GetNonDeterministicAltsForState(problemState);
NFAState nfaStart = probe.dfa.NFADecisionStartState;
// all state paths have to begin with same NFA state
int firstAlt = 0;
if (nondetAlts != null)
{
foreach (int displayAltI in nondetAlts)
{
if (DecisionProbe.verbose)
{
int tracePathAlt =
nfaStart.TranslateDisplayAltToWalkAlt((int)displayAltI);
if (firstAlt == 0)
{
firstAlt = tracePathAlt;
}
IList path =
probe.GetNFAPathStatesForAlt(firstAlt,
tracePathAlt,
labels);
st.SetAttribute("paths.{alt,states}",
displayAltI, path);
}
else
{
if (probe.dfa.IsTokensRuleDecision)
{
// alts are token rules, convert to the names instead of numbers
string tokenName =
probe.GetTokenNameForTokensRuleAlt((int)displayAltI);
st.SetAttribute("conflictingTokens", tokenName);
}
else
{
st.SetAttribute("conflictingAlts", displayAltI);
}
}
}
}
st.SetAttribute("hasPredicateBlockedByAction", problemState.dfa.hasPredicateBlockedByAction);
return(base.ToString(st));
}
/** Do a depth-first walk of the state machine graph and
* fill a DOT description template. Keep filling the
* states and edges attributes. We know this is an NFA
* for a rule so don't traverse edges to other rules and
* don't go past rule end state.
*/
protected virtual void WalkRuleNFACreatingDOT( StringTemplate dot,
State s )
{
if ( markedStates.Contains( s.stateNumber ) )
{
return; // already visited this node
}
markedStates.Add( s.stateNumber ); // mark this node as completed.
// first add this node
StringTemplate stateST;
if ( s.IsAcceptState )
{
stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "stopstate" ) );
}
else
{
stateST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "state" ) );
}
stateST.SetAttribute( "name", GetStateLabel( s ) );
dot.SetAttribute( "states", stateST );
if ( s.IsAcceptState )
{
return; // don't go past end of rule node to the follow states
}
// special case: if decision point, then line up the alt start states
// unless it's an end of block
if ( ( (NFAState)s ).IsDecisionState )
{
GrammarAST n = ( (NFAState)s ).associatedASTNode;
if ( n != null && n.Type != ANTLRParser.EOB )
{
StringTemplate rankST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "decision-rank" ) );
NFAState alt = (NFAState)s;
while ( alt != null )
{
rankST.SetAttribute( "states", GetStateLabel( alt ) );
if ( alt.transition[1] != null )
{
alt = (NFAState)alt.transition[1].target;
}
else
{
alt = null;
}
}
dot.SetAttribute( "decisionRanks", rankST );
}
}
// make a DOT edge for each transition
StringTemplate edgeST = null;
for ( int i = 0; i < s.NumberOfTransitions; i++ )
{
Transition edge = (Transition)s.GetTransition( i );
if ( edge is RuleClosureTransition )
{
RuleClosureTransition rr = ( (RuleClosureTransition)edge );
// don't jump to other rules, but display edge to follow node
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
if ( rr.rule.grammar != grammar )
{
edgeST.SetAttribute( "label", "<" + rr.rule.grammar.name + "." + rr.rule.Name + ">" );
}
else
{
edgeST.SetAttribute( "label", "<" + rr.rule.Name + ">" );
}
edgeST.SetAttribute( "src", GetStateLabel( s ) );
edgeST.SetAttribute( "target", GetStateLabel( rr.followState ) );
edgeST.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", edgeST );
WalkRuleNFACreatingDOT( dot, rr.followState );
continue;
}
if ( edge.IsAction )
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "action-edge" ) );
}
else if ( edge.IsEpsilon )
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "epsilon-edge" ) );
}
else
{
edgeST = stlib.GetInstanceOf( Path.Combine( dfaTemplateDirectoryName, "edge" ) );
}
edgeST.SetAttribute( "label", GetEdgeLabel( edge ) );
edgeST.SetAttribute( "src", GetStateLabel( s ) );
edgeST.SetAttribute( "target", GetStateLabel( edge.target ) );
edgeST.SetAttribute( "arrowhead", arrowhead );
dot.SetAttribute( "edges", edgeST );
WalkRuleNFACreatingDOT( dot, edge.target ); // keep walkin'
}
}
public void TestComplicatedArgParsingWithTranslation()
{
string action = "x, $A.text+\"3242\", (*$A).foo(21,33), 3.2+1, '\\n', " +
"\"a,oo\\nick\", {bl, \"fdkj\"eck}";
string expecting = "x, (A1!=null?A1.getText():null)+\"3242\", (*A1).foo(21,33), 3.2+1, '\\n', \"a,oo\\nick\", {bl, \"fdkj\"eck}";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
// now check in actual grammar.
Grammar g = new Grammar(
"parser grammar t;\n" +
"a[User u, int i]\n" +
" : A a[" + action + "] B\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator, "a",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
StringTemplateGroup templates =
new StringTemplateGroup( ".", typeof( AngleBracketTemplateLexer ) );
StringTemplate actionST = new StringTemplate( templates, rawTranslation );
string found = actionST.ToString();
assertEquals( expecting, found );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}
