public LeftRecursiveRule(Grammar g, string name, RuleAST ast)
: base(g, name, ast, 1)
{
originalAST = ast;
alt = new Alternative[numberOfAlts + 1]; // always just one
for (int i = 1; i <= numberOfAlts; i++)
alt[i] = new Alternative(this, i);
}
public override void DiscoverRule(RuleAST rule, GrammarAST ID,
IList<GrammarAST> modifiers, ActionAST arg,
ActionAST returns, GrammarAST thrws,
GrammarAST options, ActionAST locals,
IList<GrammarAST> actions,
GrammarAST block)
{
currentRule = g.GetRule(ID.Text);
}
public override void DiscoverRule(RuleAST rule, GrammarAST ID,
IList<GrammarAST> modifiers, ActionAST arg,
ActionAST returns, GrammarAST thrws,
GrammarAST options, ActionAST locals,
IList<GrammarAST> actions,
GrammarAST block)
{
int numAlts = block.ChildCount;
Rule r;
if (LeftRecursiveRuleAnalyzer.HasImmediateRecursiveRuleRefs(rule, ID.Text))
{
r = new LeftRecursiveRule(g, ID.Text, rule);
}
else
{
r = new Rule(g, ID.Text, rule, numAlts);
}
rules[r.name] = r;
if (arg != null)
{
r.args = ScopeParser.ParseTypedArgList(arg, arg.Text, g);
r.args.type = AttributeDict.DictType.ARG;
r.args.ast = arg;
arg.resolver = r.alt[currentOuterAltNumber];
}
if (returns != null)
{
r.retvals = ScopeParser.ParseTypedArgList(returns, returns.Text, g);
r.retvals.type = AttributeDict.DictType.RET;
r.retvals.ast = returns;
}
if (locals != null)
{
r.locals = ScopeParser.ParseTypedArgList(locals, locals.Text, g);
r.locals.type = AttributeDict.DictType.LOCAL;
r.locals.ast = locals;
}
foreach (GrammarAST a in actions)
{
// a = ^(AT ID ACTION)
ActionAST action = (ActionAST)a.GetChild(1);
r.namedActions[a.GetChild(0).Text] = action;
action.resolver = r;
}
}
/** Make sure even imaginary nodes know the input stream */
public override object Create(int tokenType, string text)
{
GrammarAST t;
if (tokenType == ANTLRParser.RULE)
{
// needed by TreeWizard to make RULE tree
t = new RuleAST(new CommonToken(tokenType, text));
}
else if (tokenType == ANTLRParser.STRING_LITERAL)
{
// implicit lexer construction done with wizard; needs this node type
// whereas grammar ANTLRParser.g can use token option to spec node type
t = new TerminalAST(new CommonToken(tokenType, text));
}
else
{
t = (GrammarAST)base.Create(tokenType, text);
}
t.Token.InputStream = input;
return t;
}
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers,
GrammarAST block)
{
currentRule = g.GetRule(ID.Text);
}
public RuleAST(RuleAST node)
: base(node)
{
}
public int actionIndex = -1; // if lexer; 0..n-1 for n actions in a rule
public Rule(Grammar g, string name, RuleAST ast, int numberOfAlts)
{
this.g = g;
this.name = name;
this.ast = ast;
this.numberOfAlts = numberOfAlts;
alt = new Alternative[numberOfAlts + 1]; // 1..n
for (int i = 1; i <= numberOfAlts; i++)
alt[i] = new Alternative(this, i);
}
public override void FinishRule(RuleAST rule, GrammarAST ID, GrammarAST block)
{
if (rule.IsLexerRule())
return;
BlockAST blk = (BlockAST)rule.GetFirstChildWithType(BLOCK);
int nalts = blk.ChildCount;
GrammarAST idAST = (GrammarAST)rule.GetChild(0);
for (int i = 0; i < nalts; i++)
{
AltAST altAST = (AltAST)blk.GetChild(i);
if (altAST.altLabel != null)
{
string altLabel = altAST.altLabel.Text;
// first check that label doesn't conflict with a rule
// label X or x can't be rule x.
Rule r;
if (ruleCollector.rules.TryGetValue(Utils.Decapitalize(altLabel), out r) && r != null)
{
g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_CONFLICTS_WITH_RULE,
g.fileName, altAST.altLabel.Token,
altLabel,
r.name);
}
// Now verify that label X or x doesn't conflict with label
// in another rule. altLabelToRuleName has both X and x mapped.
string prevRuleForLabel;
if (ruleCollector.altLabelToRuleName.TryGetValue(altLabel, out prevRuleForLabel) && prevRuleForLabel != null && !prevRuleForLabel.Equals(rule.GetRuleName()))
{
g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_REDEF,
g.fileName, altAST.altLabel.Token,
altLabel,
rule.GetRuleName(),
prevRuleForLabel);
}
}
}
IList<GrammarAST> altLabels;
int numAltLabels = 0;
if (ruleCollector.ruleToAltLabels.TryGetValue(rule.GetRuleName(), out altLabels) && altLabels != null)
numAltLabels = altLabels.Count;
if (numAltLabels > 0 && nalts != numAltLabels)
{
g.tool.errMgr.GrammarError(ErrorType.RULE_WITH_TOO_FEW_ALT_LABELS,
g.fileName, idAST.Token, rule.GetRuleName());
}
}
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers,
GrammarAST block)
{
CheckInvalidRuleDef(ID.Token);
if (modifiers != null)
{
foreach (GrammarAST tree in modifiers)
{
if (tree.Type == ANTLRParser.FRAGMENT)
{
inFragmentRule = true;
}
}
}
if (!inFragmentRule)
{
nonFragmentRuleCount++;
}
}
public override void DiscoverRule(RuleAST rule, GrammarAST ID,
IList<GrammarAST> modifiers,
ActionAST arg, ActionAST returns,
GrammarAST thrws, GrammarAST options,
ActionAST locals,
IList<GrammarAST> actions, GrammarAST block)
{
// TODO: chk that all or no alts have "# label"
CheckInvalidRuleDef(ID.Token);
}
public virtual IDictionary<string, IList<System.Tuple<int, AltAST>>> GetLabeledAlternatives(RuleAST ast)
{
AltLabelVisitor visitor = new AltLabelVisitor(new Antlr.Runtime.Tree.CommonTreeNodeStream(new GrammarASTAdaptor(), ast));
visitor.rule();
return visitor.GetLabeledAlternatives();
}
public virtual IList<AltAST> GetUnlabeledAlternatives(RuleAST ast)
{
AltLabelVisitor visitor = new AltLabelVisitor(new Antlr.Runtime.Tree.CommonTreeNodeStream(new GrammarASTAdaptor(), ast));
visitor.rule();
return visitor.GetUnlabeledAlternatives();
}
/**
* <pre>
* (RULE e int _p (returns int v)
* (BLOCK
* (ALT
* (BLOCK
* (ALT INT {$v = $INT.int;})
* (ALT '(' (= x e) ')' {$v = $x.v;})
* (ALT ID))
* (* (BLOCK
* (OPTIONS ...)
* (ALT {7 >= $_p}? '*' (= b e) {$v = $a.v * $b.v;})
* (ALT {6 >= $_p}? '+' (= b e) {$v = $a.v + $b.v;})
* (ALT {3 >= $_p}? '++') (ALT {2 >= $_p}? '--'))))))
* </pre>
*/
public virtual void SetAltASTPointers(LeftRecursiveRule r, RuleAST t)
{
//System.Console.WriteLine("RULE: " + t.ToStringTree());
BlockAST ruleBlk = (BlockAST)t.GetFirstChildWithType(ANTLRParser.BLOCK);
AltAST mainAlt = (AltAST)ruleBlk.GetChild(0);
BlockAST primaryBlk = (BlockAST)mainAlt.GetChild(0);
BlockAST opsBlk = (BlockAST)mainAlt.GetChild(1).GetChild(0); // (* BLOCK ...)
for (int i = 0; i < r.recPrimaryAlts.Count; i++)
{
LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts[i];
altInfo.altAST = (AltAST)primaryBlk.GetChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
//altInfo.originalAltAST.Parent = altInfo.altAST.Parent;
//System.Console.WriteLine(altInfo.altAST.ToStringTree());
}
for (int i = 0; i < r.recOpAlts.Count; i++)
{
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.GetElement(i);
altInfo.altAST = (AltAST)opsBlk.GetChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
//altInfo.originalAltAST.Parent = altInfo.altAST.Parent;
//System.Console.WriteLine(altInfo.altAST.ToStringTree());
}
}
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList<GrammarAST> actionsWeMoved = new List<GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
return lexerAST;
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList<GrammarAST> rulesWeMoved = new List<GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast<GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList<System.Tuple<GrammarAST, GrammarAST>> litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet<string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple<GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
goto continueNextLit;
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
return null;
return lexerAST;
}
public RuleAST(RuleAST node)
: base(node)
{
}
public override void DiscoverLexerRule(RuleAST rule, GrammarAST ID, IList<GrammarAST> modifiers,
GrammarAST block)
{
int numAlts = block.ChildCount;
Rule r = new Rule(g, ID.Text, rule, numAlts);
r.mode = currentModeName;
if (modifiers.Count > 0)
r.modifiers = modifiers;
rules[r.name] = r;
}