public override void SetAltAssoc(AltAST t, int alt)
{
ASSOC assoc = ASSOC.left;
if (t.GetOptions() != null)
{
string a = t.GetOptionString("assoc");
if (a != null)
{
if (a.Equals(ASSOC.right.ToString()))
{
assoc = ASSOC.right;
}
else if (a.Equals(ASSOC.left.ToString()))
{
assoc = ASSOC.left;
}
else
{
tool.errMgr.GrammarError(ErrorType.ILLEGAL_OPTION_VALUE, t.g.fileName, t.GetOptionAST("assoc").Token, "assoc", assoc);
}
}
}
if (altAssociativity.ContainsKey(alt) && altAssociativity[alt] != assoc)
{
tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, "all operators of alt " + alt + " of left-recursive rule must have same associativity");
}
altAssociativity[alt] = assoc;
// System.out.println("setAltAssoc: op " + alt + ": " + t.getText()+", assoc="+assoc);
}
public override void BinaryAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
string label = null;
bool isListLabel = false;
GrammarAST lrlabel = StripLeftRecursion(altTree);
if (lrlabel != null)
{
label = lrlabel.Text;
isListLabel = lrlabel.Parent.Type == PLUS_ASSIGN;
leftRecursiveRuleRefLabels.Add(Tuple.Create(lrlabel, altLabel));
}
StripAltLabel(altTree);
// rewrite e to be e_[rec_arg]
int nextPrec = NextPrecedence(alt);
altTree = AddPrecedenceArgToRules(altTree, nextPrec);
StripAltLabel(altTree);
string altText = Text(altTree);
altText = altText.Trim();
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, isListLabel, originalAltTree);
a.nextPrec = nextPrec;
binaryAlts[alt] = a;
//System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
/**
* <pre>
* (RULE e int _p (returns int v)
* (BLOCK
* (ALT
* (BLOCK
* (ALT INT {$v = $INT.int;})
* (ALT '(' (= x e) ')' {$v = $x.v;})
* (ALT ID))
* (* (BLOCK
* (OPTIONS ...)
* (ALT {7 >= $_p}? '*' (= b e) {$v = $a.v * $b.v;})
* (ALT {6 >= $_p}? '+' (= b e) {$v = $a.v + $b.v;})
* (ALT {3 >= $_p}? '++') (ALT {2 >= $_p}? '--'))))))
* </pre>
*/
public virtual void SetAltASTPointers(LeftRecursiveRule r, RuleAST t)
{
//System.Console.WriteLine("RULE: " + t.ToStringTree());
BlockAST ruleBlk = (BlockAST)t.GetFirstChildWithType(ANTLRParser.BLOCK);
AltAST mainAlt = (AltAST)ruleBlk.GetChild(0);
BlockAST primaryBlk = (BlockAST)mainAlt.GetChild(0);
BlockAST opsBlk = (BlockAST)mainAlt.GetChild(1).GetChild(0); // (* BLOCK ...)
for (int i = 0; i < r.recPrimaryAlts.Count; i++)
{
LeftRecursiveRuleAltInfo altInfo = r.recPrimaryAlts[i];
altInfo.altAST = (AltAST)primaryBlk.GetChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
//altInfo.originalAltAST.Parent = altInfo.altAST.Parent;
//System.Console.WriteLine(altInfo.altAST.ToStringTree());
}
for (int i = 0; i < r.recOpAlts.Count; i++)
{
LeftRecursiveRuleAltInfo altInfo = r.recOpAlts.GetElement(i);
altInfo.altAST = (AltAST)opsBlk.GetChild(i);
altInfo.altAST.leftRecursiveAltInfo = altInfo;
altInfo.originalAltAST.leftRecursiveAltInfo = altInfo;
//altInfo.originalAltAST.Parent = altInfo.altAST.Parent;
//System.Console.WriteLine(altInfo.altAST.ToStringTree());
}
}
public override void DiscoverOuterAlt(AltAST alt)
{
if (alt.altLabel != null)
{
ruleToAltLabels.Map(currentRuleName, alt.altLabel);
string altLabel = alt.altLabel.Text;
altLabelToRuleName[Utils.Capitalize(altLabel)] = currentRuleName;
altLabelToRuleName[Utils.Decapitalize(altLabel)] = currentRuleName;
}
}
public override void FinishRule(RuleAST rule, GrammarAST ID, GrammarAST block)
{
if (rule.IsLexerRule())
{
return;
}
BlockAST blk = (BlockAST)rule.GetFirstChildWithType(BLOCK);
int nalts = blk.ChildCount;
GrammarAST idAST = (GrammarAST)rule.GetChild(0);
for (int i = 0; i < nalts; i++)
{
AltAST altAST = (AltAST)blk.GetChild(i);
if (altAST.altLabel != null)
{
string altLabel = altAST.altLabel.Text;
// first check that label doesn't conflict with a rule
// label X or x can't be rule x.
Rule r;
if (ruleCollector.rules.TryGetValue(Utils.Decapitalize(altLabel), out r) && r != null)
{
g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_CONFLICTS_WITH_RULE,
g.fileName, altAST.altLabel.Token,
altLabel,
r.name);
}
// Now verify that label X or x doesn't conflict with label
// in another rule. altLabelToRuleName has both X and x mapped.
string prevRuleForLabel;
if (ruleCollector.altLabelToRuleName.TryGetValue(altLabel, out prevRuleForLabel) && prevRuleForLabel != null && !prevRuleForLabel.Equals(rule.GetRuleName()))
{
g.tool.errMgr.GrammarError(ErrorType.ALT_LABEL_REDEF,
g.fileName, altAST.altLabel.Token,
altLabel,
rule.GetRuleName(),
prevRuleForLabel);
}
}
}
IList <GrammarAST> altLabels;
int numAltLabels = 0;
if (ruleCollector.ruleToAltLabels.TryGetValue(rule.GetRuleName(), out altLabels) && altLabels != null)
{
numAltLabels = altLabels.Count;
}
if (numAltLabels > 0 && nalts != numAltLabels)
{
g.tool.errMgr.GrammarError(ErrorType.RULE_WITH_TOO_FEW_ALT_LABELS,
g.fileName, idAST.Token, rule.GetRuleName());
}
}
public LeftRecursiveRuleAltInfo(int altNum, string altText,
string leftRecursiveRuleRefLabel,
string altLabel,
bool isListLabel,
AltAST originalAltAST)
{
this.altNum = altNum;
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
this.altLabel = altLabel;
this.isListLabel = isListLabel;
this.originalAltAST = originalAltAST;
}
public LeftRecursiveRuleAltInfo(int altNum, string altText,
string leftRecursiveRuleRefLabel,
string altLabel,
bool isListLabel,
AltAST originalAltAST)
{
this.altNum = altNum;
this.altText = altText;
this.leftRecursiveRuleRefLabel = leftRecursiveRuleRefLabel;
this.altLabel = altLabel;
this.isListLabel = isListLabel;
this.originalAltAST = originalAltAST;
}
public override void OtherAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
StripAltLabel(altTree);
string altText = Text(altTree);
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, false, originalAltTree);
// We keep other alts with prefix alts since they are all added to the start of the generated rule, and
// we want to retain any prior ordering between them
prefixAndOtherAlts.Add(a);
// System.out.println("otherAlt " + alt + ": " + altText);
}
/**
* {@code (BLOCK (ALT .))} or {@code (BLOCK (ALT 'a') (ALT .))}.
*/
public static bool BlockHasWildcardAlt([NotNull] GrammarAST block)
{
foreach (object alt in block.Children)
{
if (!(alt is AltAST))
{
continue;
}
AltAST altAST = (AltAST)alt;
if (altAST.ChildCount == 1 || (altAST.ChildCount == 2 && altAST.GetChild(0).Type == ANTLRParser.ELEMENT_OPTIONS))
{
ITree e = altAST.GetChild(altAST.ChildCount - 1);
if (e.Type == ANTLRParser.WILDCARD)
{
return(true);
}
}
}
return(false);
}
public override void PrefixAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
StripAltLabel(altTree);
int nextPrec = Precedence(alt);
// rewrite e to be e_[prec]
altTree = AddPrecedenceArgToRules(altTree, nextPrec);
string altText = Text(altTree);
altText = altText.Trim();
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, false, originalAltTree);
a.nextPrec = nextPrec;
prefixAndOtherAlts.Add(a);
//System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
public virtual AltAST AddPrecedenceArgToRules(AltAST t, int prec)
{
if (t == null)
{
return(null);
}
// get all top-level rule refs from ALT
IList <GrammarAST> outerAltRuleRefs = t.GetNodesWithTypePreorderDFS(IntervalSet.Of(RULE_REF));
foreach (GrammarAST x in outerAltRuleRefs)
{
RuleRefAST rref = (RuleRefAST)x;
bool recursive = rref.Text.Equals(ruleName);
bool rightmost = rref == outerAltRuleRefs[outerAltRuleRefs.Count - 1];
if (recursive && rightmost)
{
GrammarAST dummyValueNode = new GrammarAST(new CommonToken(ANTLRParser.INT, "" + prec));
rref.SetOption(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME, dummyValueNode);
}
}
return(t);
}
public override void PrefixAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
StripAltLabel(altTree);
int nextPrec = Precedence(alt);
// rewrite e to be e_[prec]
altTree = AddPrecedenceArgToRules(altTree, nextPrec);
string altText = Text(altTree);
altText = altText.Trim();
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, false, originalAltTree);
a.nextPrec = nextPrec;
prefixAndOtherAlts.Add(a);
//System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
/** Given list of X and r refs in alt, compute how many of each there are */
protected virtual System.Tuple <FrequencySet <string>, FrequencySet <string> > GetElementFrequenciesForAlt(AltAST ast)
{
try
{
ElementFrequenciesVisitor visitor = new ElementFrequenciesVisitor(rule.g, new CommonTreeNodeStream(new GrammarASTAdaptor(), ast));
visitor.outerAlternative();
if (visitor.frequencies.Count != 1)
{
factory.GetGrammar().tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR);
return(Tuple.Create(new FrequencySet <string>(), new FrequencySet <string>()));
}
return(Tuple.Create(visitor.GetMinFrequencies(), visitor.frequencies.Peek()));
}
catch (RecognitionException ex)
{
factory.GetGrammar().tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, ex);
return(Tuple.Create(new FrequencySet <string>(), new FrequencySet <string>()));
}
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return false;
}
IList<GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return false;
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return false;
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return false;
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList<GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return true;
}
public override void BinaryAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
string label = null;
bool isListLabel = false;
GrammarAST lrlabel = StripLeftRecursion(altTree);
if (lrlabel != null)
{
label = lrlabel.Text;
isListLabel = lrlabel.Parent.Type == PLUS_ASSIGN;
leftRecursiveRuleRefLabels.Add(Tuple.Create(lrlabel, altLabel));
}
StripAltLabel(altTree);
// rewrite e to be e_[rec_arg]
int nextPrec = NextPrecedence(alt);
altTree = AddPrecedenceArgToRules(altTree, nextPrec);
StripAltLabel(altTree);
string altText = Text(altTree);
altText = altText.Trim();
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, isListLabel, originalAltTree);
a.nextPrec = nextPrec;
binaryAlts[alt] = a;
//System.out.println("binaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
/*
* Parser rules
*/
protected override void EnterAlternative(AltAST tree)
{
frequencies.Push(new FrequencySet<string>());
minFrequencies.Push(new FrequencySet<string>());
}
/*
* Parser rules
*/
protected override void EnterAlternative(AltAST tree)
{
frequencies.Push(new FrequencySet <string>());
minFrequencies.Push(new FrequencySet <string>());
}
public override void SuffixAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
string label = null;
bool isListLabel = false;
GrammarAST lrlabel = StripLeftRecursion(altTree);
if (lrlabel != null)
{
label = lrlabel.Text;
isListLabel = lrlabel.Parent.Type == PLUS_ASSIGN;
leftRecursiveRuleRefLabels.Add(Tuple.Create(lrlabel, altLabel));
}
StripAltLabel(altTree);
string altText = Text(altTree);
altText = altText.Trim();
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, label, altLabel, isListLabel, originalAltTree);
suffixAlts[alt] = a;
// System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText);
}
public override void OtherAlt(AltAST originalAltTree, int alt)
{
AltAST altTree = (AltAST)originalAltTree.DupTree();
StripAltLabel(altTree);
string altText = Text(altTree);
string altLabel = altTree.altLabel != null ? altTree.altLabel.Text : null;
LeftRecursiveRuleAltInfo a =
new LeftRecursiveRuleAltInfo(alt, altText, null, altLabel, false, originalAltTree);
// We keep other alts with prefix alts since they are all added to the start of the generated rule, and
// we want to retain any prior ordering between them
prefixAndOtherAlts.Add(a);
// System.out.println("otherAlt " + alt + ": " + altText);
}
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList<GrammarAST> actionsWeMoved = new List<GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
return lexerAST;
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList<GrammarAST> rulesWeMoved = new List<GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast<GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList<System.Tuple<GrammarAST, GrammarAST>> litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet<string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple<GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
goto continueNextLit;
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
return null;
return lexerAST;
}
public virtual AltAST AddPrecedenceArgToRules(AltAST t, int prec)
{
if (t == null)
return null;
// get all top-level rule refs from ALT
IList<GrammarAST> outerAltRuleRefs = t.GetNodesWithTypePreorderDFS(IntervalSet.Of(RULE_REF));
foreach (GrammarAST x in outerAltRuleRefs)
{
RuleRefAST rref = (RuleRefAST)x;
bool recursive = rref.Text.Equals(ruleName);
bool rightmost = rref == outerAltRuleRefs[outerAltRuleRefs.Count - 1];
if (recursive && rightmost)
{
GrammarAST dummyValueNode = new GrammarAST(new CommonToken(ANTLRParser.INT, "" + prec));
rref.SetOption(LeftRecursiveRuleTransformer.PRECEDENCE_OPTION_NAME, dummyValueNode);
}
}
return t;
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return(false);
}
IList <GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return(false);
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return(false);
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return(false);
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList <GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return(true);
}
public override void DiscoverOuterAlt(AltAST alt)
{
if (alt.altLabel != null)
{
IList<System.Tuple<int, AltAST>> list;
if (!labeledAlternatives.TryGetValue(alt.altLabel.Text, out list) || list == null)
{
list = new List<System.Tuple<int, AltAST>>();
labeledAlternatives[alt.altLabel.Text] = list;
}
list.Add(Tuple.Create(currentOuterAltNumber, alt));
}
else
{
unlabeledAlternatives.Add(alt);
}
}
public override void DiscoverOuterAlt(AltAST alt)
{
currentRule.alt[currentOuterAltNumber].ast = alt;
}
/** Build lexer grammar from combined grammar that looks like:
*
* (COMBINED_GRAMMAR A
* (tokens { X (= Y 'y'))
* (OPTIONS (= x 'y'))
* (@ members {foo})
* (@ lexer header {package jj;})
* (RULES (RULE .+)))
*
* Move rules and actions to new tree, don't dup. Split AST apart.
* We'll have this Grammar share token symbols later; don't generate
* tokenVocab or tokens{} section. Copy over named actions.
*
* Side-effects: it removes children from GRAMMAR & RULES nodes
* in combined AST. Anything cut out is dup'd before
* adding to lexer to avoid "who's ur daddy" issues
*/
public virtual GrammarRootAST ExtractImplicitLexer(Grammar combinedGrammar)
{
GrammarRootAST combinedAST = combinedGrammar.ast;
//tool.log("grammar", "before="+combinedAST.toStringTree());
GrammarASTAdaptor adaptor = new GrammarASTAdaptor(combinedAST.Token.InputStream);
GrammarAST[] elements = combinedAST.GetChildrenAsArray();
// MAKE A GRAMMAR ROOT and ID
string lexerName = combinedAST.GetChild(0).Text + "Lexer";
GrammarRootAST lexerAST =
new GrammarRootAST(new CommonToken(ANTLRParser.GRAMMAR, "LEXER_GRAMMAR"), combinedGrammar.ast.tokenStream);
lexerAST.grammarType = ANTLRParser.LEXER;
lexerAST.Token.InputStream = combinedAST.Token.InputStream;
lexerAST.AddChild((ITree)adaptor.Create(ANTLRParser.ID, lexerName));
// COPY OPTIONS
GrammarAST optionsRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.OPTIONS);
if (optionsRoot != null && optionsRoot.ChildCount != 0)
{
GrammarAST lexerOptionsRoot = (GrammarAST)adaptor.DupNode(optionsRoot);
lexerAST.AddChild(lexerOptionsRoot);
GrammarAST[] options = optionsRoot.GetChildrenAsArray();
foreach (GrammarAST o in options)
{
string optionName = o.GetChild(0).Text;
if (Grammar.lexerOptions.Contains(optionName) &&
!Grammar.doNotCopyOptionsToLexer.Contains(optionName))
{
GrammarAST optionTree = (GrammarAST)adaptor.DupTree(o);
lexerOptionsRoot.AddChild(optionTree);
lexerAST.SetOption(optionName, (GrammarAST)optionTree.GetChild(1));
}
}
}
// COPY all named actions, but only move those with lexer:: scope
IList <GrammarAST> actionsWeMoved = new List <GrammarAST>();
foreach (GrammarAST e in elements)
{
if (e.Type == ANTLRParser.AT)
{
lexerAST.AddChild((ITree)adaptor.DupTree(e));
if (e.GetChild(0).Text.Equals("lexer"))
{
actionsWeMoved.Add(e);
}
}
}
foreach (GrammarAST r in actionsWeMoved)
{
combinedAST.DeleteChild(r);
}
GrammarAST combinedRulesRoot =
(GrammarAST)combinedAST.GetFirstChildWithType(ANTLRParser.RULES);
if (combinedRulesRoot == null)
{
return(lexerAST);
}
// MOVE lexer rules
GrammarAST lexerRulesRoot = (GrammarAST)adaptor.Create(ANTLRParser.RULES, "RULES");
lexerAST.AddChild(lexerRulesRoot);
IList <GrammarAST> rulesWeMoved = new List <GrammarAST>();
GrammarASTWithOptions[] rules;
if (combinedRulesRoot.ChildCount > 0)
{
rules = combinedRulesRoot.Children.Cast <GrammarASTWithOptions>().ToArray();
}
else
{
rules = new GrammarASTWithOptions[0];
}
foreach (GrammarASTWithOptions r in rules)
{
string ruleName = r.GetChild(0).Text;
if (Grammar.IsTokenName(ruleName))
{
lexerRulesRoot.AddChild((ITree)adaptor.DupTree(r));
rulesWeMoved.Add(r);
}
}
foreach (GrammarAST r in rulesWeMoved)
{
combinedRulesRoot.DeleteChild(r);
}
// Will track 'if' from IF : 'if' ; rules to avoid defining new token for 'if'
IList <System.Tuple <GrammarAST, GrammarAST> > litAliases =
Grammar.GetStringLiteralAliasesFromLexerRules(lexerAST);
ISet <string> stringLiterals = combinedGrammar.GetStringLiterals();
// add strings from combined grammar (and imported grammars) into lexer
// put them first as they are keywords; must resolve ambigs to these rules
// tool.log("grammar", "strings from parser: "+stringLiterals);
int insertIndex = 0;
foreach (string lit in stringLiterals)
{
// if lexer already has a rule for literal, continue
if (litAliases != null)
{
foreach (System.Tuple <GrammarAST, GrammarAST> pair in litAliases)
{
GrammarAST litAST = pair.Item2;
if (lit.Equals(litAST.Text))
{
goto continueNextLit;
}
}
}
// create for each literal: (RULE <uniquename> (BLOCK (ALT <lit>))
string rname = combinedGrammar.GetStringLiteralLexerRuleName(lit);
// can't use wizard; need special node types
GrammarAST litRule = new RuleAST(ANTLRParser.RULE);
BlockAST blk = new BlockAST(ANTLRParser.BLOCK);
AltAST alt = new AltAST(ANTLRParser.ALT);
TerminalAST slit = new TerminalAST(new CommonToken(ANTLRParser.STRING_LITERAL, lit));
alt.AddChild(slit);
blk.AddChild(alt);
CommonToken idToken = new CommonToken(ANTLRParser.TOKEN_REF, rname);
litRule.AddChild(new TerminalAST(idToken));
litRule.AddChild(blk);
lexerRulesRoot.InsertChild(insertIndex, litRule);
// lexerRulesRoot.getChildren().add(0, litRule);
lexerRulesRoot.FreshenParentAndChildIndexes(); // reset indexes and set litRule parent
// next literal will be added after the one just added
insertIndex++;
continueNextLit:
;
}
// TODO: take out after stable if slow
lexerAST.SanityCheckParentAndChildIndexes();
combinedAST.SanityCheckParentAndChildIndexes();
// tool.log("grammar", combinedAST.toTokenString());
combinedGrammar.tool.Log("grammar", "after extract implicit lexer =" + combinedAST.ToStringTree());
combinedGrammar.tool.Log("grammar", "lexer =" + lexerAST.ToStringTree());
if (lexerRulesRoot.ChildCount == 0)
{
return(null);
}
return(lexerAST);
}
protected override void ExitAlternative(AltAST tree)
{
frequencies.Push(CombineMax(frequencies.Pop(), frequencies.Pop()));
minFrequencies.Push(combineMin(minFrequencies.Pop(), minFrequencies.Pop()));
}
/** Given list of X and r refs in alt, compute how many of each there are */
protected virtual System.Tuple<FrequencySet<string>, FrequencySet<string>> GetElementFrequenciesForAlt(AltAST ast)
{
try
{
ElementFrequenciesVisitor visitor = new ElementFrequenciesVisitor(rule.g, new CommonTreeNodeStream(new GrammarASTAdaptor(), ast));
visitor.outerAlternative();
if (visitor.frequencies.Count != 1)
{
factory.GetGrammar().tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR);
return Tuple.Create(new FrequencySet<string>(), new FrequencySet<string>());
}
return Tuple.Create(visitor.GetMinFrequencies(), visitor.frequencies.Peek());
}
catch (RecognitionException ex)
{
factory.GetGrammar().tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, ex);
return Tuple.Create(new FrequencySet<string>(), new FrequencySet<string>());
}
}
protected override void ExitAlternative(AltAST tree)
{
frequencies.Push(CombineMax(frequencies.Pop(), frequencies.Pop()));
minFrequencies.Push(combineMin(minFrequencies.Pop(), minFrequencies.Pop()));
}
public override void SetAltAssoc(AltAST t, int alt)
{
ASSOC assoc = ASSOC.left;
if (t.GetOptions() != null)
{
string a = t.GetOptionString("assoc");
if (a != null)
{
if (a.Equals(ASSOC.right.ToString()))
{
assoc = ASSOC.right;
}
else if (a.Equals(ASSOC.left.ToString()))
{
assoc = ASSOC.left;
}
else
{
tool.errMgr.GrammarError(ErrorType.ILLEGAL_OPTION_VALUE, t.g.fileName, t.GetOptionAST("assoc").Token, "assoc", assoc);
}
}
}
if (altAssociativity.ContainsKey(alt) && altAssociativity[alt] != assoc)
{
tool.errMgr.ToolError(ErrorType.INTERNAL_ERROR, "all operators of alt " + alt + " of left-recursive rule must have same associativity");
}
altAssociativity[alt] = assoc;
// System.out.println("setAltAssoc: op " + alt + ": " + t.getText()+", assoc="+assoc);
}
public override void DiscoverOuterAlt(AltAST alt)
{
currentRule.alt[currentOuterAltNumber].ast = alt;
}
public override void DiscoverOuterAlt(AltAST alt)
{
if (alt.altLabel != null)
{
ruleToAltLabels.Map(currentRuleName, alt.altLabel);
string altLabel = alt.altLabel.Text;
altLabelToRuleName[Utils.Capitalize(altLabel)] = currentRuleName;
altLabelToRuleName[Utils.Decapitalize(altLabel)] = currentRuleName;
}
}
