public virtual IList <GrammarAST> GetNodesWithType(IntervalSet types)
{
IList <GrammarAST> nodes = new List <GrammarAST>();
LinkedList <GrammarAST> work = new LinkedList <GrammarAST>();
work.AddLast(this);
GrammarAST t;
while (work.Count > 0)
{
t = work.First.Value;
work.RemoveFirst();
if (types == null || types.Contains(t.Type))
{
nodes.Add(t);
}
if (t.Children != null)
{
foreach (var child in t.GetChildrenAsArray())
{
work.AddLast(child);
}
}
}
return(nodes);
}
protected virtual void CheckSetCollision(GrammarAST ast, IntervalSet set, int el)
{
if (set.Contains(el))
{
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.Token,
(char)el, ast.Text);
}
}
public virtual ATN CreateATN()
{
_CreateATN(g.rules.Values);
Debug.Assert(atn.maxTokenType == g.GetMaxTokenType());
AddRuleFollowLinks();
AddEOFTransitionToStartRules();
ATNOptimizer.Optimize(g, atn);
foreach (System.Tuple <Rule, ATNState, ATNState> pair in preventEpsilonClosureBlocks)
{
LL1Analyzer analyzer = new LL1Analyzer(atn);
ATNState blkStart = pair.Item2;
ATNState blkStop = pair.Item3;
IntervalSet lookahead = analyzer.Look(blkStart, blkStop, PredictionContext.EmptyLocal);
if (lookahead.Contains(Antlr4.Runtime.TokenConstants.Epsilon))
{
ErrorType errorType = pair.Item1 is LeftRecursiveRule ? ErrorType.EPSILON_LR_FOLLOW : ErrorType.EPSILON_CLOSURE;
g.tool.errMgr.GrammarError(errorType, g.fileName, ((GrammarAST)pair.Item1.ast.GetChild(0)).Token, pair.Item1.name);
}
}
foreach (System.Tuple <Rule, ATNState, ATNState> pair in preventEpsilonOptionalBlocks)
{
int bypassCount = 0;
for (int i = 0; i < pair.Item2.NumberOfTransitions; i++)
{
ATNState startState = pair.Item2.Transition(i).target;
if (startState == pair.Item3)
{
bypassCount++;
continue;
}
LL1Analyzer analyzer = new LL1Analyzer(atn);
if (analyzer.Look(startState, pair.Item3, PredictionContext.EmptyLocal).Contains(Antlr4.Runtime.TokenConstants.Epsilon))
{
g.tool.errMgr.GrammarError(ErrorType.EPSILON_OPTIONAL, g.fileName, ((GrammarAST)pair.Item1.ast.GetChild(0)).Token, pair.Item1.name);
goto continueOptionalCheck;
}
}
if (bypassCount != 1)
{
throw new InvalidOperationException("Expected optional block with exactly 1 bypass alternative.");
}
continueOptionalCheck:
;
}
return(atn);
}
public virtual void GetNodesWithTypePreorderDFS_(IList <GrammarAST> nodes, IntervalSet types)
{
if (types.Contains(this.Type))
{
nodes.Add(this);
}
// walk all children of root.
for (int i = 0; i < ChildCount; i++)
{
GrammarAST child = (GrammarAST)GetChild(i);
child.GetNodesWithTypePreorderDFS_(nodes, types);
}
}
protected virtual void ProcessLexer()
{
// make sure all non-fragment lexer rules must match at least one symbol
foreach (Rule rule in g.rules.Values)
{
if (rule.IsFragment())
{
continue;
}
LL1Analyzer analyzer = new LL1Analyzer(g.atn);
IntervalSet look = analyzer.Look(g.atn.ruleToStartState[rule.index], PredictionContext.EmptyLocal);
if (look.Contains(TokenConstants.Epsilon))
{
g.tool.errMgr.GrammarError(ErrorType.EPSILON_TOKEN, g.fileName, ((GrammarAST)rule.ast.GetChild(0)).Token, rule.name);
}
}
}
protected virtual void CheckSetCollision(GrammarAST ast, IntervalSet set, int a, int b)
{
for (int i = a; i <= b; i++)
{
if (set.Contains(i))
{
string setText;
if (ast.Children == null)
{
setText = ast.Text;
}
else
{
StringBuilder sb = new StringBuilder();
foreach (object child in ast.Children)
{
if (child is RangeAST)
{
sb.Append(((RangeAST)child).GetChild(0).Text);
sb.Append("..");
sb.Append(((RangeAST)child).GetChild(1).Text);
}
else
{
sb.Append(((GrammarAST)child).Text);
}
sb.Append(" | ");
}
sb.Remove(sb.Length - 3, 3);
setText = sb.ToString();
}
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.Token,
(char)a + "-" + (char)b, setText);
break;
}
}
}
protected virtual void CheckSetCollision(GrammarAST ast, IntervalSet set, int a, int b)
{
for (int i = a; i <= b; i++)
{
if (set.Contains(i))
{
string setText;
if (ast.Children == null)
{
setText = ast.Text;
}
else
{
StringBuilder sb = new StringBuilder();
foreach (object child in ast.Children)
{
if (child is RangeAST)
{
sb.Append(((RangeAST)child).GetChild(0).Text);
sb.Append("..");
sb.Append(((RangeAST)child).GetChild(1).Text);
}
else
{
sb.Append(((GrammarAST)child).Text);
}
sb.Append(" | ");
}
sb.Remove(sb.Length - 3, 3);
setText = sb.ToString();
}
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.Token,
(char)a + "-" + (char)b, setText);
break;
}
}
}
protected virtual void CheckSetCollision(GrammarAST ast, IntervalSet set, int el)
{
if (set.Contains(el))
{
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, ast.Token,
(char)el, ast.Text);
}
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return(false);
}
IList <GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return(false);
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return(false);
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return(false);
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList <GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return(true);
}
public virtual string Text(GrammarAST t)
{
if (t == null)
{
return("");
}
int tokenStartIndex = t.TokenStartIndex;
int tokenStopIndex = t.TokenStopIndex;
// ignore tokens from existing option subtrees like:
// (ELEMENT_OPTIONS (= assoc right))
//
// element options are added back according to the values in the map
// returned by getOptions().
IntervalSet ignore = new IntervalSet();
IList <GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS);
foreach (GrammarAST sub in optionsSubTrees)
{
ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex);
}
// Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
// but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
// and add the tokenIndex option when writing these tokens.
IntervalSet noOptions = new IntervalSet();
IList <GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
foreach (GrammarAST sub in labeledSubTrees)
{
noOptions.Add(sub.GetChild(0).TokenStartIndex);
}
StringBuilder buf = new StringBuilder();
int i = tokenStartIndex;
while (i <= tokenStopIndex)
{
if (ignore.Contains(i))
{
i++;
continue;
}
IToken tok = tokenStream.Get(i);
// Compute/hold any element options
StringBuilder elementOptions = new StringBuilder();
if (!noOptions.Contains(i))
{
GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex);
if (node != null &&
(tok.Type == TOKEN_REF ||
tok.Type == STRING_LITERAL ||
tok.Type == RULE_REF))
{
elementOptions.Append("tokenIndex=").Append(tok.TokenIndex);
}
if (node is GrammarASTWithOptions)
{
GrammarASTWithOptions o = (GrammarASTWithOptions)node;
foreach (KeyValuePair <string, GrammarAST> entry in o.GetOptions())
{
if (elementOptions.Length > 0)
{
elementOptions.Append(',');
}
elementOptions.Append(entry.Key);
elementOptions.Append('=');
elementOptions.Append(entry.Value.Text);
}
}
}
buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative
i++; // move to the next token
// Are there args on a rule?
if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION)
{
buf.Append('[' + tokenStream.Get(i).Text + ']');
i++;
}
// now that we have the actual element, we can add the options.
if (elementOptions.Length > 0)
{
buf.Append('<').Append(elementOptions).Append('>');
}
}
return(buf.ToString());
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return false;
}
IList<GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return false;
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return false;
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return false;
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList<GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return true;
}
private static void OptimizeSets(Grammar g, ATN atn)
{
if (g.IsParser())
{
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
IList <DecisionState> decisions = atn.decisionToState;
foreach (DecisionState decision in decisions)
{
if (decision.ruleIndex >= 0)
{
Rule rule = g.GetRule(decision.ruleIndex);
if (char.IsLower(rule.name[0]))
{
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.NumberOfTransitions; i++)
{
Transition epsTransition = decision.Transition(i);
if (!(epsTransition is EpsilonTransition))
{
continue;
}
if (epsTransition.target.NumberOfTransitions != 1)
{
continue;
}
Transition transition = epsTransition.target.Transition(0);
if (!(transition.target is BlockEndState))
{
continue;
}
if (transition is NotSetTransition)
{
// TODO: not yet implemented
continue;
}
if (transition is AtomTransition ||
transition is RangeTransition ||
transition is SetTransition)
{
setTransitions.Add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
{
Interval interval = setTransitions.GetIntervals()[i];
if (interval.Length <= 1)
{
continue;
}
ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++)
{
Transition matchTransition = decision.Transition(j).target.Transition(0);
if (matchTransition is NotSetTransition)
{
throw new NotImplementedException();
}
IntervalSet set = matchTransition.Label;
int minElem = set.MinElement;
int maxElem = set.MaxElement;
for (int k = minElem; k <= maxElem; k++)
{
if (matchSet.Contains(k))
{
char setMin = (char)set.MinElement;
char setMax = (char)set.MaxElement;
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
break;
}
}
matchSet.AddAll(set);
}
Transition newTransition;
if (matchSet.GetIntervals().Count == 1)
{
if (matchSet.Count == 1)
{
newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
}
else
{
Interval matchInterval = matchSet.GetIntervals()[0];
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
}
else
{
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.Transition(interval.a).target.SetTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++)
{
Transition removed = decision.Transition(interval.a + 1);
decision.RemoveTransition(interval.a + 1);
atn.RemoveState(removed.target);
removedStates++;
}
}
}
//System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
public virtual string Text(GrammarAST t)
{
if (t == null)
return "";
int tokenStartIndex = t.TokenStartIndex;
int tokenStopIndex = t.TokenStopIndex;
// ignore tokens from existing option subtrees like:
// (ELEMENT_OPTIONS (= assoc right))
//
// element options are added back according to the values in the map
// returned by getOptions().
IntervalSet ignore = new IntervalSet();
IList<GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS);
foreach (GrammarAST sub in optionsSubTrees)
{
ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex);
}
// Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
// but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
// and add the tokenIndex option when writing these tokens.
IntervalSet noOptions = new IntervalSet();
IList<GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
foreach (GrammarAST sub in labeledSubTrees)
{
noOptions.Add(sub.GetChild(0).TokenStartIndex);
}
StringBuilder buf = new StringBuilder();
int i = tokenStartIndex;
while (i <= tokenStopIndex)
{
if (ignore.Contains(i))
{
i++;
continue;
}
IToken tok = tokenStream.Get(i);
// Compute/hold any element options
StringBuilder elementOptions = new StringBuilder();
if (!noOptions.Contains(i))
{
GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex);
if (node != null &&
(tok.Type == TOKEN_REF ||
tok.Type == STRING_LITERAL ||
tok.Type == RULE_REF))
{
elementOptions.Append("tokenIndex=").Append(tok.TokenIndex);
}
if (node is GrammarASTWithOptions)
{
GrammarASTWithOptions o = (GrammarASTWithOptions)node;
foreach (KeyValuePair<string, GrammarAST> entry in o.GetOptions())
{
if (elementOptions.Length > 0)
{
elementOptions.Append(',');
}
elementOptions.Append(entry.Key);
elementOptions.Append('=');
elementOptions.Append(entry.Value.Text);
}
}
}
buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative
i++; // move to the next token
// Are there args on a rule?
if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION)
{
buf.Append('[' + tokenStream.Get(i).Text + ']');
i++;
}
// now that we have the actual element, we can add the options.
if (elementOptions.Length > 0)
{
buf.Append('<').Append(elementOptions).Append('>');
}
}
return buf.ToString();
}
public virtual IList<GrammarAST> GetNodesWithType(IntervalSet types)
{
IList<GrammarAST> nodes = new List<GrammarAST>();
LinkedList<GrammarAST> work = new LinkedList<GrammarAST>();
work.AddLast(this);
GrammarAST t;
while (work.Count > 0)
{
t = work.First.Value;
work.RemoveFirst();
if (types == null || types.Contains(t.Type))
nodes.Add(t);
if (t.Children != null)
{
foreach (var child in t.GetChildrenAsArray())
work.AddLast(child);
}
}
return nodes;
}
public virtual void GetNodesWithTypePreorderDFS_(IList<GrammarAST> nodes, IntervalSet types)
{
if (types.Contains(this.Type))
nodes.Add(this);
// walk all children of root.
for (int i = 0; i < ChildCount; i++)
{
GrammarAST child = (GrammarAST)GetChild(i);
child.GetNodesWithTypePreorderDFS_(nodes, types);
}
}
private static void OptimizeSets(Grammar g, ATN atn)
{
if (g.IsParser())
{
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
IList<DecisionState> decisions = atn.decisionToState;
foreach (DecisionState decision in decisions)
{
if (decision.ruleIndex >= 0)
{
Rule rule = g.GetRule(decision.ruleIndex);
if (char.IsLower(rule.name[0]))
{
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.NumberOfTransitions; i++)
{
Transition epsTransition = decision.Transition(i);
if (!(epsTransition is EpsilonTransition))
{
continue;
}
if (epsTransition.target.NumberOfTransitions != 1)
{
continue;
}
Transition transition = epsTransition.target.Transition(0);
if (!(transition.target is BlockEndState))
{
continue;
}
if (transition is NotSetTransition)
{
// TODO: not yet implemented
continue;
}
if (transition is AtomTransition
|| transition is RangeTransition
|| transition is SetTransition)
{
setTransitions.Add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
{
Interval interval = setTransitions.GetIntervals()[i];
if (interval.Length <= 1)
{
continue;
}
ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++)
{
Transition matchTransition = decision.Transition(j).target.Transition(0);
if (matchTransition is NotSetTransition)
{
throw new NotImplementedException();
}
IntervalSet set = matchTransition.Label;
int minElem = set.MinElement;
int maxElem = set.MaxElement;
for (int k = minElem; k <= maxElem; k++)
{
if (matchSet.Contains(k))
{
char setMin = (char)set.MinElement;
char setMax = (char)set.MaxElement;
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
break;
}
}
matchSet.AddAll(set);
}
Transition newTransition;
if (matchSet.GetIntervals().Count == 1)
{
if (matchSet.Count == 1)
{
newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
}
else
{
Interval matchInterval = matchSet.GetIntervals()[0];
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
}
else
{
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.Transition(interval.a).target.SetTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++)
{
Transition removed = decision.Transition(interval.a + 1);
decision.RemoveTransition(interval.a + 1);
atn.RemoveState(removed.target);
removedStates++;
}
}
}
//System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}