public virtual IntervalSet GetSetFromCharSetLiteral(GrammarAST charSetAST)
{
string chars = charSetAST.Text;
chars = chars.Substring(1, chars.Length - 2);
string cset = '"' + chars + '"';
IntervalSet set = new IntervalSet();
if (chars.Length == 0)
{
g.tool.errMgr.GrammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED,
g.fileName, charSetAST.Token, "[]");
return(set);
}
// unescape all valid escape char like \n, leaving escaped dashes as '\-'
// so we can avoid seeing them as '-' range ops.
chars = CharSupport.GetStringFromGrammarStringLiteral(cset);
if (chars == null)
{
g.tool.errMgr.GrammarError(ErrorType.INVALID_ESCAPE_SEQUENCE,
g.fileName, charSetAST.Token);
return(set);
}
int n = chars.Length;
// now make x-y become set of char
for (int i = 0; i < n; i++)
{
int c = chars[i];
if (c == '\\' && (i + 1) < n && chars[i + 1] == '-')
{ // \-
CheckSetCollision(charSetAST, set, '-');
set.Add('-');
i++;
}
else if ((i + 2) < n && chars[i + 1] == '-')
{ // range x-y
int x = c;
int y = chars[i + 2];
if (x <= y)
{
CheckSetCollision(charSetAST, set, x, y);
set.Add(x, y);
}
else
{
g.tool.errMgr.GrammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED,
g.fileName, charSetAST.Token, "[" + (char)x + "-" + (char)y + "]");
}
i += 2;
}
else
{
CheckSetCollision(charSetAST, set, c);
set.Add(c);
}
}
return(set);
}
public static Antlr4.Runtime.Misc.IntervalSet Of(int a)
{
Antlr4.Runtime.Misc.IntervalSet s = new Antlr4.Runtime.Misc.IntervalSet();
s.Add(a);
return(s);
}
public virtual Handle Set([NotNull] GrammarAST associatedAST, [NotNull] IList <GrammarAST> terminals, bool invert)
{
ATNState left = NewState(associatedAST);
ATNState right = NewState(associatedAST);
IntervalSet set = new IntervalSet();
foreach (GrammarAST t in terminals)
{
int ttype = g.GetTokenType(t.Text);
set.Add(ttype);
}
if (invert)
{
left.AddTransition(new NotSetTransition(right, set));
}
else
{
left.AddTransition(new SetTransition(right, set));
}
associatedAST.atnState = left;
return(new Handle(left, right));
}
public virtual IntervalSet GetSetFromCharSetLiteral(GrammarAST charSetAST)
{
string chars = charSetAST.Text;
chars = chars.Substring(1, chars.Length - 2);
string cset = '"' + chars + '"';
IntervalSet set = new IntervalSet();
if (chars.Length == 0)
{
g.tool.errMgr.GrammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED,
g.fileName, charSetAST.Token, "[]");
return set;
}
// unescape all valid escape char like \n, leaving escaped dashes as '\-'
// so we can avoid seeing them as '-' range ops.
chars = CharSupport.GetStringFromGrammarStringLiteral(cset);
if (chars == null)
{
g.tool.errMgr.GrammarError(ErrorType.INVALID_ESCAPE_SEQUENCE,
g.fileName, charSetAST.Token);
return set;
}
int n = chars.Length;
// now make x-y become set of char
for (int i = 0; i < n; i++)
{
int c = chars[i];
if (c == '\\' && (i + 1) < n && chars[i + 1] == '-')
{ // \-
CheckSetCollision(charSetAST, set, '-');
set.Add('-');
i++;
}
else if ((i + 2) < n && chars[i + 1] == '-')
{ // range x-y
int x = c;
int y = chars[i + 2];
if (x <= y)
{
CheckSetCollision(charSetAST, set, x, y);
set.Add(x, y);
}
else
{
g.tool.errMgr.GrammarError(ErrorType.EMPTY_STRINGS_AND_SETS_NOT_ALLOWED,
g.fileName, charSetAST.Token, "[" + (char)x + "-" + (char)y + "]");
}
i += 2;
}
else
{
CheckSetCollision(charSetAST, set, c);
set.Add(c);
}
}
return set;
}
public override Handle Set(GrammarAST associatedAST, IList<GrammarAST> alts, bool invert)
{
ATNState left = NewState(associatedAST);
ATNState right = NewState(associatedAST);
IntervalSet set = new IntervalSet();
foreach (GrammarAST t in alts)
{
if (t.Type == ANTLRParser.RANGE)
{
int a = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(0).Text);
int b = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(1).Text);
if (CheckRange((GrammarAST)t.GetChild(0), (GrammarAST)t.GetChild(1), a, b))
{
CheckSetCollision(associatedAST, set, a, b);
set.Add(a, b);
}
}
else if (t.Type == ANTLRParser.LEXER_CHAR_SET)
{
set.AddAll(GetSetFromCharSetLiteral(t));
}
else if (t.Type == ANTLRParser.STRING_LITERAL)
{
int c = CharSupport.GetCharValueFromGrammarCharLiteral(t.Text);
if (c != -1)
{
CheckSetCollision(associatedAST, set, c);
set.Add(c);
}
else
{
g.tool.errMgr.GrammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET,
g.fileName, t.Token, t.Text);
}
}
else if (t.Type == ANTLRParser.TOKEN_REF)
{
g.tool.errMgr.GrammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET,
g.fileName, t.Token, t.Text);
}
}
if (invert)
{
left.AddTransition(new NotSetTransition(right, set));
}
else
{
Transition transition;
if (set.GetIntervals().Count == 1)
{
Interval interval = set.GetIntervals()[0];
transition = new RangeTransition(right, interval.a, interval.b);
}
else
{
transition = new SetTransition(right, set);
}
left.AddTransition(transition);
}
associatedAST.atnState = left;
return new Handle(left, right);
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return(false);
}
IList <GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return(false);
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return(false);
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return(true);
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return(false);
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList <GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return(true);
}
public virtual string Text(GrammarAST t)
{
if (t == null)
{
return("");
}
int tokenStartIndex = t.TokenStartIndex;
int tokenStopIndex = t.TokenStopIndex;
// ignore tokens from existing option subtrees like:
// (ELEMENT_OPTIONS (= assoc right))
//
// element options are added back according to the values in the map
// returned by getOptions().
IntervalSet ignore = new IntervalSet();
IList <GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS);
foreach (GrammarAST sub in optionsSubTrees)
{
ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex);
}
// Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
// but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
// and add the tokenIndex option when writing these tokens.
IntervalSet noOptions = new IntervalSet();
IList <GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
foreach (GrammarAST sub in labeledSubTrees)
{
noOptions.Add(sub.GetChild(0).TokenStartIndex);
}
StringBuilder buf = new StringBuilder();
int i = tokenStartIndex;
while (i <= tokenStopIndex)
{
if (ignore.Contains(i))
{
i++;
continue;
}
IToken tok = tokenStream.Get(i);
// Compute/hold any element options
StringBuilder elementOptions = new StringBuilder();
if (!noOptions.Contains(i))
{
GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex);
if (node != null &&
(tok.Type == TOKEN_REF ||
tok.Type == STRING_LITERAL ||
tok.Type == RULE_REF))
{
elementOptions.Append("tokenIndex=").Append(tok.TokenIndex);
}
if (node is GrammarASTWithOptions)
{
GrammarASTWithOptions o = (GrammarASTWithOptions)node;
foreach (KeyValuePair <string, GrammarAST> entry in o.GetOptions())
{
if (elementOptions.Length > 0)
{
elementOptions.Append(',');
}
elementOptions.Append(entry.Key);
elementOptions.Append('=');
elementOptions.Append(entry.Value.Text);
}
}
}
buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative
i++; // move to the next token
// Are there args on a rule?
if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION)
{
buf.Append('[' + tokenStream.Get(i).Text + ']');
i++;
}
// now that we have the actual element, we can add the options.
if (elementOptions.Length > 0)
{
buf.Append('<').Append(elementOptions).Append('>');
}
}
return(buf.ToString());
}
protected virtual bool TranslateLeftFactoredDecision(GrammarAST block, string factoredRule, bool variant, DecisionFactorMode mode, bool includeFactoredElement)
{
if (mode == DecisionFactorMode.PARTIAL_UNFACTORED && includeFactoredElement)
{
throw new ArgumentException("Cannot include the factored element in unfactored alternatives.");
}
else if (mode == DecisionFactorMode.COMBINED_FACTOR && !includeFactoredElement)
{
throw new ArgumentException("Cannot return a combined answer without the factored element.");
}
if (!ExpandOptionalQuantifiersForBlock(block, variant))
{
return false;
}
IList<GrammarAST> alternatives = block.GetAllChildrenWithType(ANTLRParser.ALT);
GrammarAST[] factoredAlternatives = new GrammarAST[alternatives.Count];
GrammarAST[] unfactoredAlternatives = new GrammarAST[alternatives.Count];
IntervalSet factoredIntervals = new IntervalSet();
IntervalSet unfactoredIntervals = new IntervalSet();
for (int i = 0; i < alternatives.Count; i++)
{
GrammarAST alternative = alternatives[i];
if (mode.IncludeUnfactoredAlts())
{
GrammarAST unfactoredAlt = TranslateLeftFactoredAlternative(alternative.DupTree(), factoredRule, variant, DecisionFactorMode.PARTIAL_UNFACTORED, false);
unfactoredAlternatives[i] = unfactoredAlt;
if (unfactoredAlt != null)
{
unfactoredIntervals.Add(i);
}
}
if (mode.IncludeFactoredAlts())
{
GrammarAST factoredAlt = TranslateLeftFactoredAlternative(alternative, factoredRule, variant, mode == DecisionFactorMode.COMBINED_FACTOR ? DecisionFactorMode.PARTIAL_FACTORED : DecisionFactorMode.FULL_FACTOR, includeFactoredElement);
factoredAlternatives[i] = factoredAlt;
if (factoredAlt != null)
{
factoredIntervals.Add(alternative.ChildIndex);
}
}
}
if (factoredIntervals.IsNil && !mode.IncludeUnfactoredAlts())
{
return false;
}
else if (unfactoredIntervals.IsNil && !mode.IncludeFactoredAlts())
{
return false;
}
if (unfactoredIntervals.IsNil && factoredIntervals.Count == alternatives.Count && mode.IncludeFactoredAlts() && !includeFactoredElement)
{
for (int i = 0; i < factoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
else if (factoredIntervals.IsNil && unfactoredIntervals.Count == alternatives.Count && mode.IncludeUnfactoredAlts())
{
for (int i = 0; i < unfactoredAlternatives.Length; i++)
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.SetChild(block, i, translatedAlt);
}
return true;
}
if (mode == DecisionFactorMode.FULL_FACTOR)
{
return false;
}
/* for a, b, c being arbitrary `element` trees, this block performs
* this transformation:
*
* factoredElement a
* | factoredElement b
* | factoredElement c
* | ...
*
* ==>
*
* factoredElement (a | b | c | ...)
*/
GrammarAST newChildren = (GrammarAST)adaptor.Nil();
for (int i = 0; i < alternatives.Count; i++)
{
if (mode.IncludeFactoredAlts() && factoredIntervals.Contains(i))
{
bool combineWithPrevious = i > 0 && factoredIntervals.Contains(i - 1) && (!mode.IncludeUnfactoredAlts() || !unfactoredIntervals.Contains(i - 1));
if (combineWithPrevious)
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
GrammarAST previous = (GrammarAST)newChildren.GetChild(newChildren.ChildCount - 1);
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
LOGGER.log(Level.FINE, translatedAlt.ToStringTree());
}
#endif
if (previous.ChildCount == 1 || previous.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (previous.ChildCount > 1)
{
adaptor.AddChild(newAlt, previous.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(previous, newBlock);
}
if (translatedAlt.ChildCount == 1 || translatedAlt.GetChild(1).Type != ANTLRParser.BLOCK)
{
GrammarAST newBlock = new BlockAST(adaptor.CreateToken(ANTLRParser.BLOCK, "BLOCK"));
GrammarAST newAlt = new AltAST(adaptor.CreateToken(ANTLRParser.ALT, "ALT"));
adaptor.AddChild(newBlock, newAlt);
while (translatedAlt.ChildCount > 1)
{
adaptor.AddChild(newAlt, translatedAlt.DeleteChild(1));
}
if (newAlt.ChildCount == 0)
{
adaptor.AddChild(newAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(translatedAlt, newBlock);
}
GrammarAST combinedBlock = (GrammarAST)previous.GetChild(1);
adaptor.AddChild(combinedBlock, translatedAlt.GetChild(1).GetChild(0));
#if false
if (LOGGER.isLoggable(Level.FINE))
{
LOGGER.log(Level.FINE, previous.ToStringTree());
}
#endif
}
else
{
GrammarAST translatedAlt = factoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
if (mode.IncludeUnfactoredAlts() && unfactoredIntervals.Contains(i))
{
GrammarAST translatedAlt = unfactoredAlternatives[i];
if (translatedAlt.ChildCount == 0)
{
adaptor.AddChild(translatedAlt, adaptor.Create(ANTLRParser.EPSILON, "EPSILON"));
}
adaptor.AddChild(newChildren, translatedAlt);
}
}
adaptor.ReplaceChildren(block, 0, block.ChildCount - 1, newChildren);
if (!variant && block.Parent is RuleAST)
{
RuleAST ruleAST = (RuleAST)block.Parent;
string ruleName = ruleAST.GetChild(0).Text;
Rule r = _rules[ruleName];
IList<GrammarAST> blockAlts = block.GetAllChildrenWithType(ANTLRParser.ALT);
r.numberOfAlts = blockAlts.Count;
r.alt = new Alternative[blockAlts.Count + 1];
for (int i = 0; i < blockAlts.Count; i++)
{
r.alt[i + 1] = new Alternative(r, i + 1);
r.alt[i + 1].ast = (AltAST)blockAlts[i];
}
}
return true;
}
/// <summary>Create a set with all ints within range [a..b] (inclusive)</summary>
public static Antlr4.Runtime.Misc.IntervalSet Of(int a, int b)
{
Antlr4.Runtime.Misc.IntervalSet s = new Antlr4.Runtime.Misc.IntervalSet();
s.Add(a, b);
return s;
}
private static void OptimizeSets(Grammar g, ATN atn)
{
if (g.IsParser())
{
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
IList <DecisionState> decisions = atn.decisionToState;
foreach (DecisionState decision in decisions)
{
if (decision.ruleIndex >= 0)
{
Rule rule = g.GetRule(decision.ruleIndex);
if (char.IsLower(rule.name[0]))
{
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.NumberOfTransitions; i++)
{
Transition epsTransition = decision.Transition(i);
if (!(epsTransition is EpsilonTransition))
{
continue;
}
if (epsTransition.target.NumberOfTransitions != 1)
{
continue;
}
Transition transition = epsTransition.target.Transition(0);
if (!(transition.target is BlockEndState))
{
continue;
}
if (transition is NotSetTransition)
{
// TODO: not yet implemented
continue;
}
if (transition is AtomTransition ||
transition is RangeTransition ||
transition is SetTransition)
{
setTransitions.Add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
{
Interval interval = setTransitions.GetIntervals()[i];
if (interval.Length <= 1)
{
continue;
}
ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++)
{
Transition matchTransition = decision.Transition(j).target.Transition(0);
if (matchTransition is NotSetTransition)
{
throw new NotImplementedException();
}
IntervalSet set = matchTransition.Label;
int minElem = set.MinElement;
int maxElem = set.MaxElement;
for (int k = minElem; k <= maxElem; k++)
{
if (matchSet.Contains(k))
{
char setMin = (char)set.MinElement;
char setMax = (char)set.MaxElement;
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
break;
}
}
matchSet.AddAll(set);
}
Transition newTransition;
if (matchSet.GetIntervals().Count == 1)
{
if (matchSet.Count == 1)
{
newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
}
else
{
Interval matchInterval = matchSet.GetIntervals()[0];
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
}
else
{
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.Transition(interval.a).target.SetTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++)
{
Transition removed = decision.Transition(interval.a + 1);
decision.RemoveTransition(interval.a + 1);
atn.RemoveState(removed.target);
removedStates++;
}
}
}
//System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
public override Handle Set(GrammarAST associatedAST, IList <GrammarAST> alts, bool invert)
{
ATNState left = NewState(associatedAST);
ATNState right = NewState(associatedAST);
IntervalSet set = new IntervalSet();
foreach (GrammarAST t in alts)
{
if (t.Type == ANTLRParser.RANGE)
{
int a = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(0).Text);
int b = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(1).Text);
if (CheckRange((GrammarAST)t.GetChild(0), (GrammarAST)t.GetChild(1), a, b))
{
CheckSetCollision(associatedAST, set, a, b);
set.Add(a, b);
}
}
else if (t.Type == ANTLRParser.LEXER_CHAR_SET)
{
set.AddAll(GetSetFromCharSetLiteral(t));
}
else if (t.Type == ANTLRParser.STRING_LITERAL)
{
int c = CharSupport.GetCharValueFromGrammarCharLiteral(t.Text);
if (c != -1)
{
CheckSetCollision(associatedAST, set, c);
set.Add(c);
}
else
{
g.tool.errMgr.GrammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET,
g.fileName, t.Token, t.Text);
}
}
else if (t.Type == ANTLRParser.TOKEN_REF)
{
g.tool.errMgr.GrammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET,
g.fileName, t.Token, t.Text);
}
}
if (invert)
{
left.AddTransition(new NotSetTransition(right, set));
}
else
{
Transition transition;
if (set.GetIntervals().Count == 1)
{
Interval interval = set.GetIntervals()[0];
transition = new RangeTransition(right, interval.a, interval.b);
}
else
{
transition = new SetTransition(right, set);
}
left.AddTransition(transition);
}
associatedAST.atnState = left;
return(new Handle(left, right));
}
public virtual string Text(GrammarAST t)
{
if (t == null)
return "";
int tokenStartIndex = t.TokenStartIndex;
int tokenStopIndex = t.TokenStopIndex;
// ignore tokens from existing option subtrees like:
// (ELEMENT_OPTIONS (= assoc right))
//
// element options are added back according to the values in the map
// returned by getOptions().
IntervalSet ignore = new IntervalSet();
IList<GrammarAST> optionsSubTrees = t.GetNodesWithType(ELEMENT_OPTIONS);
foreach (GrammarAST sub in optionsSubTrees)
{
ignore.Add(sub.TokenStartIndex, sub.TokenStopIndex);
}
// Individual labels appear as RULE_REF or TOKEN_REF tokens in the tree,
// but do not support the ELEMENT_OPTIONS syntax. Make sure to not try
// and add the tokenIndex option when writing these tokens.
IntervalSet noOptions = new IntervalSet();
IList<GrammarAST> labeledSubTrees = t.GetNodesWithType(new IntervalSet(ASSIGN, PLUS_ASSIGN));
foreach (GrammarAST sub in labeledSubTrees)
{
noOptions.Add(sub.GetChild(0).TokenStartIndex);
}
StringBuilder buf = new StringBuilder();
int i = tokenStartIndex;
while (i <= tokenStopIndex)
{
if (ignore.Contains(i))
{
i++;
continue;
}
IToken tok = tokenStream.Get(i);
// Compute/hold any element options
StringBuilder elementOptions = new StringBuilder();
if (!noOptions.Contains(i))
{
GrammarAST node = t.GetNodeWithTokenIndex(tok.TokenIndex);
if (node != null &&
(tok.Type == TOKEN_REF ||
tok.Type == STRING_LITERAL ||
tok.Type == RULE_REF))
{
elementOptions.Append("tokenIndex=").Append(tok.TokenIndex);
}
if (node is GrammarASTWithOptions)
{
GrammarASTWithOptions o = (GrammarASTWithOptions)node;
foreach (KeyValuePair<string, GrammarAST> entry in o.GetOptions())
{
if (elementOptions.Length > 0)
{
elementOptions.Append(',');
}
elementOptions.Append(entry.Key);
elementOptions.Append('=');
elementOptions.Append(entry.Value.Text);
}
}
}
buf.Append(tok.Text); // add actual text of the current token to the rewritten alternative
i++; // move to the next token
// Are there args on a rule?
if (tok.Type == RULE_REF && i <= tokenStopIndex && tokenStream.Get(i).Type == ARG_ACTION)
{
buf.Append('[' + tokenStream.Get(i).Text + ']');
i++;
}
// now that we have the actual element, we can add the options.
if (elementOptions.Length > 0)
{
buf.Append('<').Append(elementOptions).Append('>');
}
}
return buf.ToString();
}
public virtual void Process()
{
CodeGenerator gen = new CodeGenerator(g);
AbstractTarget target = gen.GetTarget();
if (target == null)
{
return;
}
IntervalSet idTypes = new IntervalSet();
idTypes.Add(ANTLRParser.ID);
idTypes.Add(ANTLRParser.RULE_REF);
idTypes.Add(ANTLRParser.TOKEN_REF);
IList<GrammarAST> idNodes = g.ast.GetNodesWithType(idTypes);
foreach (GrammarAST idNode in idNodes)
{
if (target.GrammarSymbolCausesIssueInGeneratedCode(idNode))
{
g.tool.errMgr.GrammarError(ErrorType.USE_OF_BAD_WORD,
g.fileName, idNode.Token,
idNode.Text);
}
}
// all templates are generated in memory to report the most complete
// error information possible, but actually writing output files stops
// after the first error is reported
int errorCount = g.tool.errMgr.GetNumErrors();
if (g.IsLexer())
{
if (target.NeedsHeader())
{
Template lexerHeader = gen.GenerateLexer(true); // Header file if needed.
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(lexerHeader, gen, true);
}
}
Template lexer = gen.GenerateLexer(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(lexer, gen, false);
}
}
else
{
if (target.NeedsHeader())
{
Template parserHeader = gen.GenerateParser(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(parserHeader, gen, true);
}
}
Template parser = gen.GenerateParser(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(parser, gen, false);
}
if (g.tool.gen_listener)
{
if (target.NeedsHeader())
{
Template listenerHeader = gen.GenerateListener(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteListener(listenerHeader, true);
}
}
Template listener = gen.GenerateListener(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteListener(listener, false);
}
if (target.NeedsHeader())
{
Template baseListener = gen.GenerateBaseListener(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseListener(baseListener, true);
}
}
if (target.WantsBaseListener())
{
Template baseListener = gen.GenerateBaseListener(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseListener(baseListener, false);
}
}
}
if (g.tool.gen_visitor)
{
if (target.NeedsHeader())
{
Template visitorHeader = gen.GenerateVisitor(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteVisitor(visitorHeader, true);
}
}
Template visitor = gen.GenerateVisitor(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteVisitor(visitor, false);
}
if (target.NeedsHeader())
{
Template baseVisitor = gen.GenerateBaseVisitor(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseVisitor(baseVisitor, true);
}
}
if (target.WantsBaseVisitor())
{
Template baseVisitor = gen.GenerateBaseVisitor(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseVisitor(baseVisitor, false);
}
}
}
}
gen.WriteVocabFile();
}
public virtual void Process()
{
CodeGenerator gen = new CodeGenerator(g);
AbstractTarget target = gen.GetTarget();
if (target == null)
{
return;
}
IntervalSet idTypes = new IntervalSet();
idTypes.Add(ANTLRParser.ID);
idTypes.Add(ANTLRParser.RULE_REF);
idTypes.Add(ANTLRParser.TOKEN_REF);
IList <GrammarAST> idNodes = g.ast.GetNodesWithType(idTypes);
foreach (GrammarAST idNode in idNodes)
{
if (target.GrammarSymbolCausesIssueInGeneratedCode(idNode))
{
g.tool.errMgr.GrammarError(ErrorType.USE_OF_BAD_WORD,
g.fileName, idNode.Token,
idNode.Text);
}
}
// all templates are generated in memory to report the most complete
// error information possible, but actually writing output files stops
// after the first error is reported
int errorCount = g.tool.errMgr.GetNumErrors();
if (g.IsLexer())
{
if (target.NeedsHeader())
{
Template lexerHeader = gen.GenerateLexer(true); // Header file if needed.
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(lexerHeader, gen, true);
}
}
Template lexer = gen.GenerateLexer(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(lexer, gen, false);
}
}
else
{
if (target.NeedsHeader())
{
Template parserHeader = gen.GenerateParser(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(parserHeader, gen, true);
}
}
Template parser = gen.GenerateParser(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
WriteRecognizer(parser, gen, false);
}
if (g.tool.gen_listener)
{
if (target.NeedsHeader())
{
Template listenerHeader = gen.GenerateListener(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteListener(listenerHeader, true);
}
}
Template listener = gen.GenerateListener(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteListener(listener, false);
}
if (target.NeedsHeader())
{
Template baseListener = gen.GenerateBaseListener(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseListener(baseListener, true);
}
}
if (target.WantsBaseListener())
{
Template baseListener = gen.GenerateBaseListener(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseListener(baseListener, false);
}
}
}
if (g.tool.gen_visitor)
{
if (target.NeedsHeader())
{
Template visitorHeader = gen.GenerateVisitor(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteVisitor(visitorHeader, true);
}
}
Template visitor = gen.GenerateVisitor(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteVisitor(visitor, false);
}
if (target.NeedsHeader())
{
Template baseVisitor = gen.GenerateBaseVisitor(true);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseVisitor(baseVisitor, true);
}
}
if (target.WantsBaseVisitor())
{
Template baseVisitor = gen.GenerateBaseVisitor(false);
if (g.tool.errMgr.GetNumErrors() == errorCount)
{
gen.WriteBaseVisitor(baseVisitor, false);
}
}
}
}
gen.WriteVocabFile();
}
private static void OptimizeSets(Grammar g, ATN atn)
{
if (g.IsParser())
{
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
IList<DecisionState> decisions = atn.decisionToState;
foreach (DecisionState decision in decisions)
{
if (decision.ruleIndex >= 0)
{
Rule rule = g.GetRule(decision.ruleIndex);
if (char.IsLower(rule.name[0]))
{
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.NumberOfTransitions; i++)
{
Transition epsTransition = decision.Transition(i);
if (!(epsTransition is EpsilonTransition))
{
continue;
}
if (epsTransition.target.NumberOfTransitions != 1)
{
continue;
}
Transition transition = epsTransition.target.Transition(0);
if (!(transition.target is BlockEndState))
{
continue;
}
if (transition is NotSetTransition)
{
// TODO: not yet implemented
continue;
}
if (transition is AtomTransition
|| transition is RangeTransition
|| transition is SetTransition)
{
setTransitions.Add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
{
Interval interval = setTransitions.GetIntervals()[i];
if (interval.Length <= 1)
{
continue;
}
ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++)
{
Transition matchTransition = decision.Transition(j).target.Transition(0);
if (matchTransition is NotSetTransition)
{
throw new NotImplementedException();
}
IntervalSet set = matchTransition.Label;
int minElem = set.MinElement;
int maxElem = set.MaxElement;
for (int k = minElem; k <= maxElem; k++)
{
if (matchSet.Contains(k))
{
char setMin = (char)set.MinElement;
char setMax = (char)set.MaxElement;
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
break;
}
}
matchSet.AddAll(set);
}
Transition newTransition;
if (matchSet.GetIntervals().Count == 1)
{
if (matchSet.Count == 1)
{
newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
}
else
{
Interval matchInterval = matchSet.GetIntervals()[0];
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
}
else
{
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.Transition(interval.a).target.SetTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++)
{
Transition removed = decision.Transition(interval.a + 1);
decision.RemoveTransition(interval.a + 1);
atn.RemoveState(removed.target);
removedStates++;
}
}
}
//System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
