private void HandleAtomicTransition(AtomTransition atomTransition, int currentTokenListIndex, TokenizationResult sentenceTokens)
{
IToken nextToken = sentenceTokens.Tokens.ElementAt(currentTokenListIndex);
if (atomTransition.Label == sentenceTokens.Tokens.ElementAt(currentTokenListIndex).TokenSource)
{
GetSuggestionsFromParser(atomTransition.target, currentTokenListIndex + 1, sentenceTokens);
}
}
/** Add an EOF transition to any rule end ATNState that points to nothing
* (i.e., for all those rules not invoked by another rule). These
* are start symbols then.
*
* Return the number of grammar entry points; i.e., how many rules are
* not invoked by another rule (they can only be invoked from outside).
* These are the start rules.
*/
public virtual int AddEOFTransitionToStartRules()
{
int n = 0;
ATNState eofTarget = NewState(null); // one unique EOF target for all rules
foreach (Rule r in g.rules.Values)
{
ATNState stop = atn.ruleToStopState[r.index];
if (stop.NumberOfTransitions > 0)
{
continue;
}
n++;
Transition t = new AtomTransition(eofTarget, TokenConstants.Eof);
stop.AddTransition(t);
}
return(n);
}
private static Transition updateTransition(Transition t, char openDelimiter, char closeDelimiter)
{
Transition updated = null;
if (t is RuleTransition)
{
return(null);
}
else if (t is AtomTransition)
{
AtomTransition atomTransition = (AtomTransition)t;
int newLabel;
if (atomTransition.label == OpenDelimiterPlaceholder)
{
newLabel = openDelimiter;
}
else if (atomTransition.label == CloseDelimiterPlaceholder)
{
newLabel = closeDelimiter;
}
else
{
return(null);
}
updated = new AtomTransition(t.target, newLabel);
}
else if (t is NotSetTransition)
{
NotSetTransition notSetTransition = (NotSetTransition)t;
int removeLabel;
int addLabel;
if (notSetTransition.set.Contains(OpenDelimiterPlaceholder))
{
removeLabel = OpenDelimiterPlaceholder;
addLabel = openDelimiter;
}
else if (notSetTransition.set.Contains(CloseDelimiterPlaceholder))
{
removeLabel = CloseDelimiterPlaceholder;
addLabel = closeDelimiter;
}
else
{
return(null);
}
IntervalSet set = new IntervalSet(notSetTransition.set);
set.Remove(removeLabel);
set.Add(addLabel);
set.SetReadonly(true);
updated = new NotSetTransition(t.target, set);
}
else if (t is SetTransition)
{
SetTransition setTransition = (SetTransition)t;
int removeLabel;
int addLabel;
if (setTransition.set.Contains(OpenDelimiterPlaceholder))
{
removeLabel = OpenDelimiterPlaceholder;
addLabel = openDelimiter;
}
else if (setTransition.set.Contains(CloseDelimiterPlaceholder))
{
removeLabel = CloseDelimiterPlaceholder;
addLabel = closeDelimiter;
}
else
{
return(null);
}
IntervalSet set = new IntervalSet(setTransition.set);
set.Remove(removeLabel);
set.Add(addLabel);
set.SetReadonly(true);
updated = createSetTransition(t.target, set);
}
else if (t is RangeTransition)
{
RangeTransition rangeTransition = (RangeTransition)t;
int removeLabel;
int addLabel;
if (rangeTransition.from <= OpenDelimiterPlaceholder && rangeTransition.to >= OpenDelimiterPlaceholder)
{
removeLabel = OpenDelimiterPlaceholder;
addLabel = openDelimiter;
}
else if (rangeTransition.from <= CloseDelimiterPlaceholder && rangeTransition.to >= CloseDelimiterPlaceholder)
{
removeLabel = CloseDelimiterPlaceholder;
addLabel = closeDelimiter;
}
else
{
return(null);
}
IntervalSet set = IntervalSet.Of(rangeTransition.from, rangeTransition.to);
set.Remove(removeLabel);
set.Add(addLabel);
set.SetReadonly(true);
updated = createSetTransition(t.target, set);
}
return(updated);
}
private static void OptimizeSets(Grammar g, ATN atn)
{
if (g.IsParser())
{
// parser codegen doesn't currently support SetTransition
return;
}
int removedStates = 0;
IList <DecisionState> decisions = atn.decisionToState;
foreach (DecisionState decision in decisions)
{
if (decision.ruleIndex >= 0)
{
Rule rule = g.GetRule(decision.ruleIndex);
if (char.IsLower(rule.name[0]))
{
// parser codegen doesn't currently support SetTransition
continue;
}
}
IntervalSet setTransitions = new IntervalSet();
for (int i = 0; i < decision.NumberOfTransitions; i++)
{
Transition epsTransition = decision.Transition(i);
if (!(epsTransition is EpsilonTransition))
{
continue;
}
if (epsTransition.target.NumberOfTransitions != 1)
{
continue;
}
Transition transition = epsTransition.target.Transition(0);
if (!(transition.target is BlockEndState))
{
continue;
}
if (transition is NotSetTransition)
{
// TODO: not yet implemented
continue;
}
if (transition is AtomTransition ||
transition is RangeTransition ||
transition is SetTransition)
{
setTransitions.Add(i);
}
}
// due to min alt resolution policies, can only collapse sequential alts
for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--)
{
Interval interval = setTransitions.GetIntervals()[i];
if (interval.Length <= 1)
{
continue;
}
ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target;
IntervalSet matchSet = new IntervalSet();
for (int j = interval.a; j <= interval.b; j++)
{
Transition matchTransition = decision.Transition(j).target.Transition(0);
if (matchTransition is NotSetTransition)
{
throw new NotImplementedException();
}
IntervalSet set = matchTransition.Label;
int minElem = set.MinElement;
int maxElem = set.MaxElement;
for (int k = minElem; k <= maxElem; k++)
{
if (matchSet.Contains(k))
{
char setMin = (char)set.MinElement;
char setMax = (char)set.MaxElement;
// TODO: Token is missing (i.e. position in source will not be displayed).
g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName,
null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]");
break;
}
}
matchSet.AddAll(set);
}
Transition newTransition;
if (matchSet.GetIntervals().Count == 1)
{
if (matchSet.Count == 1)
{
newTransition = new AtomTransition(blockEndState, matchSet.MinElement);
}
else
{
Interval matchInterval = matchSet.GetIntervals()[0];
newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b);
}
}
else
{
newTransition = new SetTransition(blockEndState, matchSet);
}
decision.Transition(interval.a).target.SetTransition(0, newTransition);
for (int j = interval.a + 1; j <= interval.b; j++)
{
Transition removed = decision.Transition(interval.a + 1);
decision.RemoveTransition(interval.a + 1);
atn.RemoveState(removed.target);
removedStates++;
}
}
}
//System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets.");
}
/** Return a String containing a DOT description that, when displayed,
* will show the incoming state machine visually. All nodes reachable
* from startState will be included.
*/
public virtual string GetDOT(ATNState startState, string[] ruleNames, bool isLexer)
{
if (startState == null)
{
return(null);
}
// The output DOT graph for visualization
ISet <ATNState> markedStates = new HashSet <ATNState>();
Template dot = stlib.GetInstanceOf("atn");
dot.Add("startState", startState.stateNumber);
dot.Add("rankdir", rankdir);
Queue <ATNState> work = new Queue <ATNState>();
work.Enqueue(startState);
while (work.Count > 0)
{
ATNState s = work.Peek();
if (markedStates.Contains(s))
{
work.Dequeue();
continue;
}
markedStates.Add(s);
// don't go past end of rule node to the follow states
if (s is RuleStopState)
{
continue;
}
// special case: if decision point, then line up the alt start states
// unless it's an end of block
// if ( s instanceof BlockStartState ) {
// ST rankST = stlib.getInstanceOf("decision-rank");
// DecisionState alt = (DecisionState)s;
// for (int i=0; i<alt.getNumberOfTransitions(); i++) {
// ATNState target = alt.transition(i).target;
// if ( target!=null ) {
// rankST.add("states", target.stateNumber);
// }
// }
// dot.add("decisionRanks", rankST);
// }
// make a DOT edge for each transition
Template edgeST;
for (int i = 0; i < s.NumberOfTransitions; i++)
{
Transition edge = s.Transition(i);
if (edge is RuleTransition)
{
RuleTransition rr = ((RuleTransition)edge);
// don't jump to other rules, but display edge to follow node
edgeST = stlib.GetInstanceOf("edge");
string label = "<" + ruleNames[rr.ruleIndex];
if (((RuleStartState)rr.target).isPrecedenceRule)
{
label += "[" + rr.precedence + "]";
}
label += ">";
edgeST.Add("label", label);
edgeST.Add("src", "s" + s.stateNumber);
edgeST.Add("target", "s" + rr.followState.stateNumber);
edgeST.Add("arrowhead", arrowhead);
dot.Add("edges", edgeST);
work.Enqueue(rr.followState);
continue;
}
if (edge is ActionTransition)
{
edgeST = stlib.GetInstanceOf("action-edge");
edgeST.Add("label", GetEdgeLabel(edge.ToString()));
}
else if (edge is AbstractPredicateTransition)
{
edgeST = stlib.GetInstanceOf("edge");
edgeST.Add("label", GetEdgeLabel(edge.ToString()));
}
else if (edge.IsEpsilon)
{
edgeST = stlib.GetInstanceOf("epsilon-edge");
edgeST.Add("label", GetEdgeLabel(edge.ToString()));
bool loopback = false;
if (edge.target is PlusBlockStartState)
{
loopback = s.Equals(((PlusBlockStartState)edge.target).loopBackState);
}
else if (edge.target is StarLoopEntryState)
{
loopback = s.Equals(((StarLoopEntryState)edge.target).loopBackState);
}
edgeST.Add("loopback", loopback);
}
else if (edge is AtomTransition)
{
edgeST = stlib.GetInstanceOf("edge");
AtomTransition atom = (AtomTransition)edge;
string label = atom.label.ToString();
if (isLexer)
{
label = "'" + GetEdgeLabel(((char)atom.label).ToString()) + "'";
}
else if (grammar != null)
{
label = grammar.GetTokenDisplayName(atom.label);
}
edgeST.Add("label", GetEdgeLabel(label));
}
else if (edge is SetTransition)
{
edgeST = stlib.GetInstanceOf("edge");
SetTransition set = (SetTransition)edge;
string label = set.Label.ToString();
if (isLexer)
{
label = set.Label.ToString(true);
}
else if (grammar != null)
{
label = set.Label.ToString(grammar.GetVocabulary());
}
if (edge is NotSetTransition)
{
label = "~" + label;
}
edgeST.Add("label", GetEdgeLabel(label));
}
else if (edge is RangeTransition)
{
edgeST = stlib.GetInstanceOf("edge");
RangeTransition range = (RangeTransition)edge;
string label = range.Label.ToString();
if (isLexer)
{
label = range.ToString();
}
else if (grammar != null)
{
label = range.Label.ToString(grammar.GetVocabulary());
}
edgeST.Add("label", GetEdgeLabel(label));
}
else
{
edgeST = stlib.GetInstanceOf("edge");
edgeST.Add("label", GetEdgeLabel(edge.ToString()));
}
edgeST.Add("src", "s" + s.stateNumber);
edgeST.Add("target", "s" + edge.target.stateNumber);
edgeST.Add("arrowhead", arrowhead);
if (s.NumberOfTransitions > 1)
{
edgeST.Add("transitionIndex", i);
}
else
{
edgeST.Add("transitionIndex", false);
}
dot.Add("edges", edgeST);
work.Enqueue(edge.target);
}
}
// define nodes we visited (they will appear first in DOT output)
// this is an example of ST's lazy eval :)
// define stop state first; seems to be a bug in DOT where doublecircle
// shape only works if we define them first. weird.
// ATNState stopState = startState.atn.ruleToStopState.get(startState.rule);
// if ( stopState!=null ) {
// ST st = stlib.getInstanceOf("stopstate");
// st.add("name", "s"+stopState.stateNumber);
// st.add("label", getStateLabel(stopState));
// dot.add("states", st);
// }
foreach (ATNState s in markedStates)
{
if (!(s is RuleStopState))
{
continue;
}
Template st = stlib.GetInstanceOf("stopstate");
st.Add("name", "s" + s.stateNumber);
st.Add("label", GetStateLabel(s));
dot.Add("states", st);
}
foreach (ATNState s in markedStates)
{
if (s is RuleStopState)
{
continue;
}
Template st = stlib.GetInstanceOf("state");
st.Add("name", "s" + s.stateNumber);
st.Add("label", GetStateLabel(s));
st.Add("transitions", s.Transitions);
dot.Add("states", st);
}
return(dot.Render());
}
public virtual string AsString()
{
if (start == null)
{
return(null);
}
marked = new HashSet <ATNState>();
work = new List <ATNState>();
work.Add(start);
StringBuilder buf = new StringBuilder();
ATNState s;
while (work.Count > 0)
{
s = work[0];
work.RemoveAt(0);
if (marked.Contains(s))
{
continue;
}
int n = s.NumberOfTransitions;
//System.Console.WriteLine("visit " + s + "; edges=" + n);
marked.Add(s);
for (int i = 0; i < n; i++)
{
Transition t = s.Transition(i);
if (!(s is RuleStopState))
{ // don't add follow states to work
if (t is RuleTransition)
{
work.Add(((RuleTransition)t).followState);
}
else
{
work.Add(t.target);
}
}
buf.Append(GetStateString(s));
if (t is EpsilonTransition)
{
buf.Append("->").Append(GetStateString(t.target)).Append('\n');
}
else if (t is RuleTransition)
{
buf.Append("-").Append(g.GetRule(((RuleTransition)t).ruleIndex).name).Append("->").Append(GetStateString(t.target)).Append('\n');
}
else if (t is ActionTransition)
{
ActionTransition a = (ActionTransition)t;
buf.Append("-").Append(a.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
}
else if (t is SetTransition)
{
SetTransition st = (SetTransition)t;
bool not = st is NotSetTransition;
if (g.IsLexer())
{
buf.Append("-").Append(not ? "~" : "").Append(st.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
}
else
{
buf.Append("-").Append(not ? "~" : "").Append(st.Label.ToString(g.GetVocabulary())).Append("->").Append(GetStateString(t.target)).Append('\n');
}
}
else if (t is AtomTransition)
{
AtomTransition a = (AtomTransition)t;
string label = g.GetTokenDisplayName(a.label);
buf.Append("-").Append(label).Append("->").Append(GetStateString(t.target)).Append('\n');
}
else
{
buf.Append("-").Append(t.ToString()).Append("->").Append(GetStateString(t.target)).Append('\n');
}
}
}
return(buf.ToString());
}
protected override ATNState GetReachableTarget(ATNConfig source, Transition trans, int ttype)
{
if (ttype == AntlrV4.CaretToken.CaretTokenType)
{
ATNState target = null;
AtomTransition atomTransition = trans as AtomTransition;
if (atomTransition != null)
{
if (GetWordlikeTokenTypes().Contains(atomTransition.label))
{
target = atomTransition.target;
}
}
else
{
SetTransition setTransition = trans as SetTransition;
if (setTransition != null)
{
bool not = trans is NotSetTransition;
foreach (int t in GetWordlikeTokenTypes().ToArray())
{
if (!not && setTransition.set.Contains(t) || not && !setTransition.set.Contains(t))
{
target = setTransition.target;
break;
}
}
}
else
{
RangeTransition rangeTransition = trans as RangeTransition;
if (rangeTransition != null)
{
// TODO: there must be a better algorithm here
int[] wordlikeTokenTypes = GetWordlikeTokenTypes().ToArray();
int lowerBound = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.from);
int upperBound = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.to);
if (lowerBound >= 0 || upperBound >= 0 || lowerBound != upperBound)
{
target = rangeTransition.target;
}
}
else
{
WildcardTransition wildcardTransition = trans as WildcardTransition;
if (wildcardTransition != null)
{
target = trans.target;
}
}
}
}
if (_caretTransitions == null)
{
_caretTransitions = new Dictionary <ATNConfig, IList <Transition> >();
}
IList <Transition> configTransitions;
if (!_caretTransitions.TryGetValue(source, out configTransitions))
{
configTransitions = new List <Transition>();
_caretTransitions[source] = configTransitions;
}
configTransitions.Add(trans);
return(target);
}
return(base.GetReachableTarget(source, trans, ttype));
}
