private static Transition updateTransition(Transition t, char openDelimiter, char closeDelimiter) { Transition updated = null; if (t is RuleTransition) { return(null); } else if (t is AtomTransition) { AtomTransition atomTransition = (AtomTransition)t; int newLabel; if (atomTransition.label == OpenDelimiterPlaceholder) { newLabel = openDelimiter; } else if (atomTransition.label == CloseDelimiterPlaceholder) { newLabel = closeDelimiter; } else { return(null); } updated = new AtomTransition(t.target, newLabel); } else if (t is NotSetTransition) { NotSetTransition notSetTransition = (NotSetTransition)t; int removeLabel; int addLabel; if (notSetTransition.set.Contains(OpenDelimiterPlaceholder)) { removeLabel = OpenDelimiterPlaceholder; addLabel = openDelimiter; } else if (notSetTransition.set.Contains(CloseDelimiterPlaceholder)) { removeLabel = CloseDelimiterPlaceholder; addLabel = closeDelimiter; } else { return(null); } IntervalSet set = new IntervalSet(notSetTransition.set); set.Remove(removeLabel); set.Add(addLabel); set.SetReadonly(true); updated = new NotSetTransition(t.target, set); } else if (t is SetTransition) { SetTransition setTransition = (SetTransition)t; int removeLabel; int addLabel; if (setTransition.set.Contains(OpenDelimiterPlaceholder)) { removeLabel = OpenDelimiterPlaceholder; addLabel = openDelimiter; } else if (setTransition.set.Contains(CloseDelimiterPlaceholder)) { removeLabel = CloseDelimiterPlaceholder; addLabel = closeDelimiter; } else { return(null); } IntervalSet set = new IntervalSet(setTransition.set); set.Remove(removeLabel); set.Add(addLabel); set.SetReadonly(true); updated = createSetTransition(t.target, set); } else if (t is RangeTransition) { RangeTransition rangeTransition = (RangeTransition)t; int removeLabel; int addLabel; if (rangeTransition.from <= OpenDelimiterPlaceholder && rangeTransition.to >= OpenDelimiterPlaceholder) { removeLabel = OpenDelimiterPlaceholder; addLabel = openDelimiter; } else if (rangeTransition.from <= CloseDelimiterPlaceholder && rangeTransition.to >= CloseDelimiterPlaceholder) { removeLabel = CloseDelimiterPlaceholder; addLabel = closeDelimiter; } else { return(null); } IntervalSet set = IntervalSet.Of(rangeTransition.from, rangeTransition.to); set.Remove(removeLabel); set.Add(addLabel); set.SetReadonly(true); updated = createSetTransition(t.target, set); } return(updated); }
private static void OptimizeSets(Grammar g, ATN atn) { if (g.IsParser()) { // parser codegen doesn't currently support SetTransition return; } int removedStates = 0; IList <DecisionState> decisions = atn.decisionToState; foreach (DecisionState decision in decisions) { if (decision.ruleIndex >= 0) { Rule rule = g.GetRule(decision.ruleIndex); if (char.IsLower(rule.name[0])) { // parser codegen doesn't currently support SetTransition continue; } } IntervalSet setTransitions = new IntervalSet(); for (int i = 0; i < decision.NumberOfTransitions; i++) { Transition epsTransition = decision.Transition(i); if (!(epsTransition is EpsilonTransition)) { continue; } if (epsTransition.target.NumberOfTransitions != 1) { continue; } Transition transition = epsTransition.target.Transition(0); if (!(transition.target is BlockEndState)) { continue; } if (transition is NotSetTransition) { // TODO: not yet implemented continue; } if (transition is AtomTransition || transition is RangeTransition || transition is SetTransition) { setTransitions.Add(i); } } // due to min alt resolution policies, can only collapse sequential alts for (int i = setTransitions.GetIntervals().Count - 1; i >= 0; i--) { Interval interval = setTransitions.GetIntervals()[i]; if (interval.Length <= 1) { continue; } ATNState blockEndState = decision.Transition(interval.a).target.Transition(0).target; IntervalSet matchSet = new IntervalSet(); for (int j = interval.a; j <= interval.b; j++) { Transition matchTransition = decision.Transition(j).target.Transition(0); if (matchTransition is NotSetTransition) { throw new NotImplementedException(); } IntervalSet set = matchTransition.Label; int minElem = set.MinElement; int maxElem = set.MaxElement; for (int k = minElem; k <= maxElem; k++) { if (matchSet.Contains(k)) { char setMin = (char)set.MinElement; char setMax = (char)set.MaxElement; // TODO: Token is missing (i.e. position in source will not be displayed). g.tool.errMgr.GrammarError(ErrorType.CHARACTERS_COLLISION_IN_SET, g.fileName, null, (char)minElem + "-" + (char)maxElem, "[" + setMin + "-" + setMax + "]"); break; } } matchSet.AddAll(set); } Transition newTransition; if (matchSet.GetIntervals().Count == 1) { if (matchSet.Count == 1) { newTransition = new AtomTransition(blockEndState, matchSet.MinElement); } else { Interval matchInterval = matchSet.GetIntervals()[0]; newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b); } } else { newTransition = new SetTransition(blockEndState, matchSet); } decision.Transition(interval.a).target.SetTransition(0, newTransition); for (int j = interval.a + 1; j <= interval.b; j++) { Transition removed = decision.Transition(interval.a + 1); decision.RemoveTransition(interval.a + 1); atn.RemoveState(removed.target); removedStates++; } } } //System.Console.WriteLine("ATN optimizer removed " + removedStates + " states by collapsing sets."); }
public override Handle Set(GrammarAST associatedAST, IList <GrammarAST> alts, bool invert) { ATNState left = NewState(associatedAST); ATNState right = NewState(associatedAST); IntervalSet set = new IntervalSet(); foreach (GrammarAST t in alts) { if (t.Type == ANTLRParser.RANGE) { int a = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(0).Text); int b = CharSupport.GetCharValueFromGrammarCharLiteral(t.GetChild(1).Text); if (CheckRange((GrammarAST)t.GetChild(0), (GrammarAST)t.GetChild(1), a, b)) { CheckSetCollision(associatedAST, set, a, b); set.Add(a, b); } } else if (t.Type == ANTLRParser.LEXER_CHAR_SET) { set.AddAll(GetSetFromCharSetLiteral(t)); } else if (t.Type == ANTLRParser.STRING_LITERAL) { int c = CharSupport.GetCharValueFromGrammarCharLiteral(t.Text); if (c != -1) { CheckSetCollision(associatedAST, set, c); set.Add(c); } else { g.tool.errMgr.GrammarError(ErrorType.INVALID_LITERAL_IN_LEXER_SET, g.fileName, t.Token, t.Text); } } else if (t.Type == ANTLRParser.TOKEN_REF) { g.tool.errMgr.GrammarError(ErrorType.UNSUPPORTED_REFERENCE_IN_LEXER_SET, g.fileName, t.Token, t.Text); } } if (invert) { left.AddTransition(new NotSetTransition(right, set)); } else { Transition transition; if (set.GetIntervals().Count == 1) { Interval interval = set.GetIntervals()[0]; transition = new RangeTransition(right, interval.a, interval.b); } else { transition = new SetTransition(right, set); } left.AddTransition(transition); } associatedAST.atnState = left; return(new Handle(left, right)); }
/** Return a String containing a DOT description that, when displayed, * will show the incoming state machine visually. All nodes reachable * from startState will be included. */ public virtual string GetDOT(ATNState startState, string[] ruleNames, bool isLexer) { if (startState == null) { return(null); } // The output DOT graph for visualization ISet <ATNState> markedStates = new HashSet <ATNState>(); Template dot = stlib.GetInstanceOf("atn"); dot.Add("startState", startState.stateNumber); dot.Add("rankdir", rankdir); Queue <ATNState> work = new Queue <ATNState>(); work.Enqueue(startState); while (work.Count > 0) { ATNState s = work.Peek(); if (markedStates.Contains(s)) { work.Dequeue(); continue; } markedStates.Add(s); // don't go past end of rule node to the follow states if (s is RuleStopState) { continue; } // special case: if decision point, then line up the alt start states // unless it's an end of block // if ( s instanceof BlockStartState ) { // ST rankST = stlib.getInstanceOf("decision-rank"); // DecisionState alt = (DecisionState)s; // for (int i=0; i<alt.getNumberOfTransitions(); i++) { // ATNState target = alt.transition(i).target; // if ( target!=null ) { // rankST.add("states", target.stateNumber); // } // } // dot.add("decisionRanks", rankST); // } // make a DOT edge for each transition Template edgeST; for (int i = 0; i < s.NumberOfTransitions; i++) { Transition edge = s.Transition(i); if (edge is RuleTransition) { RuleTransition rr = ((RuleTransition)edge); // don't jump to other rules, but display edge to follow node edgeST = stlib.GetInstanceOf("edge"); string label = "<" + ruleNames[rr.ruleIndex]; if (((RuleStartState)rr.target).isPrecedenceRule) { label += "[" + rr.precedence + "]"; } label += ">"; edgeST.Add("label", label); edgeST.Add("src", "s" + s.stateNumber); edgeST.Add("target", "s" + rr.followState.stateNumber); edgeST.Add("arrowhead", arrowhead); dot.Add("edges", edgeST); work.Enqueue(rr.followState); continue; } if (edge is ActionTransition) { edgeST = stlib.GetInstanceOf("action-edge"); edgeST.Add("label", GetEdgeLabel(edge.ToString())); } else if (edge is AbstractPredicateTransition) { edgeST = stlib.GetInstanceOf("edge"); edgeST.Add("label", GetEdgeLabel(edge.ToString())); } else if (edge.IsEpsilon) { edgeST = stlib.GetInstanceOf("epsilon-edge"); edgeST.Add("label", GetEdgeLabel(edge.ToString())); bool loopback = false; if (edge.target is PlusBlockStartState) { loopback = s.Equals(((PlusBlockStartState)edge.target).loopBackState); } else if (edge.target is StarLoopEntryState) { loopback = s.Equals(((StarLoopEntryState)edge.target).loopBackState); } edgeST.Add("loopback", loopback); } else if (edge is AtomTransition) { edgeST = stlib.GetInstanceOf("edge"); AtomTransition atom = (AtomTransition)edge; string label = atom.label.ToString(); if (isLexer) { label = "'" + GetEdgeLabel(((char)atom.label).ToString()) + "'"; } else if (grammar != null) { label = grammar.GetTokenDisplayName(atom.label); } edgeST.Add("label", GetEdgeLabel(label)); } else if (edge is SetTransition) { edgeST = stlib.GetInstanceOf("edge"); SetTransition set = (SetTransition)edge; string label = set.Label.ToString(); if (isLexer) { label = set.Label.ToString(true); } else if (grammar != null) { label = set.Label.ToString(grammar.GetVocabulary()); } if (edge is NotSetTransition) { label = "~" + label; } edgeST.Add("label", GetEdgeLabel(label)); } else if (edge is RangeTransition) { edgeST = stlib.GetInstanceOf("edge"); RangeTransition range = (RangeTransition)edge; string label = range.Label.ToString(); if (isLexer) { label = range.ToString(); } else if (grammar != null) { label = range.Label.ToString(grammar.GetVocabulary()); } edgeST.Add("label", GetEdgeLabel(label)); } else { edgeST = stlib.GetInstanceOf("edge"); edgeST.Add("label", GetEdgeLabel(edge.ToString())); } edgeST.Add("src", "s" + s.stateNumber); edgeST.Add("target", "s" + edge.target.stateNumber); edgeST.Add("arrowhead", arrowhead); if (s.NumberOfTransitions > 1) { edgeST.Add("transitionIndex", i); } else { edgeST.Add("transitionIndex", false); } dot.Add("edges", edgeST); work.Enqueue(edge.target); } } // define nodes we visited (they will appear first in DOT output) // this is an example of ST's lazy eval :) // define stop state first; seems to be a bug in DOT where doublecircle // shape only works if we define them first. weird. // ATNState stopState = startState.atn.ruleToStopState.get(startState.rule); // if ( stopState!=null ) { // ST st = stlib.getInstanceOf("stopstate"); // st.add("name", "s"+stopState.stateNumber); // st.add("label", getStateLabel(stopState)); // dot.add("states", st); // } foreach (ATNState s in markedStates) { if (!(s is RuleStopState)) { continue; } Template st = stlib.GetInstanceOf("stopstate"); st.Add("name", "s" + s.stateNumber); st.Add("label", GetStateLabel(s)); dot.Add("states", st); } foreach (ATNState s in markedStates) { if (s is RuleStopState) { continue; } Template st = stlib.GetInstanceOf("state"); st.Add("name", "s" + s.stateNumber); st.Add("label", GetStateLabel(s)); st.Add("transitions", s.Transitions); dot.Add("states", st); } return(dot.Render()); }
protected override ATNState GetReachableTarget(ATNConfig source, Transition trans, int ttype) { if (ttype == AntlrV4.CaretToken.CaretTokenType) { ATNState target = null; AtomTransition atomTransition = trans as AtomTransition; if (atomTransition != null) { if (GetWordlikeTokenTypes().Contains(atomTransition.label)) { target = atomTransition.target; } } else { SetTransition setTransition = trans as SetTransition; if (setTransition != null) { bool not = trans is NotSetTransition; foreach (int t in GetWordlikeTokenTypes().ToArray()) { if (!not && setTransition.set.Contains(t) || not && !setTransition.set.Contains(t)) { target = setTransition.target; break; } } } else { RangeTransition rangeTransition = trans as RangeTransition; if (rangeTransition != null) { // TODO: there must be a better algorithm here int[] wordlikeTokenTypes = GetWordlikeTokenTypes().ToArray(); int lowerBound = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.from); int upperBound = Array.BinarySearch(wordlikeTokenTypes, rangeTransition.to); if (lowerBound >= 0 || upperBound >= 0 || lowerBound != upperBound) { target = rangeTransition.target; } } else { WildcardTransition wildcardTransition = trans as WildcardTransition; if (wildcardTransition != null) { target = trans.target; } } } } if (_caretTransitions == null) { _caretTransitions = new Dictionary <ATNConfig, IList <Transition> >(); } IList <Transition> configTransitions; if (!_caretTransitions.TryGetValue(source, out configTransitions)) { configTransitions = new List <Transition>(); _caretTransitions[source] = configTransitions; } configTransitions.Add(trans); return(target); } return(base.GetReachableTarget(source, trans, ttype)); }