/// <summary> /// Compute set of tokens that can follow /// <code>s</code> /// in the ATN in the /// specified /// <code>ctx</code> /// . /// <p/> /// If /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>stopState</code> /// or the end of the rule containing /// <code>s</code> /// is reached, /// <see cref="TokenConstants.Epsilon"/> /// is added to the result set. If /// <code>ctx</code> /// is not /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>addEOF</code> /// is /// <code>true</code> /// and /// <code>stopState</code> /// or the end of the outermost rule is reached, /// <see cref="TokenConstants.Eof"/> /// is added to the result set. /// </summary> /// <param name="s">the ATN state.</param> /// <param name="stopState"> /// the ATN state to stop at. This can be a /// <see cref="BlockEndState">BlockEndState</see> /// to detect epsilon paths through a closure. /// </param> /// <param name="ctx"> /// The outer context, or /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// if /// the outer context should not be used. /// </param> /// <param name="look">The result lookahead set.</param> /// <param name="lookBusy"> /// A set used for preventing epsilon closures in the ATN /// from causing a stack overflow. Outside code should pass /// <code>new HashSet<ATNConfig></code> /// for this argument. /// </param> /// <param name="calledRuleStack"> /// A set used for preventing left recursion in the /// ATN from causing a stack overflow. Outside code should pass /// <code>new BitSet()</code> /// for this argument. /// </param> /// <param name="seeThruPreds"> /// /// <code>true</code> /// to true semantic predicates as /// implicitly /// <code>true</code> /// and "see through them", otherwise /// <code>false</code> /// to treat semantic predicates as opaque and add /// <see cref="HitPred">HitPred</see> /// to the /// result if one is encountered. /// </param> /// <param name="addEOF"> /// Add /// <see cref="TokenConstants.Eof"/> /// to the result if the end of the /// outermost context is reached. This parameter has no effect if /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// . /// </param> protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet <ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = ATNConfig.Create(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } } if (s is RuleStopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } for (int i = 0; i < ctx.Size; i++) { if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; // System.out.println("popping back to "+retState); for (int j = 0; j < ctx.Size; j++) { bool removed = calledRuleStack.Get(returnState.ruleIndex); try { calledRuleStack.Clear(returnState.ruleIndex); Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { if (removed) { calledRuleStack.Set(returnState.ruleIndex); } } } return; } } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t.GetType() == typeof(RuleTransition)) { if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex)) { continue; } PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber ); try { calledRuleStack.Set(((RuleTransition)t).target.ruleIndex); Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex); } } else { if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { look.Add(HitPred); } } else { if (t.IsEpsilon) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { if (t.GetType() == typeof(WildcardTransition)) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { // System.out.println("adding "+ t); IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType )); } look.AddAll(set); } } } } } } }
/// <summary> /// Serialize state descriptors, edge descriptors, and decision→state map /// into list of ints: /// grammar-type, (ANTLRParser.LEXER, ...) /// max token type, /// num states, /// state-0-type ruleIndex, state-1-type ruleIndex, ... /// </summary> /// <remarks> /// Serialize state descriptors, edge descriptors, and decision→state map /// into list of ints: /// grammar-type, (ANTLRParser.LEXER, ...) /// max token type, /// num states, /// state-0-type ruleIndex, state-1-type ruleIndex, ... state-i-type ruleIndex optional-arg ... /// num rules, /// rule-1-start-state rule-1-args, rule-2-start-state rule-2-args, ... /// (args are token type,actionIndex in lexer else 0,0) /// num modes, /// mode-0-start-state, mode-1-start-state, ... (parser has 0 modes) /// num sets /// set-0-interval-count intervals, set-1-interval-count intervals, ... /// num total edges, /// src, trg, edge-type, edge arg1, optional edge arg2 (present always), ... /// num decisions, /// decision-0-start-state, decision-1-start-state, ... /// Convenient to pack into unsigned shorts to make as Java string. /// </remarks> public virtual List <int> Serialize() { List <int> data = new List <int>(); data.Add(ATNDeserializer.SerializedVersion); SerializeUUID(data, ATNDeserializer.SerializedUuid); // convert grammar type to ATN const to avoid dependence on ANTLRParser data.Add((int)(atn.grammarType)); data.Add(atn.maxTokenType); int nedges = 0; IDictionary <IntervalSet, int> setIndices = new Dictionary <IntervalSet, int>(); IList <IntervalSet> sets = new List <IntervalSet>(); // dump states, count edges and collect sets while doing so List <int> nonGreedyStates = new List <int>(); List <int> sllStates = new List <int>(); List <int> precedenceStates = new List <int>(); data.Add(atn.states.Count); foreach (ATNState s in atn.states) { if (s == null) { // might be optimized away data.Add((int)(StateType.InvalidType)); continue; } StateType stateType = s.StateType; if (s is DecisionState) { DecisionState decisionState = (DecisionState)s; if (decisionState.nonGreedy) { nonGreedyStates.Add(s.stateNumber); } if (decisionState.sll) { sllStates.Add(s.stateNumber); } } if (s is RuleStartState && ((RuleStartState)s).isPrecedenceRule) { precedenceStates.Add(s.stateNumber); } data.Add((int)(stateType)); if (s.ruleIndex == -1) { data.Add(char.MaxValue); } else { data.Add(s.ruleIndex); } if (s.StateType == StateType.LoopEnd) { data.Add(((LoopEndState)s).loopBackState.stateNumber); } else { if (s is BlockStartState) { data.Add(((BlockStartState)s).endState.stateNumber); } } if (s.StateType != StateType.RuleStop) { // the deserializer can trivially derive these edges, so there's no need to serialize them nedges += s.NumberOfTransitions; } for (int i = 0; i < s.NumberOfTransitions; i++) { Transition t = s.Transition(i); TransitionType edgeType = Transition.serializationTypes.Get(t.GetType()); if (edgeType == TransitionType.Set || edgeType == TransitionType.NotSet) { SetTransition st = (SetTransition)t; if (!setIndices.ContainsKey(st.set)) { sets.AddItem(st.set); setIndices.Put(st.set, sets.Count - 1); } } } } // non-greedy states data.Add(nonGreedyStates.Size()); for (int i_1 = 0; i_1 < nonGreedyStates.Size(); i_1++) { data.Add(nonGreedyStates.Get(i_1)); } // SLL decisions data.Add(sllStates.Size()); for (int i_2 = 0; i_2 < sllStates.Size(); i_2++) { data.Add(sllStates.Get(i_2)); } // precedence states data.Add(precedenceStates.Size()); for (int i_3 = 0; i_3 < precedenceStates.Size(); i_3++) { data.Add(precedenceStates.Get(i_3)); } int nrules = atn.ruleToStartState.Length; data.Add(nrules); for (int r = 0; r < nrules; r++) { ATNState ruleStartState = atn.ruleToStartState[r]; data.Add(ruleStartState.stateNumber); bool leftFactored = ruleNames[ruleStartState.ruleIndex].IndexOf(ATNSimulator.RuleVariantDelimiter) >= 0; data.Add(leftFactored ? 1 : 0); if (atn.grammarType == ATNType.Lexer) { if (atn.ruleToTokenType[r] == TokenConstants.Eof) { data.Add(char.MaxValue); } else { data.Add(atn.ruleToTokenType[r]); } } } int nmodes = atn.modeToStartState.Count; data.Add(nmodes); if (nmodes > 0) { foreach (ATNState modeStartState in atn.modeToStartState) { data.Add(modeStartState.stateNumber); } } int nsets = sets.Count; data.Add(nsets); foreach (IntervalSet set in sets) { bool containsEof = set.Contains(TokenConstants.Eof); if (containsEof && set.GetIntervals()[0].b == TokenConstants.Eof) { data.Add(set.GetIntervals().Count - 1); } else { data.Add(set.GetIntervals().Count); } data.Add(containsEof ? 1 : 0); foreach (Interval I in set.GetIntervals()) { if (I.a == TokenConstants.Eof) { if (I.b == TokenConstants.Eof) { continue; } else { data.Add(0); } } else { data.Add(I.a); } data.Add(I.b); } } data.Add(nedges); foreach (ATNState s_1 in atn.states) { if (s_1 == null) { // might be optimized away continue; } if (s_1.StateType == StateType.RuleStop) { continue; } for (int i = 0; i_3 < s_1.NumberOfTransitions; i_3++) { Transition t = s_1.Transition(i_3); if (atn.states[t.target.stateNumber] == null) { throw new InvalidOperationException("Cannot serialize a transition to a removed state."); } int src = s_1.stateNumber; int trg = t.target.stateNumber; TransitionType edgeType = Transition.serializationTypes.Get(t.GetType()); int arg1 = 0; int arg2 = 0; int arg3 = 0; switch (edgeType) { case TransitionType.Rule: { trg = ((RuleTransition)t).followState.stateNumber; arg1 = ((RuleTransition)t).target.stateNumber; arg2 = ((RuleTransition)t).ruleIndex; arg3 = ((RuleTransition)t).precedence; break; } case TransitionType.Precedence: { PrecedencePredicateTransition ppt = (PrecedencePredicateTransition)t; arg1 = ppt.precedence; break; } case TransitionType.Predicate: { PredicateTransition pt = (PredicateTransition)t; arg1 = pt.ruleIndex; arg2 = pt.predIndex; arg3 = pt.isCtxDependent ? 1 : 0; break; } case TransitionType.Range: { arg1 = ((RangeTransition)t).from; arg2 = ((RangeTransition)t).to; if (arg1 == TokenConstants.Eof) { arg1 = 0; arg3 = 1; } break; } case TransitionType.Atom: { arg1 = ((AtomTransition)t).label; if (arg1 == TokenConstants.Eof) { arg1 = 0; arg3 = 1; } break; } case TransitionType.Action: { ActionTransition at = (ActionTransition)t; arg1 = at.ruleIndex; arg2 = at.actionIndex; if (arg2 == -1) { arg2 = unchecked ((int)(0xFFFF)); } arg3 = at.isCtxDependent ? 1 : 0; break; } case TransitionType.Set: { arg1 = setIndices.Get(((SetTransition)t).set); break; } case TransitionType.NotSet: { arg1 = setIndices.Get(((SetTransition)t).set); break; } case TransitionType.Wildcard: { break; } } data.Add(src); data.Add(trg); data.Add((int)(edgeType)); data.Add(arg1); data.Add(arg2); data.Add(arg3); } } int ndecisions = atn.decisionToState.Count; data.Add(ndecisions); foreach (DecisionState decStartState in atn.decisionToState) { data.Add(decStartState.stateNumber); } // // LEXER ACTIONS // if (atn.grammarType == ATNType.Lexer) { data.Add(atn.lexerActions.Length); foreach (ILexerAction action in atn.lexerActions) { data.Add((int)(action.GetActionType())); switch (action.GetActionType()) { case LexerActionType.Channel: { int channel = ((LexerChannelAction)action).GetChannel(); data.Add(channel != -1 ? channel : unchecked ((int)(0xFFFF))); data.Add(0); break; } case LexerActionType.Custom: { int ruleIndex = ((LexerCustomAction)action).GetRuleIndex(); int actionIndex = ((LexerCustomAction)action).GetActionIndex(); data.Add(ruleIndex != -1 ? ruleIndex : unchecked ((int)(0xFFFF))); data.Add(actionIndex != -1 ? actionIndex : unchecked ((int)(0xFFFF))); break; } case LexerActionType.Mode: { int mode = ((LexerModeAction)action).GetMode(); data.Add(mode != -1 ? mode : unchecked ((int)(0xFFFF))); data.Add(0); break; } case LexerActionType.More: { data.Add(0); data.Add(0); break; } case LexerActionType.PopMode: { data.Add(0); data.Add(0); break; } case LexerActionType.PushMode: { mode = ((LexerPushModeAction)action).GetMode(); data.Add(mode != -1 ? mode : unchecked ((int)(0xFFFF))); data.Add(0); break; } case LexerActionType.Skip: { data.Add(0); data.Add(0); break; } case LexerActionType.Type: { int type = ((LexerTypeAction)action).GetType(); data.Add(type != -1 ? type : unchecked ((int)(0xFFFF))); data.Add(0); break; } default: { string message = string.Format(CultureInfo.CurrentCulture, "The specified lexer action type {0} is not valid.", action.GetActionType()); throw new ArgumentException(message); } } } } // don't adjust the first value since that's the version number for (int i_4 = 1; i_4 < data.Size(); i_4++) { if (data.Get(i_4) < char.MinValue || data.Get(i_4) > char.MaxValue) { throw new NotSupportedException("Serialized ATN data element out of range."); } int value = (data.Get(i_4) + 2) & unchecked ((int)(0xFFFF)); data.Set(i_4, value); } return(data); }
/** * Compute set of tokens that can follow {@code s} in the ATN in the * specified {@code ctx}. * * <p>If {@code ctx} is {@code null} and {@code stopState} or the end of the * rule containing {@code s} is reached, {@link Token#EPSILON} is added to * the result set. If {@code ctx} is not {@code null} and {@code addEOF} is * {@code true} and {@code stopState} or the end of the outermost rule is * reached, {@link Token#EOF} is added to the result set.</p> * * @param s the ATN state. * @param stopState the ATN state to stop at. This can be a * {@link BlockEndState} to detect epsilon paths through a closure. * @param ctx The outer context, or {@code null} if the outer context should * not be used. * @param look The result lookahead set. * @param lookBusy A set used for preventing epsilon closures in the ATN * from causing a stack overflow. Outside code should pass * {@code new HashSet<ATNConfig>} for this argument. * @param calledRuleStack A set used for preventing left recursion in the * ATN from causing a stack overflow. Outside code should pass * {@code new BitSet()} for this argument. * @param seeThruPreds {@code true} to true semantic predicates as * implicitly {@code true} and "see through them", otherwise {@code false} * to treat semantic predicates as opaque and add {@link #HIT_PRED} to the * result if one is encountered. * @param addEOF Add {@link Token#EOF} to the result if the end of the * outermost context is reached. This parameter has no effect if {@code ctx} * is {@code null}. */ protected internal virtual void Look_(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { ATNConfig c = new ATNConfig(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (ctx == null) { look.Add(TokenConstants.EPSILON); return; } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } } if (s is RuleStopState) { if (ctx == null) { look.Add(TokenConstants.EPSILON); return; } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } if (ctx != PredictionContext.EMPTY) { bool removed = calledRuleStack.Get(s.ruleIndex); try { calledRuleStack.Clear(s.ruleIndex); for (int i = 0; i < ctx.Size; i++) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; Look_(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } } finally { if (removed) { calledRuleStack.Set(s.ruleIndex); } } return; } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t.GetType() == typeof(RuleTransition)) { RuleTransition ruleTransition = (RuleTransition)t; if (calledRuleStack.Get(ruleTransition.ruleIndex)) { continue; } PredictionContext newContext = SingletonPredictionContext.Create(ctx, ruleTransition.followState.stateNumber); try { calledRuleStack.Set(ruleTransition.target.ruleIndex); Look_(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } finally { calledRuleStack.Clear(ruleTransition.target.ruleIndex); } } else if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else { look.Add(HitPred); } } else if (t.IsEpsilon) { Look_(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else if (t.GetType() == typeof(WildcardTransition)) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } look.AddAll(set); } } } }