public void TestMixedRangesAndElements() { IntervalSet s = new IntervalSet(); s.Add(1); s.Add('a', 'z'); s.Add('0', '9'); String expecting = "{1, 48..57, 97..122}"; Assert.AreEqual(s.ToString(), expecting); }
public void TestIsolatedElements() { IntervalSet s = new IntervalSet(); s.Add(1); s.Add('z'); s.Add('\uFFF0'); String expecting = "{1, 122, 65520}"; Assert.AreEqual(s.ToString(), expecting); }
public static Antlr4.Runtime.Misc.IntervalSet Of(int a) { Antlr4.Runtime.Misc.IntervalSet s = new Antlr4.Runtime.Misc.IntervalSet(); s.Add(a); return(s); }
public static IntervalSet ToSet(BitSet bits) { IntervalSet s = new IntervalSet(); int i = bits.NextSetBit(0); while (i >= 0) { s.Add(i); i = bits.NextSetBit(i + 1); } return(s); }
protected internal virtual IList<IntervalSet> ReadSets(ATN atn) { // // SETS // IList<IntervalSet> sets = new List<IntervalSet>(); int nsets = ReadInt(); for (int i_8 = 0; i_8 < nsets; i_8++) { IntervalSet set = new IntervalSet(); sets.Add(set); int nintervals = ReadInt(); bool containsEof = ReadInt() != 0; if (containsEof) { set.Add(-1); } for (int j = 0; j < nintervals; j++) { set.Add(ReadInt(), ReadInt()); } } return sets; }
private static int OptimizeSets(ATN atn, bool preserveOrder) { if (preserveOrder) { // this optimization currently doesn't preserve edge order. return 0; } int removedPaths = 0; IList<DecisionState> decisions = atn.decisionToState; foreach (DecisionState decision in decisions) { IntervalSet setTransitions = new IntervalSet(); for (int i = 0; i < decision.NumberOfOptimizedTransitions; i++) { Transition epsTransition = decision.GetOptimizedTransition(i); if (!(epsTransition is EpsilonTransition)) { continue; } if (epsTransition.target.NumberOfOptimizedTransitions != 1) { continue; } Transition transition = epsTransition.target.GetOptimizedTransition(0); if (!(transition.target is BlockEndState)) { continue; } if (transition is NotSetTransition) { // TODO: not yet implemented continue; } if (transition is AtomTransition || transition is RangeTransition || transition is SetTransition) { setTransitions.Add(i); } } if (setTransitions.Count <= 1) { continue; } IList<Transition> optimizedTransitions = new List<Transition>(); for (int i_1 = 0; i_1 < decision.NumberOfOptimizedTransitions; i_1++) { if (!setTransitions.Contains(i_1)) { optimizedTransitions.Add(decision.GetOptimizedTransition(i_1)); } } ATNState blockEndState = decision.GetOptimizedTransition(setTransitions.MinElement).target.GetOptimizedTransition(0).target; IntervalSet matchSet = new IntervalSet(); for (int i_2 = 0; i_2 < setTransitions.GetIntervals().Count; i_2++) { Interval interval = setTransitions.GetIntervals()[i_2]; for (int j = interval.a; j <= interval.b; j++) { Transition matchTransition = decision.GetOptimizedTransition(j).target.GetOptimizedTransition(0); if (matchTransition is NotSetTransition) { throw new NotSupportedException("Not yet implemented."); } else { matchSet.AddAll(matchTransition.Label); } } } Transition newTransition; if (matchSet.GetIntervals().Count == 1) { if (matchSet.Count == 1) { newTransition = new AtomTransition(blockEndState, matchSet.MinElement); } else { Interval matchInterval = matchSet.GetIntervals()[0]; newTransition = new RangeTransition(blockEndState, matchInterval.a, matchInterval.b); } } else { newTransition = new SetTransition(blockEndState, matchSet); } ATNState setOptimizedState = new BasicState(); setOptimizedState.SetRuleIndex(decision.ruleIndex); atn.AddState(setOptimizedState); setOptimizedState.AddTransition(newTransition); optimizedTransitions.Add(new EpsilonTransition(setOptimizedState)); removedPaths += decision.NumberOfOptimizedTransitions - optimizedTransitions.Count; if (decision.IsOptimized) { while (decision.NumberOfOptimizedTransitions > 0) { decision.RemoveOptimizedTransition(decision.NumberOfOptimizedTransitions - 1); } } foreach (Transition transition_1 in optimizedTransitions) { decision.AddOptimizedTransition(transition_1); } } return removedPaths; }
public void TestMergeWhereAdditionMergesThreeExistingIntervals() { IntervalSet s = new IntervalSet(); s.Add(0); s.Add(3); s.Add(5); s.Add(0, 7); String expecting = "{0..7}"; String result = s.ToString(); Assert.AreEqual(expecting, result); }
public virtual IntervalSet GetExpectedTokens(int stateNumber, RuleContext context) { if (stateNumber < 0 || stateNumber >= states.Count) { throw new ArgumentException("Invalid state number."); } RuleContext ctx = context; ATNState s = states[stateNumber]; IntervalSet following = NextTokens(s); if (!following.Contains(TokenConstants.Epsilon)) { return following; } IntervalSet expected = new IntervalSet(); expected.AddAll(following); expected.Remove(TokenConstants.Epsilon); while (ctx != null && ctx.invokingState >= 0 && following.Contains(TokenConstants.Epsilon)) { ATNState invokingState = states[ctx.invokingState]; RuleTransition rt = (RuleTransition)invokingState.Transition(0); following = NextTokens(rt.followState); expected.AddAll(following); expected.Remove(TokenConstants.Epsilon); ctx = ctx.parent; } if (following.Contains(TokenConstants.Epsilon)) { expected.Add(TokenConstants.Eof); } return expected; }
protected int getAltThatFinishedDecisionEntryRule(ATNConfigSet configSet) { IntervalSet alts = new IntervalSet(); foreach (ATNConfig c in configSet.configs) { if (c.OuterContextDepth > 0 || (c.state is RuleStopState && c.context.HasEmptyPath)) { alts.Add(c.alt); } } if (alts.Count == 0) return ATN.INVALID_ALT_NUMBER; return alts.MinElement; }
public virtual ATN Deserialize(char[] data) { data = (char[])data.Clone(); // don't adjust the first value since that's the version number for (int i = 1; i < data.Length; i++) { data[i] = (char)(data[i] - 2); } int p = 0; int version = ToInt(data[p++]); if (version != SerializedVersion) { string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with version {0} (expected {1}).", version, SerializedVersion); throw new NotSupportedException(reason); } Guid uuid = ToUUID(data, p); p += 8; if (!SupportedUuids.Contains(uuid)) { string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID {0} (expected {1} or a legacy UUID).", uuid, SerializedUuid); throw new NotSupportedException(reason); } bool supportsLexerActions = IsFeatureSupported(AddedLexerActions, uuid); ATNType grammarType = (ATNType)ToInt(data[p++]); int maxTokenType = ToInt(data[p++]); ATN atn = new ATN(grammarType, maxTokenType); // // STATES // IList<Tuple<LoopEndState, int>> loopBackStateNumbers = new List<Tuple<LoopEndState, int>>(); IList<Tuple<BlockStartState, int>> endStateNumbers = new List<Tuple<BlockStartState, int>>(); int nstates = ToInt(data[p++]); for (int i_1 = 0; i_1 < nstates; i_1++) { StateType stype = (StateType)ToInt(data[p++]); // ignore bad type of states if (stype == StateType.InvalidType) { atn.AddState(null); continue; } int ruleIndex = ToInt(data[p++]); if (ruleIndex == char.MaxValue) { ruleIndex = -1; } ATNState s = StateFactory(stype, ruleIndex); if (stype == StateType.LoopEnd) { // special case int loopBackStateNumber = ToInt(data[p++]); loopBackStateNumbers.Add(Tuple.Create((LoopEndState)s, loopBackStateNumber)); } else { if (s is BlockStartState) { int endStateNumber = ToInt(data[p++]); endStateNumbers.Add(Tuple.Create((BlockStartState)s, endStateNumber)); } } atn.AddState(s); } // delay the assignment of loop back and end states until we know all the state instances have been initialized foreach (Tuple<LoopEndState, int> pair in loopBackStateNumbers) { pair.Item1.loopBackState = atn.states[pair.Item2]; } foreach (Tuple<BlockStartState, int> pair_1 in endStateNumbers) { pair_1.Item1.endState = (BlockEndState)atn.states[pair_1.Item2]; } int numNonGreedyStates = ToInt(data[p++]); for (int i_2 = 0; i_2 < numNonGreedyStates; i_2++) { int stateNumber = ToInt(data[p++]); ((DecisionState)atn.states[stateNumber]).nonGreedy = true; } int numSllDecisions = ToInt(data[p++]); for (int i_3 = 0; i_3 < numSllDecisions; i_3++) { int stateNumber = ToInt(data[p++]); ((DecisionState)atn.states[stateNumber]).sll = true; } int numPrecedenceStates = ToInt(data[p++]); for (int i_4 = 0; i_4 < numPrecedenceStates; i_4++) { int stateNumber = ToInt(data[p++]); ((RuleStartState)atn.states[stateNumber]).isPrecedenceRule = true; } // // RULES // int nrules = ToInt(data[p++]); if (atn.grammarType == ATNType.Lexer) { atn.ruleToTokenType = new int[nrules]; } atn.ruleToStartState = new RuleStartState[nrules]; for (int i_5 = 0; i_5 < nrules; i_5++) { int s = ToInt(data[p++]); RuleStartState startState = (RuleStartState)atn.states[s]; startState.leftFactored = ToInt(data[p++]) != 0; atn.ruleToStartState[i_5] = startState; if (atn.grammarType == ATNType.Lexer) { int tokenType = ToInt(data[p++]); if (tokenType == unchecked((int)(0xFFFF))) { tokenType = TokenConstants.Eof; } atn.ruleToTokenType[i_5] = tokenType; if (!IsFeatureSupported(AddedLexerActions, uuid)) { // this piece of unused metadata was serialized prior to the // addition of LexerAction int actionIndexIgnored = ToInt(data[p++]); if (actionIndexIgnored == unchecked((int)(0xFFFF))) { actionIndexIgnored = -1; } } } } atn.ruleToStopState = new RuleStopState[nrules]; foreach (ATNState state in atn.states) { if (!(state is RuleStopState)) { continue; } RuleStopState stopState = (RuleStopState)state; atn.ruleToStopState[state.ruleIndex] = stopState; atn.ruleToStartState[state.ruleIndex].stopState = stopState; } // // MODES // int nmodes = ToInt(data[p++]); for (int i_6 = 0; i_6 < nmodes; i_6++) { int s = ToInt(data[p++]); atn.modeToStartState.Add((TokensStartState)atn.states[s]); } atn.modeToDFA = new DFA[nmodes]; for (int i_7 = 0; i_7 < nmodes; i_7++) { atn.modeToDFA[i_7] = new DFA(atn.modeToStartState[i_7]); } // // SETS // IList<IntervalSet> sets = new List<IntervalSet>(); int nsets = ToInt(data[p++]); for (int i_8 = 0; i_8 < nsets; i_8++) { int nintervals = ToInt(data[p]); p++; IntervalSet set = new IntervalSet(); sets.Add(set); bool containsEof = ToInt(data[p++]) != 0; if (containsEof) { set.Add(-1); } for (int j = 0; j < nintervals; j++) { set.Add(ToInt(data[p]), ToInt(data[p + 1])); p += 2; } } // // EDGES // int nedges = ToInt(data[p++]); for (int i_9 = 0; i_9 < nedges; i_9++) { int src = ToInt(data[p]); int trg = ToInt(data[p + 1]); TransitionType ttype = (TransitionType)ToInt(data[p + 2]); int arg1 = ToInt(data[p + 3]); int arg2 = ToInt(data[p + 4]); int arg3 = ToInt(data[p + 5]); Transition trans = EdgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets); // System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+ // src+"->"+trg+ // " "+Transition.serializationNames[ttype]+ // " "+arg1+","+arg2+","+arg3); ATNState srcState = atn.states[src]; srcState.AddTransition(trans); p += 6; } // edges for rule stop states can be derived, so they aren't serialized // Map rule stop state -> return state -> outermost precedence return HashSet<Tuple<int, int, int>> returnTransitionsSet = new HashSet<Tuple<int, int, int>>(); List<Tuple<int, int, int>> returnTransitions = new List<Tuple<int, int, int>>(); foreach (ATNState state_1 in atn.states) { bool returningToLeftFactored = state_1.ruleIndex >= 0 && atn.ruleToStartState[state_1.ruleIndex].leftFactored; for (int i_10 = 0; i_10 < state_1.NumberOfTransitions; i_10++) { Transition t = state_1.Transition(i_10); if (!(t is RuleTransition)) { continue; } RuleTransition ruleTransition = (RuleTransition)t; bool returningFromLeftFactored = atn.ruleToStartState[ruleTransition.target.ruleIndex].leftFactored; if (!returningFromLeftFactored && returningToLeftFactored) { continue; } int outermostPrecedenceReturn = -1; if (atn.ruleToStartState[ruleTransition.target.ruleIndex].isPrecedenceRule) { if (ruleTransition.precedence == 0) { outermostPrecedenceReturn = ruleTransition.target.ruleIndex; } } var returnTransition = Tuple.Create(ruleTransition.target.ruleIndex, ruleTransition.followState.stateNumber, outermostPrecedenceReturn); if (returnTransitionsSet.Add(returnTransition)) returnTransitions.Add(returnTransition); } } // Add all elements from returnTransitions to the ATN foreach (Tuple<int, int, int> returnTransition in returnTransitions) { EpsilonTransition transition = new EpsilonTransition(atn.states[returnTransition.Item2], returnTransition.Item3); atn.ruleToStopState[returnTransition.Item1].AddTransition(transition); } foreach (ATNState state_2 in atn.states) { if (state_2 is BlockStartState) { // we need to know the end state to set its start state if (((BlockStartState)state_2).endState == null) { throw new InvalidOperationException(); } // block end states can only be associated to a single block start state if (((BlockStartState)state_2).endState.startState != null) { throw new InvalidOperationException(); } ((BlockStartState)state_2).endState.startState = (BlockStartState)state_2; } if (state_2 is PlusLoopbackState) { PlusLoopbackState loopbackState = (PlusLoopbackState)state_2; for (int i_10 = 0; i_10 < loopbackState.NumberOfTransitions; i_10++) { ATNState target = loopbackState.Transition(i_10).target; if (target is PlusBlockStartState) { ((PlusBlockStartState)target).loopBackState = loopbackState; } } } else { if (state_2 is StarLoopbackState) { StarLoopbackState loopbackState = (StarLoopbackState)state_2; for (int i_10 = 0; i_10 < loopbackState.NumberOfTransitions; i_10++) { ATNState target = loopbackState.Transition(i_10).target; if (target is StarLoopEntryState) { ((StarLoopEntryState)target).loopBackState = loopbackState; } } } } } // // DECISIONS // int ndecisions = ToInt(data[p++]); for (int i_11 = 1; i_11 <= ndecisions; i_11++) { int s = ToInt(data[p++]); DecisionState decState = (DecisionState)atn.states[s]; atn.decisionToState.Add(decState); decState.decision = i_11 - 1; } // // LEXER ACTIONS // if (atn.grammarType == ATNType.Lexer) { if (supportsLexerActions) { atn.lexerActions = new ILexerAction[ToInt(data[p++])]; for (int i_10 = 0; i_10 < atn.lexerActions.Length; i_10++) { LexerActionType actionType = (LexerActionType)ToInt(data[p++]); int data1 = ToInt(data[p++]); if (data1 == unchecked((int)(0xFFFF))) { data1 = -1; } int data2 = ToInt(data[p++]); if (data2 == unchecked((int)(0xFFFF))) { data2 = -1; } ILexerAction lexerAction = LexerActionFactory(actionType, data1, data2); atn.lexerActions[i_10] = lexerAction; } } else { // for compatibility with older serialized ATNs, convert the old // serialized action index for action transitions to the new // form, which is the index of a LexerCustomAction List<ILexerAction> legacyLexerActions = new List<ILexerAction>(); foreach (ATNState state_3 in atn.states) { for (int i_10 = 0; i_10 < state_3.NumberOfTransitions; i_10++) { Transition transition = state_3.Transition(i_10); if (!(transition is ActionTransition)) { continue; } int ruleIndex = ((ActionTransition)transition).ruleIndex; int actionIndex = ((ActionTransition)transition).actionIndex; LexerCustomAction lexerAction = new LexerCustomAction(ruleIndex, actionIndex); state_3.SetTransition(i_10, new ActionTransition(transition.target, ruleIndex, legacyLexerActions.Count, false)); legacyLexerActions.Add(lexerAction); } } atn.lexerActions = legacyLexerActions.ToArray(); } } MarkPrecedenceDecisions(atn); atn.decisionToDFA = new DFA[ndecisions]; for (int i_12 = 0; i_12 < ndecisions; i_12++) { atn.decisionToDFA[i_12] = new DFA(atn.decisionToState[i_12], i_12); } if (deserializationOptions.VerifyAtn) { VerifyATN(atn); } if (deserializationOptions.GenerateRuleBypassTransitions && atn.grammarType == ATNType.Parser) { atn.ruleToTokenType = new int[atn.ruleToStartState.Length]; for (int i_10 = 0; i_10 < atn.ruleToStartState.Length; i_10++) { atn.ruleToTokenType[i_10] = atn.maxTokenType + i_10 + 1; } for (int i_13 = 0; i_13 < atn.ruleToStartState.Length; i_13++) { BasicBlockStartState bypassStart = new BasicBlockStartState(); bypassStart.ruleIndex = i_13; atn.AddState(bypassStart); BlockEndState bypassStop = new BlockEndState(); bypassStop.ruleIndex = i_13; atn.AddState(bypassStop); bypassStart.endState = bypassStop; atn.DefineDecisionState(bypassStart); bypassStop.startState = bypassStart; ATNState endState; Transition excludeTransition = null; if (atn.ruleToStartState[i_13].isPrecedenceRule) { // wrap from the beginning of the rule to the StarLoopEntryState endState = null; foreach (ATNState state_3 in atn.states) { if (state_3.ruleIndex != i_13) { continue; } if (!(state_3 is StarLoopEntryState)) { continue; } ATNState maybeLoopEndState = state_3.Transition(state_3.NumberOfTransitions - 1).target; if (!(maybeLoopEndState is LoopEndState)) { continue; } if (maybeLoopEndState.epsilonOnlyTransitions && maybeLoopEndState.Transition(0).target is RuleStopState) { endState = state_3; break; } } if (endState == null) { throw new NotSupportedException("Couldn't identify final state of the precedence rule prefix section."); } excludeTransition = ((StarLoopEntryState)endState).loopBackState.Transition(0); } else { endState = atn.ruleToStopState[i_13]; } // all non-excluded transitions that currently target end state need to target blockEnd instead foreach (ATNState state_4 in atn.states) { foreach (Transition transition in state_4.transitions) { if (transition == excludeTransition) { continue; } if (transition.target == endState) { transition.target = bypassStop; } } } // all transitions leaving the rule start state need to leave blockStart instead while (atn.ruleToStartState[i_13].NumberOfTransitions > 0) { Transition transition = atn.ruleToStartState[i_13].Transition(atn.ruleToStartState[i_13].NumberOfTransitions - 1); atn.ruleToStartState[i_13].RemoveTransition(atn.ruleToStartState[i_13].NumberOfTransitions - 1); bypassStart.AddTransition(transition); } // link the new states atn.ruleToStartState[i_13].AddTransition(new EpsilonTransition(bypassStart)); bypassStop.AddTransition(new EpsilonTransition(endState)); ATNState matchState = new BasicState(); atn.AddState(matchState); matchState.AddTransition(new AtomTransition(bypassStop, atn.ruleToTokenType[i_13])); bypassStart.AddTransition(new EpsilonTransition(matchState)); } if (deserializationOptions.VerifyAtn) { // reverify after modification VerifyATN(atn); } } if (deserializationOptions.Optimize) { while (true) { int optimizationCount = 0; optimizationCount += InlineSetRules(atn); optimizationCount += CombineChainedEpsilons(atn); bool preserveOrder = atn.grammarType == ATNType.Lexer; optimizationCount += OptimizeSets(atn, preserveOrder); if (optimizationCount == 0) { break; } } if (deserializationOptions.VerifyAtn) { // reverify after modification VerifyATN(atn); } } IdentifyTailCalls(atn); return atn; }
/// <summary>Create a set with all ints within range [a..b] (inclusive)</summary> public static Antlr4.Runtime.Misc.IntervalSet Of(int a, int b) { Antlr4.Runtime.Misc.IntervalSet s = new Antlr4.Runtime.Misc.IntervalSet(); s.Add(a, b); return s; }
/// <summary> /// Compute set of tokens that can follow /// <code>s</code> /// in the ATN in the /// specified /// <code>ctx</code> /// . /// <p/> /// If /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>stopState</code> /// or the end of the rule containing /// <code>s</code> /// is reached, /// <see cref="TokenConstants.Epsilon"/> /// is added to the result set. If /// <code>ctx</code> /// is not /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>addEOF</code> /// is /// <code>true</code> /// and /// <code>stopState</code> /// or the end of the outermost rule is reached, /// <see cref="TokenConstants.Eof"/> /// is added to the result set. /// </summary> /// <param name="s">the ATN state.</param> /// <param name="stopState"> /// the ATN state to stop at. This can be a /// <see cref="BlockEndState">BlockEndState</see> /// to detect epsilon paths through a closure. /// </param> /// <param name="ctx"> /// The outer context, or /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// if /// the outer context should not be used. /// </param> /// <param name="look">The result lookahead set.</param> /// <param name="lookBusy"> /// A set used for preventing epsilon closures in the ATN /// from causing a stack overflow. Outside code should pass /// <code>new HashSet<ATNConfig></code> /// for this argument. /// </param> /// <param name="calledRuleStack"> /// A set used for preventing left recursion in the /// ATN from causing a stack overflow. Outside code should pass /// <code>new BitSet()</code> /// for this argument. /// </param> /// <param name="seeThruPreds"> /// /// <code>true</code> /// to true semantic predicates as /// implicitly /// <code>true</code> /// and "see through them", otherwise /// <code>false</code> /// to treat semantic predicates as opaque and add /// <see cref="HitPred">HitPred</see> /// to the /// result if one is encountered. /// </param> /// <param name="addEOF"> /// Add /// <see cref="TokenConstants.Eof"/> /// to the result if the end of the /// outermost context is reached. This parameter has no effect if /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// . /// </param> protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = ATNConfig.Create(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } } if (s is RuleStopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } for (int i = 0; i < ctx.Size; i++) { if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; // System.out.println("popping back to "+retState); for (int j = 0; j < ctx.Size; j++) { bool removed = calledRuleStack.Get(returnState.ruleIndex); try { calledRuleStack.Clear(returnState.ruleIndex); Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { if (removed) { calledRuleStack.Set(returnState.ruleIndex); } } } return; } } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t.GetType() == typeof(RuleTransition)) { if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex)) { continue; } PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber ); try { calledRuleStack.Set(((RuleTransition)t).target.ruleIndex); Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex); } } else { if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { look.Add(HitPred); } } else { if (t.IsEpsilon) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { if (t.GetType() == typeof(WildcardTransition)) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { // System.out.println("adding "+ t); IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType )); } look.AddAll(set); } } } } } } }
public static ATN Deserialize(char[] data, bool optimize) { data = (char[])data.Clone(); // don't adjust the first value since that's the version number for (int i = 1; i < data.Length; i++) { data[i] = (char)(data[i] - 2); } int p = 0; int version = ToInt(data[p++]); if (version != SerializedVersion) { string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with version {0} (expected {1})." , version, SerializedVersion); throw new NotSupportedException(reason); } Guid uuid = ToUUID(data, p); p += 8; if (!uuid.Equals(SerializedUuid)) { string reason = string.Format(CultureInfo.CurrentCulture, "Could not deserialize ATN with UUID {0} (expected {1})." , uuid, SerializedUuid); throw new NotSupportedException(reason); } ATNType grammarType = (ATNType)ToInt(data[p++]); int maxTokenType = ToInt(data[p++]); ATN atn = new ATN(grammarType, maxTokenType); // // STATES // IList<Tuple<LoopEndState, int>> loopBackStateNumbers = new List<Tuple<LoopEndState , int>>(); IList<Tuple<BlockStartState, int>> endStateNumbers = new List<Tuple<BlockStartState , int>>(); int nstates = ToInt(data[p++]); for (int i_1 = 0; i_1 < nstates; i_1++) { StateType stype = (StateType)ToInt(data[p++]); // ignore bad type of states if (stype == StateType.InvalidType) { atn.AddState(null); continue; } int ruleIndex = ToInt(data[p++]); if (ruleIndex == char.MaxValue) { ruleIndex = -1; } ATNState s = StateFactory(stype, ruleIndex); if (stype == StateType.LoopEnd) { // special case int loopBackStateNumber = ToInt(data[p++]); loopBackStateNumbers.Add(Tuple.Create((LoopEndState)s, loopBackStateNumber)); } else { if (s is BlockStartState) { int endStateNumber = ToInt(data[p++]); endStateNumbers.Add(Tuple.Create((BlockStartState)s, endStateNumber)); } } atn.AddState(s); } // delay the assignment of loop back and end states until we know all the state instances have been initialized foreach (Tuple<LoopEndState, int> pair in loopBackStateNumbers) { pair.Item1.loopBackState = atn.states[pair.Item2]; } foreach (Tuple<BlockStartState, int> pair_1 in endStateNumbers) { pair_1.Item1.endState = (BlockEndState)atn.states[pair_1.Item2]; } int numNonGreedyStates = ToInt(data[p++]); for (int i_2 = 0; i_2 < numNonGreedyStates; i_2++) { int stateNumber = ToInt(data[p++]); ((DecisionState)atn.states[stateNumber]).nonGreedy = true; } int numSllDecisions = ToInt(data[p++]); for (int i_3 = 0; i_3 < numSllDecisions; i_3++) { int stateNumber = ToInt(data[p++]); ((DecisionState)atn.states[stateNumber]).sll = true; } int numPrecedenceStates = ToInt(data[p++]); for (int i_4 = 0; i_4 < numPrecedenceStates; i_4++) { int stateNumber = ToInt(data[p++]); ((RuleStartState)atn.states[stateNumber]).isPrecedenceRule = true; } // // RULES // int nrules = ToInt(data[p++]); if (atn.grammarType == ATNType.Lexer) { atn.ruleToTokenType = new int[nrules]; atn.ruleToActionIndex = new int[nrules]; } atn.ruleToStartState = new RuleStartState[nrules]; for (int i_5 = 0; i_5 < nrules; i_5++) { int s = ToInt(data[p++]); RuleStartState startState = (RuleStartState)atn.states[s]; startState.leftFactored = ToInt(data[p++]) != 0; atn.ruleToStartState[i_5] = startState; if (atn.grammarType == ATNType.Lexer) { int tokenType = ToInt(data[p++]); if (tokenType == unchecked((int)(0xFFFF))) { tokenType = TokenConstants.Eof; } atn.ruleToTokenType[i_5] = tokenType; int actionIndex = ToInt(data[p++]); if (actionIndex == unchecked((int)(0xFFFF))) { actionIndex = -1; } atn.ruleToActionIndex[i_5] = actionIndex; } } atn.ruleToStopState = new RuleStopState[nrules]; foreach (ATNState state in atn.states) { if (!(state is RuleStopState)) { continue; } RuleStopState stopState = (RuleStopState)state; atn.ruleToStopState[state.ruleIndex] = stopState; atn.ruleToStartState[state.ruleIndex].stopState = stopState; } // // MODES // int nmodes = ToInt(data[p++]); for (int i_6 = 0; i_6 < nmodes; i_6++) { int s = ToInt(data[p++]); atn.modeToStartState.Add((TokensStartState)atn.states[s]); } atn.modeToDFA = new DFA[nmodes]; for (int i_7 = 0; i_7 < nmodes; i_7++) { atn.modeToDFA[i_7] = new DFA(atn.modeToStartState[i_7]); } // // SETS // IList<IntervalSet> sets = new List<IntervalSet>(); int nsets = ToInt(data[p++]); for (int i_8 = 0; i_8 < nsets; i_8++) { int nintervals = ToInt(data[p]); p++; IntervalSet set = new IntervalSet(); sets.Add(set); bool containsEof = ToInt(data[p++]) != 0; if (containsEof) { set.Add(-1); } for (int j = 0; j < nintervals; j++) { set.Add(ToInt(data[p]), ToInt(data[p + 1])); p += 2; } } // // EDGES // int nedges = ToInt(data[p++]); for (int i_9 = 0; i_9 < nedges; i_9++) { int src = ToInt(data[p]); int trg = ToInt(data[p + 1]); TransitionType ttype = (TransitionType)ToInt(data[p + 2]); int arg1 = ToInt(data[p + 3]); int arg2 = ToInt(data[p + 4]); int arg3 = ToInt(data[p + 5]); Transition trans = EdgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets); // System.out.println("EDGE "+trans.getClass().getSimpleName()+" "+ // src+"->"+trg+ // " "+Transition.serializationNames[ttype]+ // " "+arg1+","+arg2+","+arg3); ATNState srcState = atn.states[src]; srcState.AddTransition(trans); p += 6; } // edges for rule stop states can be derived, so they aren't serialized foreach (ATNState state_1 in atn.states) { bool returningToLeftFactored = state_1.ruleIndex >= 0 && atn.ruleToStartState[state_1 .ruleIndex].leftFactored; for (int i_10 = 0; i_10 < state_1.NumberOfTransitions; i_10++) { Transition t = state_1.Transition(i_10); if (!(t is RuleTransition)) { continue; } RuleTransition ruleTransition = (RuleTransition)t; bool returningFromLeftFactored = atn.ruleToStartState[ruleTransition.target.ruleIndex ].leftFactored; if (!returningFromLeftFactored && returningToLeftFactored) { continue; } atn.ruleToStopState[ruleTransition.target.ruleIndex].AddTransition(new EpsilonTransition (ruleTransition.followState)); } } foreach (ATNState state_2 in atn.states) { if (state_2 is BlockStartState) { // we need to know the end state to set its start state if (((BlockStartState)state_2).endState == null) { throw new InvalidOperationException(); } // block end states can only be associated to a single block start state if (((BlockStartState)state_2).endState.startState != null) { throw new InvalidOperationException(); } ((BlockStartState)state_2).endState.startState = (BlockStartState)state_2; } if (state_2 is PlusLoopbackState) { PlusLoopbackState loopbackState = (PlusLoopbackState)state_2; for (int i_10 = 0; i_10 < loopbackState.NumberOfTransitions; i_10++) { ATNState target = loopbackState.Transition(i_10).target; if (target is PlusBlockStartState) { ((PlusBlockStartState)target).loopBackState = loopbackState; } } } else { if (state_2 is StarLoopbackState) { StarLoopbackState loopbackState = (StarLoopbackState)state_2; for (int i_10 = 0; i_10 < loopbackState.NumberOfTransitions; i_10++) { ATNState target = loopbackState.Transition(i_10).target; if (target is StarLoopEntryState) { ((StarLoopEntryState)target).loopBackState = loopbackState; } } } } } // // DECISIONS // int ndecisions = ToInt(data[p++]); for (int i_11 = 1; i_11 <= ndecisions; i_11++) { int s = ToInt(data[p++]); DecisionState decState = (DecisionState)atn.states[s]; atn.decisionToState.Add(decState); decState.decision = i_11 - 1; } atn.decisionToDFA = new DFA[ndecisions]; for (int i_12 = 0; i_12 < ndecisions; i_12++) { atn.decisionToDFA[i_12] = new DFA(atn.decisionToState[i_12], i_12); } if (optimize) { while (true) { int optimizationCount = 0; optimizationCount += InlineSetRules(atn); optimizationCount += CombineChainedEpsilons(atn); bool preserveOrder = atn.grammarType == ATNType.Lexer; optimizationCount += OptimizeSets(atn, preserveOrder); if (optimizationCount == 0) { break; } } } IdentifyTailCalls(atn); VerifyATN(atn); return atn; }
/// <summary> /// Compute set of tokens that can follow /// <paramref name="s"/> /// in the ATN in the /// specified /// <paramref name="ctx"/> /// . /// <p/> /// If /// <paramref name="ctx"/> /// is /// <see cref="PredictionContext.EmptyLocal"/> /// and /// <paramref name="stopState"/> /// or the end of the rule containing /// <paramref name="s"/> /// is reached, /// <see cref="TokenConstants.EPSILON"/> /// is added to the result set. If /// <paramref name="ctx"/> /// is not /// <see cref="PredictionContext.EmptyLocal"/> /// and /// <paramref name="addEOF"/> /// is /// <see langword="true"/> /// and /// <paramref name="stopState"/> /// or the end of the outermost rule is reached, /// <see cref="TokenConstants.EOF"/> /// is added to the result set. /// </summary> /// <param name="s">the ATN state.</param> /// <param name="stopState"> /// the ATN state to stop at. This can be a /// <see cref="BlockEndState"/> /// to detect epsilon paths through a closure. /// </param> /// <param name="ctx"> /// The outer context, or /// <see cref="PredictionContext.EmptyLocal"/> /// if /// the outer context should not be used. /// </param> /// <param name="look">The result lookahead set.</param> /// <param name="lookBusy"> /// A set used for preventing epsilon closures in the ATN /// from causing a stack overflow. Outside code should pass /// <c>new HashSet<ATNConfig></c> /// for this argument. /// </param> /// <param name="calledRuleStack"> /// A set used for preventing left recursion in the /// ATN from causing a stack overflow. Outside code should pass /// <c>new BitSet()</c> /// for this argument. /// </param> /// <param name="seeThruPreds"> /// /// <see langword="true"/> /// to true semantic predicates as /// implicitly /// <see langword="true"/> /// and "see through them", otherwise /// <see langword="false"/> /// to treat semantic predicates as opaque and add /// <see cref="HitPred"/> /// to the /// result if one is encountered. /// </param> /// <param name="addEOF"> /// Add /// <see cref="TokenConstants.EOF"/> /// to the result if the end of the /// outermost context is reached. This parameter has no effect if /// <paramref name="ctx"/> /// is /// <see cref="PredictionContext.EmptyLocal"/> /// . /// </param> protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = new ATNConfig(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (ctx == null) { look.Add(TokenConstants.EPSILON); return; } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } } if (s is RuleStopState) { if (ctx == null) { look.Add(TokenConstants.EPSILON); return; } else if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.EOF); return; } if (ctx != PredictionContext.EMPTY) { for (int i = 0; i < ctx.Size; i++) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; bool removed = calledRuleStack.Get(returnState.ruleIndex); try { calledRuleStack.Clear(returnState.ruleIndex); Look(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } finally { if (removed) { calledRuleStack.Set(returnState.ruleIndex); } } } return; } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t is RuleTransition) { RuleTransition ruleTransition = (RuleTransition)t; if (calledRuleStack.Get(ruleTransition.ruleIndex)) { continue; } PredictionContext newContext = SingletonPredictionContext.Create(ctx, ruleTransition.followState.stateNumber); try { calledRuleStack.Set(ruleTransition.target.ruleIndex); Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } finally { calledRuleStack.Clear(ruleTransition.target.ruleIndex); } } else { if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else { look.Add(HitPred); } } else { if (t.IsEpsilon) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF); } else { if (t is WildcardTransition) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } look.AddAll(set); } } } } } } }