/// <summary> /// Given a start and stop index, return a /// <code>List</code> /// of all tokens in /// the token type /// <code>BitSet</code> /// . Return /// <code>null</code> /// if no tokens were found. This /// method looks at both on and off channel tokens. /// </summary> public virtual IList<IToken> GetTokens(int start, int stop, BitSet types) { LazyInit(); if (start < 0 || stop >= tokens.Count || stop < 0 || start >= tokens.Count) { throw new ArgumentOutOfRangeException("start " + start + " or stop " + stop + " not in 0.." + (tokens.Count - 1)); } if (start > stop) { return null; } // list = tokens[start:stop]:{T t, t.getType() in types} IList<IToken> filteredTokens = new List<IToken>(); for (int i = start; i <= stop; i++) { IToken t = tokens[i]; if (types == null || types.Get(t.Type)) { filteredTokens.Add(t); } } if (filteredTokens.Count == 0) { filteredTokens = null; } return filteredTokens; }
private static bool TestTailCall(ATN atn, RuleTransition transition, bool optimizedPath ) { if (!optimizedPath && transition.tailCall) { return true; } if (optimizedPath && transition.optimizedTailCall) { return true; } BitSet reachable = new BitSet(atn.states.Count); Stack<ATNState> worklist = new Stack<ATNState>(); worklist.Push(transition.followState); while (worklist.Count > 0) { ATNState state = worklist.Pop(); if (reachable.Get(state.stateNumber)) { continue; } if (state is RuleStopState) { continue; } if (!state.OnlyHasEpsilonTransitions) { return false; } IList<Transition> transitions = optimizedPath ? state.optimizedTransitions : state .transitions; foreach (Transition t in transitions) { if (t.TransitionType != TransitionType.Epsilon) { return false; } worklist.Push(t.target); } } return true; }
/// <summary> /// Compute set of tokens that can follow /// <code>s</code> /// in the ATN in the /// specified /// <code>ctx</code> /// . /// <p/> /// If /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>stopState</code> /// or the end of the rule containing /// <code>s</code> /// is reached, /// <see cref="TokenConstants.Epsilon"/> /// is added to the result set. If /// <code>ctx</code> /// is not /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// and /// <code>addEOF</code> /// is /// <code>true</code> /// and /// <code>stopState</code> /// or the end of the outermost rule is reached, /// <see cref="TokenConstants.Eof"/> /// is added to the result set. /// </summary> /// <param name="s">the ATN state.</param> /// <param name="stopState"> /// the ATN state to stop at. This can be a /// <see cref="BlockEndState">BlockEndState</see> /// to detect epsilon paths through a closure. /// </param> /// <param name="ctx"> /// The outer context, or /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// if /// the outer context should not be used. /// </param> /// <param name="look">The result lookahead set.</param> /// <param name="lookBusy"> /// A set used for preventing epsilon closures in the ATN /// from causing a stack overflow. Outside code should pass /// <code>new HashSet<ATNConfig></code> /// for this argument. /// </param> /// <param name="calledRuleStack"> /// A set used for preventing left recursion in the /// ATN from causing a stack overflow. Outside code should pass /// <code>new BitSet()</code> /// for this argument. /// </param> /// <param name="seeThruPreds"> /// /// <code>true</code> /// to true semantic predicates as /// implicitly /// <code>true</code> /// and "see through them", otherwise /// <code>false</code> /// to treat semantic predicates as opaque and add /// <see cref="HitPred">HitPred</see> /// to the /// result if one is encountered. /// </param> /// <param name="addEOF"> /// Add /// <see cref="TokenConstants.Eof"/> /// to the result if the end of the /// outermost context is reached. This parameter has no effect if /// <code>ctx</code> /// is /// <see cref="PredictionContext.EmptyLocal">PredictionContext.EmptyLocal</see> /// . /// </param> protected internal virtual void Look(ATNState s, ATNState stopState, PredictionContext ctx, IntervalSet look, HashSet<ATNConfig> lookBusy, BitSet calledRuleStack, bool seeThruPreds, bool addEOF) { // System.out.println("_LOOK("+s.stateNumber+", ctx="+ctx); ATNConfig c = ATNConfig.Create(s, 0, ctx); if (!lookBusy.Add(c)) { return; } if (s == stopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } } if (s is RuleStopState) { if (PredictionContext.IsEmptyLocal(ctx)) { look.Add(TokenConstants.Epsilon); return; } else { if (ctx.IsEmpty && addEOF) { look.Add(TokenConstants.Eof); return; } } for (int i = 0; i < ctx.Size; i++) { if (ctx.GetReturnState(i) != PredictionContext.EmptyFullStateKey) { ATNState returnState = atn.states[ctx.GetReturnState(i)]; // System.out.println("popping back to "+retState); for (int j = 0; j < ctx.Size; j++) { bool removed = calledRuleStack.Get(returnState.ruleIndex); try { calledRuleStack.Clear(returnState.ruleIndex); Look(returnState, stopState, ctx.GetParent(j), look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { if (removed) { calledRuleStack.Set(returnState.ruleIndex); } } } return; } } } int n = s.NumberOfTransitions; for (int i_1 = 0; i_1 < n; i_1++) { Transition t = s.Transition(i_1); if (t.GetType() == typeof(RuleTransition)) { if (calledRuleStack.Get(((RuleTransition)t).target.ruleIndex)) { continue; } PredictionContext newContext = ctx.GetChild(((RuleTransition)t).followState.stateNumber ); try { calledRuleStack.Set(((RuleTransition)t).target.ruleIndex); Look(t.target, stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds , addEOF); } finally { calledRuleStack.Clear(((RuleTransition)t).target.ruleIndex); } } else { if (t is AbstractPredicateTransition) { if (seeThruPreds) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { look.Add(HitPred); } } else { if (t.IsEpsilon) { Look(t.target, stopState, ctx, look, lookBusy, calledRuleStack, seeThruPreds, addEOF ); } else { if (t.GetType() == typeof(WildcardTransition)) { look.AddAll(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType)); } else { // System.out.println("adding "+ t); IntervalSet set = t.Label; if (set != null) { if (t is NotSetTransition) { set = set.Complement(IntervalSet.Of(TokenConstants.MinUserTokenType, atn.maxTokenType )); } look.AddAll(set); } } } } } } }
private static void CheckDependencies(IList<Tuple<RuleDependencyAttribute, ICustomAttributeProvider >> dependencies, Type recognizerType) { string[] ruleNames = GetRuleNames(recognizerType); int[] ruleVersions = GetRuleVersions(recognizerType, ruleNames); RuleDependencyChecker.RuleRelations relations = ExtractRuleRelations(recognizerType ); StringBuilder errors = new StringBuilder(); foreach (Tuple<RuleDependencyAttribute, ICustomAttributeProvider> dependency in dependencies) { if (!dependency.Item1.Recognizer.IsAssignableFrom(recognizerType)) { continue; } // this is the rule in the dependency set with the highest version number int effectiveRule = dependency.Item1.Rule; if (effectiveRule < 0 || effectiveRule >= ruleVersions.Length) { string message = string.Format("Rule dependency on unknown rule {0}@{1} in {2}", dependency .Item1.Rule, dependency.Item1.Version, dependency.Item1.Recognizer.ToString ()); errors.AppendLine(dependency.Item2.ToString()); errors.AppendLine(message); continue; } Dependents dependents = Dependents.Self | dependency.Item1.Dependents; ReportUnimplementedDependents(errors, dependency, dependents); BitSet @checked = new BitSet(); int highestRequiredDependency = CheckDependencyVersion(errors, dependency, ruleNames , ruleVersions, effectiveRule, null); if ((dependents & Dependents.Parents) != 0) { BitSet parents = relations.parents[dependency.Item1.Rule]; for (int parent = parents.NextSetBit(0); parent >= 0; parent = parents.NextSetBit (parent + 1)) { if (parent < 0 || parent >= ruleVersions.Length || @checked.Get(parent)) { continue; } @checked.Set(parent); int required = CheckDependencyVersion(errors, dependency, ruleNames, ruleVersions , parent, "parent"); highestRequiredDependency = Math.Max(highestRequiredDependency, required); } } if ((dependents & Dependents.Children) != 0) { BitSet children = relations.children[dependency.Item1.Rule]; for (int child = children.NextSetBit(0); child >= 0; child = children.NextSetBit( child + 1)) { if (child < 0 || child >= ruleVersions.Length || @checked.Get(child)) { continue; } @checked.Set(child); int required = CheckDependencyVersion(errors, dependency, ruleNames, ruleVersions , child, "child"); highestRequiredDependency = Math.Max(highestRequiredDependency, required); } } if ((dependents & Dependents.Ancestors) != 0) { BitSet ancestors = relations.GetAncestors(dependency.Item1.Rule); for (int ancestor = ancestors.NextSetBit(0); ancestor >= 0; ancestor = ancestors. NextSetBit(ancestor + 1)) { if (ancestor < 0 || ancestor >= ruleVersions.Length || @checked.Get(ancestor)) { continue; } @checked.Set(ancestor); int required = CheckDependencyVersion(errors, dependency, ruleNames, ruleVersions , ancestor, "ancestor"); highestRequiredDependency = Math.Max(highestRequiredDependency, required); } } if ((dependents & Dependents.Descendants) != 0) { BitSet descendants = relations.GetDescendants(dependency.Item1.Rule); for (int descendant = descendants.NextSetBit(0); descendant >= 0; descendant = descendants .NextSetBit(descendant + 1)) { if (descendant < 0 || descendant >= ruleVersions.Length || @checked.Get(descendant )) { continue; } @checked.Set(descendant); int required = CheckDependencyVersion(errors, dependency, ruleNames, ruleVersions , descendant, "descendant"); highestRequiredDependency = Math.Max(highestRequiredDependency, required); } } int declaredVersion = dependency.Item1.Version; if (declaredVersion > highestRequiredDependency) { string message = string.Format("Rule dependency version mismatch: {0} has maximum dependency version {1} (expected {2}) in {3}" , ruleNames[dependency.Item1.Rule], highestRequiredDependency, declaredVersion , dependency.Item1.Recognizer.ToString()); errors.AppendLine(dependency.Item2.ToString()); errors.AppendLine(message); } } if (errors.Length > 0) { throw new InvalidOperationException(errors.ToString()); } }