public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet ambigAlts, ATNConfigSet configs) { string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; recognizer.NotifyErrorListeners(string.Format(format, GetDecisionDescription(recognizer , dfa.decision), ambigAlts, ((ITokenStream)recognizer.InputStream).GetText(Interval .Of(startIndex, stopIndex)))); }
public NoViableAltException(IRecognizer recognizer, ITokenStream input, IToken startToken, IToken offendingToken, ATNConfigSet deadEndConfigs, ParserRuleContext ctx) : base(recognizer, input, ctx) { // LL(1) error this.deadEndConfigs = deadEndConfigs; this.startToken = startToken; this.OffendingToken = offendingToken; }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { if (exactOnly && !exact) { return; } string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; string decision = GetDecisionDescription(recognizer, dfa); BitSet conflictingAlts = GetConflictingAlts(ambigAlts, configs); string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex, stopIndex)); string message = string.Format(format, decision, conflictingAlts, text); recognizer.NotifyErrorListeners(message); }
// configs that LL not SLL considered conflicting /** If context sensitive parsing, we know it's ambiguity not conflict */ protected virtual void ReportAmbiguity(DFA dfa, DFAState D, // the DFA state from execATN() that had SLL conflicts int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { if (debug || retry_debug) { Interval interval = Interval.Of(startIndex, stopIndex); Console.WriteLine("ReportAmbiguity " + ambigAlts + ":" + configs + ", input=" + parser.TokenStream.GetText(interval)); } if (parser != null) parser.ErrorListenerDispatch.ReportAmbiguity(parser, dfa, startIndex, stopIndex, exact, ambigAlts, configs); }
public DFAState(DFA dfa, ATNConfigSet configs) : this(dfa.EmptyEdgeMap, dfa.EmptyContextEdgeMap, configs) { }
public LexerNoViableAltException(Lexer lexer, ICharStream input, int startIndex, ATNConfigSet deadEndConfigs) : base(lexer, input) { this.startIndex = startIndex; this.deadEndConfigs = deadEndConfigs; }
protected override DFAState CreateDFAState(DFA dfa, ATNConfigSet configs) { int t = _input.La(1); if (t == AntlrV4.CaretToken.CaretTokenType && !_computingStartState) { _caretToken = (ICaretToken)_input.Lt(1); throw NoViableAlt(_input, _outerContext, configs, _startIndex); } return base.CreateDFAState(dfa, configs); }
protected internal virtual BitSet GetConflictingAlts(BitSet reportedAlts, ATNConfigSet configs) { if (reportedAlts != null) { return reportedAlts; } BitSet result = new BitSet(); foreach (ATNConfig config in configs) { result.Set(config.Alt); } return result; }
public override IToken RecoverInline(Parser recognizer) { if (recognizer is ICodeCompletionParser && ((ITokenStream)recognizer.InputStream).Lt(1) is ICaretToken) { ICodeCompletionParser parser = (ICodeCompletionParser)recognizer; ICaretToken token = (ICaretToken)((ITokenStream)recognizer.InputStream).Lt(1); CompletionParserATNSimulator interp = parser.Interpreter; int stateNumber = recognizer.State; ATNState state = interp.atn.states[stateNumber]; PredictionContext context = PredictionContext.FromRuleContext(interp.atn, recognizer.Context, false); ATNConfigSet intermediate = new ATNConfigSet(); ATNConfigSet closure = new ATNConfigSet(); for (int i = 0; i < state.NumberOfTransitions; i++) { Transition transition = state.Transition(i); if (transition.IsEpsilon) { ATNState target = transition.target; ATNConfig config = ATNConfig.Create(target, i + 1, context); intermediate.Add(config); } } bool collectPredicates = false; bool hasMoreContext = true; interp.ClosureHelper(intermediate, closure, collectPredicates, hasMoreContext, PredictionContextCache.Uncached, false); if (!state.OnlyHasEpsilonTransitions) { for (int i = 0; i < state.NumberOfTransitions; i++) { closure.Add(ATNConfig.Create(state, i + 1, PredictionContext.FromRuleContext(interp.atn, recognizer.Context))); } } Dictionary<ATNConfig, IList<Transition>> transitions = null; int ncl = closure.Count; // TODO: foreach for (int ci = 0; ci < ncl; ci++) { ATNConfig c = closure[ci]; List<Transition> configTransitions = null; int n = c.State.NumberOfTransitions; for (int ti = 0; ti < n; ti++) { // for each transition Transition trans = c.State.Transition(ti); ATNState target = interp.GetReachableTargetHelper(c, trans, CaretToken.CaretTokenType); if (target != null) { if (transitions == null) { transitions = new Dictionary<ATNConfig, IList<Transition>>(); } if (configTransitions == null) { configTransitions = new List<Transition>(); transitions[c] = configTransitions; } configTransitions.Add(trans); } } } /* * This should be null if the intended token is not "wordlike", and * should be a single transition from the current state. */ if (transitions != null) { Debug.Assert(transitions.Count == 1); Debug.Assert(transitions.Values.First().Count == 1); Debug.Assert(state.NumberOfTransitions == 1); Debug.Assert(transitions.Values.First()[0] == state.Transition(0)); } throw new CaretReachedException(parser.Context, token, transitions, null); } return base.RecoverInline(recognizer); }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; string decision = GetDecisionDescription(recognizer, dfa); BitSet conflictingAlts = GetConflictingAlts(ambigAlts, configs); string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex, stopIndex)); string message = string.Format(format, decision, conflictingAlts, text); this.AmbiguityErrors.Add(message); }
public DFAState(ATNConfigSet configs) { this.configSet = configs; }
public void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { throw new Exception("Ambiguity error "); }
public override void ReportAmbiguity(Antlr4.Runtime.Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { }
public void ClosureHelper(ATNConfigSet sourceConfigs, ATNConfigSet configs, bool collectPredicates, bool hasMoreContext, PredictionContextCache contextCache, bool treatEofAsEpsilon) { Closure(sourceConfigs, configs, collectPredicates, hasMoreContext, contextCache, treatEofAsEpsilon); }
protected virtual void ReportContextSensitivity(DFA dfa, int prediction, ATNConfigSet configs, int startIndex, int stopIndex) { if (debug || retry_debug) { Interval interval = Interval.Of(startIndex, stopIndex); Console.WriteLine("ReportContextSensitivity decision=" + dfa.decision + ":" + configs + ", input=" + parser.TokenStream.GetText(interval)); } if (parser != null) parser.ErrorListenerDispatch.ReportContextSensitivity(parser, dfa, startIndex, stopIndex, prediction, null /*configs*/); }
protected static int GetUniqueAlt(ATNConfigSet configSet) { int alt = ATN.INVALID_ALT_NUMBER; foreach (ATNConfig c in configSet.configs) { if (alt == ATN.INVALID_ALT_NUMBER) { alt = c.alt; // found first alt } else if (c.alt != alt) { return ATN.INVALID_ALT_NUMBER; } } return alt; }
// XXX: Ved ikke om disse er relevante at override. Tror aldrig de rapporterer om fejl. // Men nu har vi fjernet ANTLRs egen error logger, så tænker vi bør beholde dem for nu, hvis de skulle vise sig brugbare. public override void ReportAmbiguity([NotNull] Parser recognizer, [NotNull] DFA dfa, int startIndex, int stopIndex, bool exact, [Nullable] BitSet ambigAlts, [NotNull] ATNConfigSet configs) { LogError("Some ambiguity is going on", new SourcePosition(startIndex, 0)); }
protected BitSet GetConflictingAlts(BitSet reportedAlts, ATNConfigSet configSet) => reportedAlts ?? configSet.ConflictingAlts;
public override void ReportAmbiguity([NotNull] Parser recognizer, [NotNull] DFA dfa, int startIndex, int stopIndex, bool exact, [Nullable] BitSet ambigAlts, [NotNull] ATNConfigSet configs) { base.ReportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); }
public DFAState(ATNConfigSet configs, int minSymbol, int maxSymbol) { this.configs = configs; this.minSymbol = minSymbol; this.maxSymbol = maxSymbol; }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { base.ReportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); }
public override void ReportAmbiguity([NotNull] Parser recognizer, [NotNull] DFA dfa, int startIndex, int stopIndex, bool exact, [Nullable] BitSet ambigAlts, [NotNull] ATNConfigSet configs) { if (exactOnly && !exact) { return; } string format = "reportAmbiguity d={0}: ambigAlts={1}, input='{2}'"; string decision = GetDecisionDescription(recognizer, dfa); BitSet conflictingAlts = GetConflictingAlts(ambigAlts, configs); string text = ((ITokenStream)recognizer.InputStream).GetText(Interval.Of(startIndex, stopIndex)); string message = string.Format(format, decision, conflictingAlts, text); recognizer.NotifyErrorListeners(message); }
protected override DFAState CreateDFAState([NotNull] ATNConfigSet configs) { return(new DFAState(configs, -1, -1)); }
protected override DFAState AddDFAState(ATNConfigSet configs) { return(null); }
public virtual void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet ambigAlts, ATNConfigSet configs) { }
public void ReportAmbiguity([NotNull] Parser recognizer, [NotNull] DFA dfa, int startIndex, int stopIndex, bool exact, [Nullable] BitSet ambigAlts, [NotNull] ATNConfigSet configs) { // Ignore }
public DFAState([NotNull] EmptyEdgeMap <DFAState> emptyEdges, [NotNull] EmptyEdgeMap <DFAState> emptyContextEdges, [NotNull] ATNConfigSet configs) { this.configs = configs; this.edges = emptyEdges; this.contextEdges = emptyContextEdges; }
public override IToken RecoverInline(Parser recognizer) { if (recognizer is ICodeCompletionParser && ((ITokenStream)recognizer.InputStream).Lt(1) is ICaretToken) { ICodeCompletionParser parser = (ICodeCompletionParser)recognizer; ICaretToken token = (ICaretToken)((ITokenStream)recognizer.InputStream).Lt(1); CompletionParserATNSimulator interp = parser.Interpreter; int stateNumber = recognizer.State; ATNState state = interp.atn.states[stateNumber]; PredictionContext context = PredictionContext.FromRuleContext(interp.atn, recognizer.Context, false); ATNConfigSet intermediate = new ATNConfigSet(); ATNConfigSet closure = new ATNConfigSet(); for (int i = 0; i < state.NumberOfTransitions; i++) { Transition transition = state.Transition(i); if (transition.IsEpsilon) { ATNState target = transition.target; ATNConfig config = ATNConfig.Create(target, i + 1, context); intermediate.Add(config); } } bool collectPredicates = false; bool hasMoreContext = true; interp.ClosureHelper(intermediate, closure, collectPredicates, hasMoreContext, PredictionContextCache.Uncached, false); if (!state.OnlyHasEpsilonTransitions) { for (int i = 0; i < state.NumberOfTransitions; i++) { closure.Add(ATNConfig.Create(state, i + 1, PredictionContext.FromRuleContext(interp.atn, recognizer.Context))); } } Dictionary <ATNConfig, IList <Transition> > transitions = null; int ncl = closure.Count; // TODO: foreach for (int ci = 0; ci < ncl; ci++) { ATNConfig c = closure[ci]; List <Transition> configTransitions = null; int n = c.State.NumberOfTransitions; for (int ti = 0; ti < n; ti++) { // for each transition Transition trans = c.State.Transition(ti); ATNState target = interp.GetReachableTargetHelper(c, trans, CaretToken.CaretTokenType); if (target != null) { if (transitions == null) { transitions = new Dictionary <ATNConfig, IList <Transition> >(); } if (configTransitions == null) { configTransitions = new List <Transition>(); transitions[c] = configTransitions; } configTransitions.Add(trans); } } } /* * This should be null if the intended token is not "wordlike", and * should be a single transition from the current state. */ if (transitions != null) { Debug.Assert(transitions.Count == 1); Debug.Assert(transitions.Values.First().Count == 1); Debug.Assert(state.NumberOfTransitions == 1); Debug.Assert(transitions.Values.First()[0] == state.Transition(0)); } throw new CaretReachedException(parser.Context, token, transitions, null); } return(base.RecoverInline(recognizer)); }
protected virtual void ReportAttemptingFullContext(DFA dfa, BitSet conflictingAlts, ATNConfigSet configs, int startIndex, int stopIndex) { if (debug || retry_debug) { Interval interval = Interval.Of(startIndex, stopIndex); Console.WriteLine("reportAttemptingFullContext decision=" + dfa.decision + ":" + configs + ", input=" + parser.TokenStream.GetText(interval)); } if (parser != null) parser.ErrorListenerDispatch.ReportAttemptingFullContext(parser, dfa, startIndex, stopIndex, conflictingAlts, null /*configs*/); }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { ErrorCount++; }
/** Walk the list of configurations and split them according to * those that have preds evaluating to true/false. If no pred, assume * true pred and include in succeeded set. Returns Pair of sets. * * Create a new set so as not to alter the incoming parameter. * * Assumption: the input stream has been restored to the starting point * prediction, which is where predicates need to evaluate. */ protected Pair<ATNConfigSet, ATNConfigSet> SplitAccordingToSemanticValidity( ATNConfigSet configSet, ParserRuleContext outerContext) { ATNConfigSet succeeded = new ATNConfigSet(configSet.fullCtx); ATNConfigSet failed = new ATNConfigSet(configSet.fullCtx); foreach (ATNConfig c in configSet.configs) { if (c.semanticContext != SemanticContext.NONE) { bool predicateEvaluationResult = EvalSemanticContext(c.semanticContext, outerContext, c.alt, configSet.fullCtx); if (predicateEvaluationResult) { succeeded.Add(c); } else { failed.Add(c); } } else { succeeded.Add(c); } } return new Pair<ATNConfigSet, ATNConfigSet>(succeeded, failed); }
protected void ClosureCheckingStopState(ATNConfig config, ATNConfigSet configSet, HashSet<ATNConfig> closureBusy, bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon) { if (debug) Console.WriteLine("closure(" + config.ToString(parser, true) + ")"); if (config.state is RuleStopState) { // We hit rule end. If we have context info, use it // run thru all possible stack tops in ctx if (!config.context.IsEmpty) { for (int i = 0; i < config.context.Size; i++) { if (config.context.GetReturnState(i) == PredictionContext.EMPTY_RETURN_STATE) { if (fullCtx) { configSet.Add(new ATNConfig(config, config.state, PredictionContext.EMPTY), mergeCache); continue; } else { // we have no context info, just chase follow links (if greedy) if (debug) Console.WriteLine("FALLING off rule " + GetRuleName(config.state.ruleIndex)); Closure_(config, configSet, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon); } continue; } ATNState returnState = atn.states[config.context.GetReturnState(i)]; PredictionContext newContext = config.context.GetParent(i); // "pop" return state ATNConfig c = new ATNConfig(returnState, config.alt, newContext, config.semanticContext); // While we have context to pop back from, we may have // gotten that context AFTER having falling off a rule. // Make sure we track that we are now out of context. // // This assignment also propagates the // isPrecedenceFilterSuppressed() value to the new // configuration. c.reachesIntoOuterContext = config.OuterContextDepth; ClosureCheckingStopState(c, configSet, closureBusy, collectPredicates, fullCtx, depth - 1, treatEofAsEpsilon); } return; } else if (fullCtx) { // reached end of start rule configSet.Add(config, mergeCache); return; } else { // else if we have no context info, just chase follow links (if greedy) if (debug) Console.WriteLine("FALLING off rule " + GetRuleName(config.state.ruleIndex)); } } Closure_(config, configSet, closureBusy, collectPredicates, fullCtx, depth, treatEofAsEpsilon); }
/* parrt internal source braindump that doesn't mess up * external API spec. context-sensitive in that they can only be properly evaluated in the context of the proper prec argument. Without pruning, these predicates are normal predicates evaluated when we reach conflict state (or unique prediction). As we cannot evaluate these predicates out of context, the resulting conflict leads to full LL evaluation and nonlinear prediction which shows up very clearly with fairly large expressions. Example grammar: e : e '*' e | e '+' e | INT ; We convert that to the following: e[int prec] : INT ( {3>=prec}? '*' e[4] | {2>=prec}? '+' e[3] )* ; The (..)* loop has a decision for the inner block as well as an enter or exit decision, which is what concerns us here. At the 1st + of input 1+2+3, the loop entry sees both predicates and the loop exit also sees both predicates by falling off the edge of e. This is because we have no stack information with SLL and find the follow of e, which will hit the return states inside the loop after e[4] and e[3], which brings it back to the enter or exit decision. In this case, we know that we cannot evaluate those predicates because we have fallen off the edge of the stack and will in general not know which prec parameter is the right one to use in the predicate. Because we have special information, that these are precedence predicates, we can resolve them without failing over to full LL despite their context sensitive nature. We make an assumption that prec[-1] <= prec[0], meaning that the current precedence level is greater than or equal to the precedence level of recursive invocations above us in the stack. For example, if predicate {3>=prec}? is true of the current prec, then one option is to enter the loop to match it now. The other option is to exit the loop and the left recursive rule to match the current operator in rule invocation further up the stack. But, we know that all of those prec are lower or the same value and so we can decide to enter the loop instead of matching it later. That means we can strip out the other configuration for the exit branch. So imagine we have (14,1,$,{2>=prec}?) and then (14,2,$-dipsIntoOuterContext,{2>=prec}?). The optimization allows us to collapse these two configurations. We know that if {2>=prec}? is true for the current prec parameter, it will also be true for any prec from an invoking e call, indicated by dipsIntoOuterContext. As the predicates are both true, we have the option to evaluate them early in the decision start state. We do this by stripping both predicates and choosing to enter the loop as it is consistent with the notion of operator precedence. It's also how the full LL conflict resolution would work. The solution requires a different DFA start state for each precedence level. The basic filter mechanism is to remove configurations of the form (p, 2, pi) if (p, 1, pi) exists for the same p and pi. In other words, for the same ATN state and predicate context, remove any configuration associated with an exit branch if there is a configuration associated with the enter branch. It's also the case that the filter evaluates precedence predicates and resolves conflicts according to precedence levels. For example, for input 1+2+3 at the first +, we see prediction filtering [(11,1,[$],{3>=prec}?), (14,1,[$],{2>=prec}?), (5,2,[$],up=1), (11,2,[$],up=1), (14,2,[$],up=1)],hasSemanticContext=true,dipsIntoOuterContext to [(11,1,[$]), (14,1,[$]), (5,2,[$],up=1)],dipsIntoOuterContext This filters because {3>=prec}? evals to true and collapses (11,1,[$],{3>=prec}?) and (11,2,[$],up=1) since early conflict resolution based upon rules of operator precedence fits with our usual match first alt upon conflict. We noticed a problem where a recursive call resets precedence to 0. Sam's fix: each config has flag indicating if it has returned from an expr[0] call. then just don't filter any config with that flag set. flag is carried along in closure(). so to avoid adding field, set bit just under sign bit of dipsIntoOuterContext (SUPPRESS_PRECEDENCE_FILTER). With the change you filter "unless (p, 2, pi) was reached after leaving the rule stop state of the LR rule containing state p, corresponding to a rule invocation with precedence level 0" */ /** * This method transforms the start state computed by * {@link #computeStartState} to the special start state used by a * precedence DFA for a particular precedence value. The transformation * process applies the following changes to the start state's configuration * set. * * <ol> * <li>Evaluate the precedence predicates for each configuration using * {@link SemanticContext#evalPrecedence}.</li> * <li>When {@link ATNConfig#isPrecedenceFilterSuppressed} is {@code false}, * remove all configurations which predict an alternative greater than 1, * for which another configuration that predicts alternative 1 is in the * same ATN state with the same prediction context. This transformation is * valid for the following reasons: * <ul> * <li>The closure block cannot contain any epsilon transitions which bypass * the body of the closure, so all states reachable via alternative 1 are * part of the precedence alternatives of the transformed left-recursive * rule.</li> * <li>The "primary" portion of a left recursive rule cannot contain an * epsilon transition, so the only way an alternative other than 1 can exist * in a state that is also reachable via alternative 1 is by nesting calls * to the left-recursive rule, with the outer calls not being at the * preferred precedence level. The * {@link ATNConfig#isPrecedenceFilterSuppressed} property marks ATN * configurations which do not meet this condition, and therefore are not * eligible for elimination during the filtering process.</li> * </ul> * </li> * </ol> * * <p> * The prediction context must be considered by this filter to address * situations like the following. * </p> * <code> * <pre> * grammar TA; * prog: statement* EOF; * statement: letterA | statement letterA 'b' ; * letterA: 'a'; * </pre> * </code> * <p> * If the above grammar, the ATN state immediately before the token * reference {@code 'a'} in {@code letterA} is reachable from the left edge * of both the primary and closure blocks of the left-recursive rule * {@code statement}. The prediction context associated with each of these * configurations distinguishes between them, and prevents the alternative * which stepped out to {@code prog} (and then back in to {@code statement} * from being eliminated by the filter. * </p> * * @param configs The configuration set computed by * {@link #computeStartState} as the start state for the DFA. * @return The transformed configuration set representing the start state * for a precedence DFA at a particular precedence level (determined by * calling {@link Parser#getPrecedence}). */ protected ATNConfigSet ApplyPrecedenceFilter(ATNConfigSet configSet) { Dictionary<int, PredictionContext> statesFromAlt1 = new Dictionary<int, PredictionContext>(); ATNConfigSet result = new ATNConfigSet(configSet.fullCtx); foreach (ATNConfig config in configSet.configs) { // handle alt 1 first if (config.alt != 1) { continue; } SemanticContext updatedContext = config.semanticContext.EvalPrecedence(parser, context); if (updatedContext == null) { // the configuration was eliminated continue; } statesFromAlt1[config.state.stateNumber] = config.context; if (updatedContext != config.semanticContext) { result.Add(new ATNConfig(config, updatedContext), mergeCache); } else { result.Add(config, mergeCache); } } foreach (ATNConfig config in configSet.configs) { if (config.alt == 1) { // already handled continue; } if (!config.IsPrecedenceFilterSuppressed) { /* In the future, this elimination step could be updated to also * filter the prediction context for alternatives predicting alt>1 * (basically a graph subtraction algorithm). */ PredictionContext ctx; if (statesFromAlt1.TryGetValue(config.state.stateNumber, out ctx)) { if (ctx != null && ctx.Equals(config.context)) { // eliminated continue; } } } result.Add(config, mergeCache); } return result; }
/** Do the actual work of walking epsilon edges */ protected void Closure_(ATNConfig config, ATNConfigSet configs, HashSet<ATNConfig> closureBusy, bool collectPredicates, bool fullCtx, int depth, bool treatEofAsEpsilon) { ATNState p = config.state; // optimization if (!p.OnlyHasEpsilonTransitions) { configs.Add(config, mergeCache); // make sure to not return here, because EOF transitions can act as // both epsilon transitions and non-epsilon transitions. // if ( debug ) Console.WriteLine("added config "+configs); } for (int i = 0; i < p.NumberOfTransitions; i++) { if (i == 0 && CanDropLoopEntryEdgeInLeftRecursiveRule(config)) continue; Transition t = p.Transition(i); bool continueCollecting = !(t is ActionTransition) && collectPredicates; ATNConfig c = GetEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEofAsEpsilon); if (c != null) { if (!t.IsEpsilon && !closureBusy.Add(c)) { // avoid infinite recursion for EOF* and EOF+ continue; } int newDepth = depth; if (config.state is RuleStopState) { // target fell off end of rule; mark resulting c as having dipped into outer context // We can't get here if incoming config was rule stop and we had context // track how far we dip into outer context. Might // come in handy and we avoid evaluating context dependent // preds if this is > 0. if (!closureBusy.Add(c)) { // avoid infinite recursion for right-recursive rules continue; } if (thisDfa != null && thisDfa.IsPrecedenceDfa) { int outermostPrecedenceReturn = ((EpsilonTransition)t).OutermostPrecedenceReturn; if (outermostPrecedenceReturn == thisDfa.atnStartState.ruleIndex) { c.SetPrecedenceFilterSuppressed(true); } } c.reachesIntoOuterContext++; configs.dipsIntoOuterContext = true; // TODO: can remove? only care when we add to set per middle of this method newDepth--; if (debug) Console.WriteLine("dips into outer ctx: " + c); } else if (t is RuleTransition) { // latch when newDepth goes negative - once we step out of the entry context we can't return if (newDepth >= 0) { newDepth++; } } ClosureCheckingStopState(c, configs, closureBusy, continueCollecting, fullCtx, newDepth, treatEofAsEpsilon); } } }
public DFAState(EmptyEdgeMap <DFAState> emptyEdges, EmptyEdgeMap <DFAState> emptyContextEdges, ATNConfigSet configs) { this.configs = configs; this.edges = emptyEdges; this.contextEdges = emptyContextEdges; }
protected virtual ATNConfigSet ComputeReachSet(ATNConfigSet closure, int t, bool fullCtx) { if (debug) Console.WriteLine("in computeReachSet, starting closure: " + closure); if (mergeCache == null) { mergeCache = new MergeCache(); } ATNConfigSet intermediate = new ATNConfigSet(fullCtx); /* Configurations already in a rule stop state indicate reaching the end * of the decision rule (local context) or end of the start rule (full * context). Once reached, these configurations are never updated by a * closure operation, so they are handled separately for the performance * advantage of having a smaller intermediate set when calling closure. * * For full-context reach operations, separate handling is required to * ensure that the alternative matching the longest overall sequence is * chosen when multiple such configurations can match the input. */ List<ATNConfig> skippedStopStates = null; // First figure out where we can reach on input t foreach (ATNConfig c in closure.configs) { if (debug) Console.WriteLine("testing " + GetTokenName(t) + " at " + c.ToString()); if (c.state is RuleStopState) { if (fullCtx || t == IntStreamConstants.EOF) { if (skippedStopStates == null) { skippedStopStates = new List<ATNConfig>(); } skippedStopStates.Add(c); } continue; } int n = c.state.NumberOfTransitions; for (int ti = 0; ti < n; ti++) { // for each transition Transition trans = c.state.Transition(ti); ATNState target = GetReachableTarget(trans, t); if (target != null) { intermediate.Add(new ATNConfig(c, target), mergeCache); } } } // Now figure out where the reach operation can take us... ATNConfigSet reach = null; /* This block optimizes the reach operation for intermediate sets which * trivially indicate a termination state for the overall * adaptivePredict operation. * * The conditions assume that intermediate * contains all configurations relevant to the reach set, but this * condition is not true when one or more configurations have been * withheld in skippedStopStates, or when the current symbol is EOF. */ if (skippedStopStates == null && t != TokenConstants.EOF) { if (intermediate.Count == 1) { // Don't pursue the closure if there is just one state. // It can only have one alternative; just add to result // Also don't pursue the closure if there is unique alternative // among the configurations. reach = intermediate; } else if (GetUniqueAlt(intermediate) != ATN.INVALID_ALT_NUMBER) { // Also don't pursue the closure if there is unique alternative // among the configurations. reach = intermediate; } } /* If the reach set could not be trivially determined, perform a closure * operation on the intermediate set to compute its initial value. */ if (reach == null) { reach = new ATNConfigSet(fullCtx); HashSet<ATNConfig> closureBusy = new HashSet<ATNConfig>(); bool treatEofAsEpsilon = t == TokenConstants.EOF; foreach (ATNConfig c in intermediate.configs) { Closure(c, reach, closureBusy, false, fullCtx, treatEofAsEpsilon); } } if (t == IntStreamConstants.EOF) { /* After consuming EOF no additional input is possible, so we are * only interested in configurations which reached the end of the * decision rule (local context) or end of the start rule (full * context). Update reach to contain only these configurations. This * handles both explicit EOF transitions in the grammar and implicit * EOF transitions following the end of the decision or start rule. * * When reach==intermediate, no closure operation was performed. In * this case, removeAllConfigsNotInRuleStopState needs to check for * reachable rule stop states as well as configurations already in * a rule stop state. * * This is handled before the configurations in skippedStopStates, * because any configurations potentially added from that list are * already guaranteed to meet this condition whether or not it's * required. */ reach = RemoveAllConfigsNotInRuleStopState(reach, reach == intermediate); } /* If skippedStopStates is not null, then it contains at least one * configuration. For full-context reach operations, these * configurations reached the end of the start rule, in which case we * only add them back to reach if no configuration during the current * closure operation reached such a state. This ensures adaptivePredict * chooses an alternative matching the longest overall sequence when * multiple alternatives are viable. */ if (skippedStopStates != null && (!fullCtx || !PredictionMode.HasConfigInRuleStopState(reach.configs))) { foreach (ATNConfig c in skippedStopStates) { reach.Add(c, mergeCache); } } if (reach.Empty) return null; return reach; }
public virtual void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { }
protected ATNConfigSet ComputeStartState(ATNState p, RuleContext ctx, bool fullCtx) { // always at least the implicit call to start rule PredictionContext initialContext = PredictionContext.FromRuleContext(atn, ctx); ATNConfigSet configs = new ATNConfigSet(fullCtx); for (int i = 0; i < p.NumberOfTransitions; i++) { ATNState target = p.Transition(i).target; ATNConfig c = new ATNConfig(target, i + 1, initialContext); HashSet<ATNConfig> closureBusy = new HashSet<ATNConfig>(); Closure(c, configs, closureBusy, true, fullCtx, false); } return configs; }
// comes back with reach.UniqueAlt set to a valid alt protected int ExecATNWithFullContext(DFA dfa, DFAState D, // how far we got in SLL DFA before failing over ATNConfigSet s0, ITokenStream input, int startIndex, ParserRuleContext outerContext) { if (debug || debug_list_atn_decisions) { Console.WriteLine("execATNWithFullContext " + s0); } bool fullCtx = true; bool foundExactAmbig = false; ATNConfigSet reach = null; ATNConfigSet previous = s0; input.Seek(startIndex); int t = input.LA(1); int predictedAlt; while (true) { // while more work // Console.WriteLine("LL REACH "+GetLookaheadName(input)+ // " from configs.size="+previous.size()+ // " line "+input.LT(1)Line+":"+input.LT(1).Column); reach = ComputeReachSet(previous, t, fullCtx); if (reach == null) { // if any configs in previous dipped into outer context, that // means that input up to t actually finished entry rule // at least for LL decision. Full LL doesn't dip into outer // so don't need special case. // We will get an error no matter what so delay until after // decision; better error message. Also, no reachable target // ATN states in SLL implies LL will also get nowhere. // If conflict in states that dip out, choose min since we // will get error no matter what. NoViableAltException e = NoViableAlt(input, outerContext, previous, startIndex); input.Seek(startIndex); int alt = GetSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previous, outerContext); if (alt != ATN.INVALID_ALT_NUMBER) { return alt; } throw e; } ICollection<BitSet> altSubSets = PredictionMode.GetConflictingAltSubsets(reach.configs); if (debug) { Console.WriteLine("LL altSubSets=" + altSubSets + ", predict=" + PredictionMode.GetUniqueAlt(altSubSets) + ", ResolvesToJustOneViableAlt=" + PredictionMode.ResolvesToJustOneViableAlt(altSubSets)); } // Console.WriteLine("altSubSets: "+altSubSets); // System.err.println("reach="+reach+", "+reach.conflictingAlts); reach.uniqueAlt = GetUniqueAlt(reach); // unique prediction? if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { predictedAlt = reach.uniqueAlt; break; } if (mode != PredictionMode.LL_EXACT_AMBIG_DETECTION) { predictedAlt = PredictionMode.ResolvesToJustOneViableAlt(altSubSets); if (predictedAlt != ATN.INVALID_ALT_NUMBER) { break; } } else { // In exact ambiguity mode, we never try to terminate early. // Just keeps scarfing until we know what the conflict is if (PredictionMode.AllSubsetsConflict(altSubSets) && PredictionMode.AllSubsetsEqual(altSubSets)) { foundExactAmbig = true; predictedAlt = PredictionMode.GetSingleViableAlt(altSubSets); break; } // else there are multiple non-conflicting subsets or // we're not sure what the ambiguity is yet. // So, keep going. } previous = reach; if (t != IntStreamConstants.EOF) { input.Consume(); t = input.LA(1); } } // If the configuration set uniquely predicts an alternative, // without conflict, then we know that it's a full LL decision // not SLL. if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { ReportContextSensitivity(dfa, predictedAlt, reach, startIndex, input.Index); return predictedAlt; } // We do not check predicates here because we have checked them // on-the-fly when doing full context prediction. /* In non-exact ambiguity detection mode, we might actually be able to detect an exact ambiguity, but I'm not going to spend the cycles needed to check. We only emit ambiguity warnings in exact ambiguity mode. For example, we might know that we have conflicting configurations. But, that does not mean that there is no way forward without a conflict. It's possible to have nonconflicting alt subsets as in: LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}] from [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]), (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])] In this case, (17,1,[5 $]) indicates there is some next sequence that would resolve this without conflict to alternative 1. Any other viable next sequence, however, is associated with a conflict. We stop looking for input because no amount of further lookahead will alter the fact that we should predict alternative 1. We just can't say for sure that there is an ambiguity without looking further. */ ReportAmbiguity(dfa, D, startIndex, input.Index, foundExactAmbig, reach.GetAlts(), reach); return predictedAlt; }
/// <summary> /// Sets <see cref="IsValid"/> to false /// </summary> public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { IsValid = false; }
protected int getAltThatFinishedDecisionEntryRule(ATNConfigSet configSet) { IntervalSet alts = new IntervalSet(); foreach (ATNConfig c in configSet.configs) { if (c.OuterContextDepth > 0 || (c.state is RuleStopState && c.context.HasEmptyPath)) { alts.Add(c.alt); } } if (alts.Count == 0) return ATN.INVALID_ALT_NUMBER; return alts.MinElement; }
protected internal virtual BitSet GetConflictingAlts([Nullable] BitSet reportedAlts, [NotNull] ATNConfigSet configs) { if (reportedAlts != null) { return(reportedAlts); } BitSet result = new BitSet(); foreach (ATNConfig config in configs) { result.Set(config.Alt); } return(result); }
/** * Gets a {@link BitSet} containing the alternatives in {@code configs} * which are part of one or more conflicting alternative subsets. * * @param configs The {@link ATNConfigSet} to analyze. * @return The alternatives in {@code configs} which are part of one or more * conflicting alternative subsets. If {@code configs} does not contain any * conflicting subsets, this method returns an empty {@link BitSet}. */ protected BitSet GetConflictingAlts(ATNConfigSet configSet) { ICollection<BitSet> altsets = PredictionMode.GetConflictingAltSubsets(configSet.configs); return PredictionMode.GetAlts(altsets); }
/** Sam pointed out a problem with the previous definition, v3, of ambiguous states. If we have another state associated with conflicting alternatives, we should keep going. For example, the following grammar s : (ID | ID ID?) ';' ; When the ATN simulation reaches the state before ';', it has a DFA state that looks like: [12|1|[], 6|2|[], 12|2|[]]. Naturally 12|1|[] and 12|2|[] conflict, but we cannot stop processing this node because alternative to has another way to continue, via [6|2|[]]. The key is that we have a single state that has config's only associated with a single alternative, 2, and crucially the state transitions among the configurations are all non-epsilon transitions. That means we don't consider any conflicts that include alternative 2. So, we ignore the conflict between alts 1 and 2. We ignore a set of conflicting alts when there is an intersection with an alternative associated with a single alt state in the state→config-list map. It's also the case that we might have two conflicting configurations but also a 3rd nonconflicting configuration for a different alternative: [1|1|[], 1|2|[], 8|3|[]]. This can come about from grammar: a : A | A | A B ; After matching input A, we reach the stop state for rule A, state 1. State 8 is the state right before B. Clearly alternatives 1 and 2 conflict and no amount of further lookahead will separate the two. However, alternative 3 will be able to continue and so we do not stop working on this state. In the previous example, we're concerned with states associated with the conflicting alternatives. Here alt 3 is not associated with the conflicting configs, but since we can continue looking for input reasonably, I don't declare the state done. We ignore a set of conflicting alts when we have an alternative that we still need to pursue. */ protected BitSet GetConflictingAltsOrUniqueAlt(ATNConfigSet configSet) { BitSet conflictingAlts; if (configSet.uniqueAlt != ATN.INVALID_ALT_NUMBER) { conflictingAlts = new BitSet(); conflictingAlts[configSet.uniqueAlt] = true; } else { conflictingAlts = configSet.conflictingAlts; } return conflictingAlts; }
public override void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { if (!REPORT_AMBIGUITIES) { return; } base.ReportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); }
protected SemanticContext[] GetPredsForAmbigAlts(BitSet ambigAlts, ATNConfigSet configSet, int nalts) { // REACH=[1|1|[]|0:0, 1|2|[]|0:1] /* altToPred starts as an array of all null contexts. The entry at index i * corresponds to alternative i. altToPred[i] may have one of three values: * 1. null: no ATNConfig c is found such that c.alt==i * 2. SemanticContext.NONE: At least one ATNConfig c exists such that * c.alt==i and c.semanticContext==SemanticContext.NONE. In other words, * alt i has at least one unpredicated config. * 3. Non-NONE Semantic Context: There exists at least one, and for all * ATNConfig c such that c.alt==i, c.semanticContext!=SemanticContext.NONE. * * From this, it is clear that NONE||anything==NONE. */ SemanticContext[] altToPred = new SemanticContext[nalts + 1]; foreach (ATNConfig c in configSet.configs) { if (ambigAlts[c.alt]) { altToPred[c.alt] = SemanticContext.OrOp(altToPred[c.alt], c.semanticContext); } } int nPredAlts = 0; for (int i = 1; i <= nalts; i++) { if (altToPred[i] == null) { altToPred[i] = SemanticContext.NONE; } else if (altToPred[i] != SemanticContext.NONE) { nPredAlts++; } } // // Optimize away p||p and p&&p TODO: optimize() was a no-op // for (int i = 0; i < altToPred.length; i++) { // altToPred[i] = altToPred[i].optimize(); // } // nonambig alts are null in altToPred if (nPredAlts == 0) altToPred = null; if (debug) Console.WriteLine("getPredsForAmbigAlts result " + Arrays.ToString(altToPred)); return altToPred; }
protected override DFAState CreateDFAState([NotNull] DFA dfa, [NotNull] ATNConfigSet configs) { return(new DFAState(emptyMap, dfa.EmptyContextEdgeMap, configs)); }
/** * This method is used to improve the localization of error messages by * choosing an alternative rather than throwing a * {@link NoViableAltException} in particular prediction scenarios where the * {@link #ERROR} state was reached during ATN simulation. * * <p> * The default implementation of this method uses the following * algorithm to identify an ATN configuration which successfully parsed the * decision entry rule. Choosing such an alternative ensures that the * {@link ParserRuleContext} returned by the calling rule will be complete * and valid, and the syntax error will be reported later at a more * localized location.</p> * * <ul> * <li>If a syntactically valid path or paths reach the end of the decision rule and * they are semantically valid if predicated, return the min associated alt.</li> * <li>Else, if a semantically invalid but syntactically valid path exist * or paths exist, return the minimum associated alt. * </li> * <li>Otherwise, return {@link ATN#INVALID_ALT_NUMBER}.</li> * </ul> * * <p> * In some scenarios, the algorithm described above could predict an * alternative which will result in a {@link FailedPredicateException} in * the parser. Specifically, this could occur if the <em>only</em> configuration * capable of successfully parsing to the end of the decision rule is * blocked by a semantic predicate. By choosing this alternative within * {@link #adaptivePredict} instead of throwing a * {@link NoViableAltException}, the resulting * {@link FailedPredicateException} in the parser will identify the specific * predicate which is preventing the parser from successfully parsing the * decision rule, which helps developers identify and correct logic errors * in semantic predicates. * </p> * * @param configs The ATN configurations which were valid immediately before * the {@link #ERROR} state was reached * @param outerContext The is the \gamma_0 initial parser context from the paper * or the parser stack at the instant before prediction commences. * * @return The value to return from {@link #adaptivePredict}, or * {@link ATN#INVALID_ALT_NUMBER} if a suitable alternative was not * identified and {@link #adaptivePredict} should report an error instead. */ protected int GetSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(ATNConfigSet configs, ParserRuleContext outerContext) { Pair<ATNConfigSet, ATNConfigSet> sets = SplitAccordingToSemanticValidity(configs, outerContext); ATNConfigSet semValidConfigs = sets.a; ATNConfigSet semInvalidConfigs = sets.b; int alt = getAltThatFinishedDecisionEntryRule(semValidConfigs); if (alt != ATN.INVALID_ALT_NUMBER) { // semantically/syntactically viable path exists return alt; } // Is there a syntactically valid path with a failed pred? if (semInvalidConfigs.Count > 0) { alt = getAltThatFinishedDecisionEntryRule(semInvalidConfigs); if (alt != ATN.INVALID_ALT_NUMBER) { // syntactically viable path exists return alt; } } return ATN.INVALID_ALT_NUMBER; }
protected NoViableAltException NoViableAlt(ITokenStream input, ParserRuleContext outerContext, ATNConfigSet configs, int startIndex) { return new NoViableAltException(parser, input, input.Get(startIndex), input.LT(1), configs, outerContext); }
public virtual void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { foreach (IAntlrErrorListener <IToken> listener in Delegates) { if (!(listener is IParserErrorListener)) { continue; } IParserErrorListener parserErrorListener = (IParserErrorListener)listener; parserErrorListener.ReportAmbiguity(recognizer, dfa, startIndex, stopIndex, exact, ambigAlts, configs); } }
/** * Return a configuration set containing only the configurations from * {@code configs} which are in a {@link RuleStopState}. If all * configurations in {@code configs} are already in a rule stop state, this * method simply returns {@code configs}. * * <p>When {@code lookToEndOfRule} is true, this method uses * {@link ATN#nextTokens} for each configuration in {@code configs} which is * not already in a rule stop state to see if a rule stop state is reachable * from the configuration via epsilon-only transitions.</p> * * @param configs the configuration set to update * @param lookToEndOfRule when true, this method checks for rule stop states * reachable by epsilon-only transitions from each configuration in * {@code configs}. * * @return {@code configs} if all configurations in {@code configs} are in a * rule stop state, otherwise return a new configuration set containing only * the configurations from {@code configs} which are in a rule stop state */ protected ATNConfigSet RemoveAllConfigsNotInRuleStopState(ATNConfigSet configSet, bool lookToEndOfRule) { if (PredictionMode.AllConfigsInRuleStopStates(configSet.configs)) { return configSet; } ATNConfigSet result = new ATNConfigSet(configSet.fullCtx); foreach (ATNConfig config in configSet.configs) { if (config.state is RuleStopState) { result.Add(config, mergeCache); continue; } if (lookToEndOfRule && config.state.OnlyHasEpsilonTransitions) { IntervalSet nextTokens = atn.NextTokens(config.state); if (nextTokens.Contains(TokenConstants.EPSILON)) { ATNState endOfRuleState = atn.ruleToStopState[config.state.ruleIndex]; result.Add(new ATNConfig(config, endOfRuleState), mergeCache); } } } return result; }
public DFAState(EmptyEdgeMap<DFAState> emptyEdges, EmptyEdgeMap<DFAState> emptyContextEdges, ATNConfigSet configs) { this.configs = configs; this.edges = emptyEdges; this.contextEdges = emptyContextEdges; }
public DFAState([NotNull] DFA dfa, [NotNull] ATNConfigSet configs) : this(dfa.EmptyEdgeMap, dfa.EmptyContextEdgeMap, configs) { }