protected virtual void ReportMessageAndThrowTemplateException(ITokenStream tokens, IToken templateToken, Parser parser, RecognitionException re) { if (re.Token.Type == TemplateLexer.EOF_TYPE) { string msg = "premature EOF"; ErrorManager.CompiletimeError(ErrorType.SYNTAX_ERROR, templateToken, re.Token, msg); } else if (re is NoViableAltException) { string msg = "'" + re.Token.Text + "' came as a complete surprise to me"; ErrorManager.CompiletimeError(ErrorType.SYNTAX_ERROR, templateToken, re.Token, msg); } else if (tokens.Index == 0) { // couldn't parse anything string msg = string.Format("this doesn't look like a template: \"{0}\"", tokens); ErrorManager.CompiletimeError(ErrorType.SYNTAX_ERROR, templateToken, re.Token, msg); } else if (tokens.LA(1) == TemplateLexer.LDELIM) { // couldn't parse expr string msg = "doesn't look like an expression"; ErrorManager.CompiletimeError(ErrorType.SYNTAX_ERROR, templateToken, re.Token, msg); } else { string msg = parser.GetErrorMessage(re, parser.TokenNames); ErrorManager.CompiletimeError(ErrorType.SYNTAX_ERROR, templateToken, re.Token, msg); } // we have reported the error, so just blast out throw new TemplateException(); }
public virtual int LA(int i) { if (initialStreamState) { ConsumeInitialHiddenTokens(); } dbg.LT(i, input.LT(i)); return(input.LA(i)); }
/// <summary> /// The default implementation of /// <see cref="IAntlrErrorStrategy.Sync(Parser)"/> /// makes sure /// that the current lookahead symbol is consistent with what were expecting /// at this point in the ATN. You can call this anytime but ANTLR only /// generates code to check before subrules/loops and each iteration. /// <p>Implements Jim Idle's magic sync mechanism in closures and optional /// subrules. E.g.,</p> /// <pre> /// a : sync ( stuff sync )* ; /// sync : {consume to what can follow sync} ; /// </pre> /// At the start of a sub rule upon error, /// <see cref="Sync(Parser)"/> /// performs single /// token deletion, if possible. If it can't do that, it bails on the current /// rule and uses the default error recovery, which consumes until the /// resynchronization set of the current rule. /// <p>If the sub rule is optional ( /// <c>(...)?</c> /// , /// <c>(...)*</c> /// , or block /// with an empty alternative), then the expected set includes what follows /// the subrule.</p> /// <p>During loop iteration, it consumes until it sees a token that can start a /// sub rule or what follows loop. Yes, that is pretty aggressive. We opt to /// stay in the loop as long as possible.</p> /// <p><strong>ORIGINS</strong></p> /// <p>Previous versions of ANTLR did a poor job of their recovery within loops. /// A single mismatch token or missing token would force the parser to bail /// out of the entire rules surrounding the loop. So, for rule</p> /// <pre> /// classDef : 'class' ID '{' member* '}' /// </pre> /// input with an extra token between members would force the parser to /// consume until it found the next class definition rather than the next /// member definition of the current class. /// <p>This functionality cost a little bit of effort because the parser has to /// compare token set at the start of the loop and at each iteration. If for /// some reason speed is suffering for you, you can turn off this /// functionality by simply overriding this method as a blank { }.</p> /// </summary> /// <exception cref="Antlr4.Runtime.RecognitionException"/> public virtual void Sync(Parser recognizer) { ATNState s = recognizer.Interpreter.atn.states[recognizer.State]; // System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName()); // If already recovering, don't try to sync if (InErrorRecoveryMode(recognizer)) { return; } ITokenStream tokens = ((ITokenStream)recognizer.InputStream); int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off if (recognizer.Atn.NextTokens(s).Contains(la) || la == TokenConstants.EOF) { return; } // Return but don't end recovery. only do that upon valid token match if (recognizer.IsExpectedToken(la)) { return; } switch (s.StateType) { case StateType.BlockStart: case StateType.StarBlockStart: case StateType.PlusBlockStart: case StateType.StarLoopEntry: { // report error and recover if possible if (SingleTokenDeletion(recognizer) != null) { return; } throw new InputMismatchException(recognizer); } case StateType.PlusLoopBack: case StateType.StarLoopBack: { // System.err.println("at loop back: "+s.getClass().getSimpleName()); ReportUnwantedToken(recognizer); IntervalSet expecting = recognizer.GetExpectedTokens(); IntervalSet whatFollowsLoopIterationOrRule = expecting.Or(GetErrorRecoverySet(recognizer)); ConsumeUntil(recognizer, whatFollowsLoopIterationOrRule); break; } default: { // do nothing if we can't identify the exact kind of ATN state break; } } }
public override void Recover(Parser recognizer, RecognitionException e) { for (; ;) { var p = stream.LA(1); if (p == -1) { break; } if (p != ReplParser.VWS) { stream.Consume(); } } base.Recover(recognizer, e); }
public TokenStreamVisualizerForm(ITokenStream tokenStream) { if (tokenStream == null) { throw new ArgumentNullException("tokenStream"); } InitializeComponent(); List <IToken> tokens = new List <IToken>(); int marker = tokenStream.Mark(); int currentPosition = tokenStream.Index; try { tokenStream.Seek(0); while (tokenStream.LA(1) != CharStreamConstants.EndOfFile) { tokenStream.Consume(); } for (int i = 0; i < tokenStream.Count; i++) { tokens.Add(tokenStream.Get(i)); } } finally { tokenStream.Rewind(marker); } this._tokenStream = tokenStream; this._tokens = tokens.ToArray(); if (tokenStream.TokenSource != null) { this._tokenNames = tokenStream.TokenSource.TokenNames; } this._tokenNames = this._tokenNames ?? new string[0]; UpdateTokenTypes(); UpdateHighlighting(); listBox1.BackColor = Color.Wheat; }
public TokenStreamVisualizerForm( ITokenStream tokenStream ) { if (tokenStream == null) throw new ArgumentNullException("tokenStream"); InitializeComponent(); List<IToken> tokens = new List<IToken>(); int marker = tokenStream.Mark(); int currentPosition = tokenStream.Index; try { tokenStream.Seek(0); while (tokenStream.LA(1) != CharStreamConstants.EndOfFile) tokenStream.Consume(); for (int i = 0; i < tokenStream.Count; i++) tokens.Add(tokenStream.Get(i)); } finally { tokenStream.Rewind(marker); } this._tokenStream = tokenStream; this._tokens = tokens.ToArray(); if (tokenStream.TokenSource != null) this._tokenNames = tokenStream.TokenSource.TokenNames; this._tokenNames = this._tokenNames ?? new string[0]; UpdateTokenTypes(); UpdateHighlighting(); listBox1.BackColor = Color.Wheat; }
public string GetLookaheadName(ITokenStream input) { return GetTokenName(input.LA(1)); }
// comes back with reach.UniqueAlt set to a valid alt protected int ExecATNWithFullContext(DFA dfa, DFAState D, // how far we got in SLL DFA before failing over ATNConfigSet s0, ITokenStream input, int startIndex, ParserRuleContext outerContext) { if (debug || debug_list_atn_decisions) { Console.WriteLine("execATNWithFullContext " + s0); } bool fullCtx = true; bool foundExactAmbig = false; ATNConfigSet reach = null; ATNConfigSet previous = s0; input.Seek(startIndex); int t = input.LA(1); int predictedAlt; while (true) { // while more work // Console.WriteLine("LL REACH "+GetLookaheadName(input)+ // " from configs.size="+previous.size()+ // " line "+input.LT(1)Line+":"+input.LT(1).Column); reach = ComputeReachSet(previous, t, fullCtx); if (reach == null) { // if any configs in previous dipped into outer context, that // means that input up to t actually finished entry rule // at least for LL decision. Full LL doesn't dip into outer // so don't need special case. // We will get an error no matter what so delay until after // decision; better error message. Also, no reachable target // ATN states in SLL implies LL will also get nowhere. // If conflict in states that dip out, choose min since we // will get error no matter what. NoViableAltException e = NoViableAlt(input, outerContext, previous, startIndex); input.Seek(startIndex); int alt = GetSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previous, outerContext); if (alt != ATN.INVALID_ALT_NUMBER) { return alt; } throw e; } ICollection<BitSet> altSubSets = PredictionMode.GetConflictingAltSubsets(reach.configs); if (debug) { Console.WriteLine("LL altSubSets=" + altSubSets + ", predict=" + PredictionMode.GetUniqueAlt(altSubSets) + ", ResolvesToJustOneViableAlt=" + PredictionMode.ResolvesToJustOneViableAlt(altSubSets)); } // Console.WriteLine("altSubSets: "+altSubSets); // System.err.println("reach="+reach+", "+reach.conflictingAlts); reach.uniqueAlt = GetUniqueAlt(reach); // unique prediction? if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { predictedAlt = reach.uniqueAlt; break; } if (mode != PredictionMode.LL_EXACT_AMBIG_DETECTION) { predictedAlt = PredictionMode.ResolvesToJustOneViableAlt(altSubSets); if (predictedAlt != ATN.INVALID_ALT_NUMBER) { break; } } else { // In exact ambiguity mode, we never try to terminate early. // Just keeps scarfing until we know what the conflict is if (PredictionMode.AllSubsetsConflict(altSubSets) && PredictionMode.AllSubsetsEqual(altSubSets)) { foundExactAmbig = true; predictedAlt = PredictionMode.GetSingleViableAlt(altSubSets); break; } // else there are multiple non-conflicting subsets or // we're not sure what the ambiguity is yet. // So, keep going. } previous = reach; if (t != IntStreamConstants.EOF) { input.Consume(); t = input.LA(1); } } // If the configuration set uniquely predicts an alternative, // without conflict, then we know that it's a full LL decision // not SLL. if (reach.uniqueAlt != ATN.INVALID_ALT_NUMBER) { ReportContextSensitivity(dfa, predictedAlt, reach, startIndex, input.Index); return predictedAlt; } // We do not check predicates here because we have checked them // on-the-fly when doing full context prediction. /* In non-exact ambiguity detection mode, we might actually be able to detect an exact ambiguity, but I'm not going to spend the cycles needed to check. We only emit ambiguity warnings in exact ambiguity mode. For example, we might know that we have conflicting configurations. But, that does not mean that there is no way forward without a conflict. It's possible to have nonconflicting alt subsets as in: LL altSubSets=[{1, 2}, {1, 2}, {1}, {1, 2}] from [(17,1,[5 $]), (13,1,[5 10 $]), (21,1,[5 10 $]), (11,1,[$]), (13,2,[5 10 $]), (21,2,[5 10 $]), (11,2,[$])] In this case, (17,1,[5 $]) indicates there is some next sequence that would resolve this without conflict to alternative 1. Any other viable next sequence, however, is associated with a conflict. We stop looking for input because no amount of further lookahead will alter the fact that we should predict alternative 1. We just can't say for sure that there is an ambiguity without looking further. */ ReportAmbiguity(dfa, D, startIndex, input.Index, foundExactAmbig, reach.GetAlts(), reach); return predictedAlt; }
/** Performs ATN simulation to compute a predicted alternative based * upon the remaining input, but also updates the DFA cache to avoid * having to traverse the ATN again for the same input sequence. There are some key conditions we're looking for after computing a new set of ATN configs (proposed DFA state): * if the set is empty, there is no viable alternative for current symbol * does the state uniquely predict an alternative? * does the state have a conflict that would prevent us from putting it on the work list? We also have some key operations to do: * add an edge from previous DFA state to potentially new DFA state, D, upon current symbol but only if adding to work list, which means in all cases except no viable alternative (and possibly non-greedy decisions?) * collecting predicates and adding semantic context to DFA accept states * adding rule context to context-sensitive DFA accept states * consuming an input symbol * reporting a conflict * reporting an ambiguity * reporting a context sensitivity * reporting insufficient predicates cover these cases: dead end single alt single alt + preds conflict conflict + preds */ protected int ExecATN(DFA dfa, DFAState s0, ITokenStream input, int startIndex, ParserRuleContext outerContext) { if (debug || debug_list_atn_decisions) { Console.WriteLine("execATN decision " + dfa.decision + " exec LA(1)==" + GetLookaheadName(input) + " line " + input.LT(1).Line + ":" + input.LT(1).Column); } DFAState previousD = s0; if (debug) Console.WriteLine("s0 = " + s0); int t = input.LA(1); while (true) { // while more work DFAState D = GetExistingTargetState(previousD, t); if (D == null) { D = ComputeTargetState(dfa, previousD, t); } if (D == ERROR) { // if any configs in previous dipped into outer context, that // means that input up to t actually finished entry rule // at least for SLL decision. Full LL doesn't dip into outer // so don't need special case. // We will get an error no matter what so delay until after // decision; better error message. Also, no reachable target // ATN states in SLL implies LL will also get nowhere. // If conflict in states that dip out, choose min since we // will get error no matter what. NoViableAltException e = NoViableAlt(input, outerContext, previousD.configSet, startIndex); input.Seek(startIndex); int alt = GetSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(previousD.configSet, outerContext); if (alt != ATN.INVALID_ALT_NUMBER) { return alt; } throw e; } if (D.requiresFullContext && mode != PredictionMode.SLL) { // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) BitSet conflictingAlts = D.configSet.conflictingAlts; if (D.predicates != null) { if (debug) Console.WriteLine("DFA state has preds in DFA sim LL failover"); int conflictIndex = input.Index; if (conflictIndex != startIndex) { input.Seek(startIndex); } conflictingAlts = EvalSemanticContext(D.predicates, outerContext, true); if (conflictingAlts.Cardinality() == 1) { if (debug) Console.WriteLine("Full LL avoided"); return conflictingAlts.NextSetBit(0); } if (conflictIndex != startIndex) { // restore the index so reporting the fallback to full // context occurs with the index at the correct spot input.Seek(conflictIndex); } } if (dfa_debug) Console.WriteLine("ctx sensitive state " + outerContext + " in " + D); bool fullCtx = true; ATNConfigSet s0_closure = ComputeStartState(dfa.atnStartState, outerContext, fullCtx); ReportAttemptingFullContext(dfa, conflictingAlts, D.configSet, startIndex, input.Index); int alt = ExecATNWithFullContext(dfa, D, s0_closure, input, startIndex, outerContext); return alt; } if (D.isAcceptState) { if (D.predicates == null) { return D.prediction; } int stopIndex = input.Index; input.Seek(startIndex); BitSet alts = EvalSemanticContext(D.predicates, outerContext, true); switch (alts.Cardinality()) { case 0: throw NoViableAlt(input, outerContext, D.configSet, startIndex); case 1: return alts.NextSetBit(0); default: // report ambiguity after predicate evaluation to make sure the correct // set of ambig alts is reported. ReportAmbiguity(dfa, D, startIndex, stopIndex, false, alts, D.configSet); return alts.NextSetBit(0); } } previousD = D; if (t != IntStreamConstants.EOF) { input.Consume(); t = input.LA(1); } } }
/// <summary> /// The default implementation of /// <see cref="IAntlrErrorStrategy.Sync(Parser)"/> /// makes sure /// that the current lookahead symbol is consistent with what were expecting /// at this point in the ATN. You can call this anytime but ANTLR only /// generates code to check before subrules/loops and each iteration. /// <p>Implements Jim Idle's magic sync mechanism in closures and optional /// subrules. E.g.,</p> /// <pre> /// a : sync ( stuff sync )* ; /// sync : {consume to what can follow sync} ; /// </pre> /// At the start of a sub rule upon error, /// <see cref="Sync(Parser)"/> /// performs single /// token deletion, if possible. If it can't do that, it bails on the current /// rule and uses the default error recovery, which consumes until the /// resynchronization set of the current rule. /// <p>If the sub rule is optional ( /// <c>(...)?</c> /// , /// <c>(...)*</c> /// , or block /// with an empty alternative), then the expected set includes what follows /// the subrule.</p> /// <p>During loop iteration, it consumes until it sees a token that can start a /// sub rule or what follows loop. Yes, that is pretty aggressive. We opt to /// stay in the loop as long as possible.</p> /// <p><strong>ORIGINS</strong></p> /// <p>Previous versions of ANTLR did a poor job of their recovery within loops. /// A single mismatch token or missing token would force the parser to bail /// out of the entire rules surrounding the loop. So, for rule</p> /// <pre> /// classDef : 'class' ID '{' member* '}' /// </pre> /// input with an extra token between members would force the parser to /// consume until it found the next class definition rather than the next /// member definition of the current class. /// <p>This functionality cost a little bit of effort because the parser has to /// compare token set at the start of the loop and at each iteration. If for /// some reason speed is suffering for you, you can turn off this /// functionality by simply overriding this method as a blank { }.</p> /// </summary> /// <exception cref="Antlr4.Runtime.RecognitionException"/> public virtual void Sync(Parser recognizer) { ATNState s = recognizer.Interpreter.atn.states[recognizer.State]; // System.err.println("sync @ "+s.stateNumber+"="+s.getClass().getSimpleName()); // If already recovering, don't try to sync if (InErrorRecoveryMode(recognizer)) { return; } ITokenStream tokens = ((ITokenStream)recognizer.InputStream); int la = tokens.LA(1); // try cheaper subset first; might get lucky. seems to shave a wee bit off var nextTokens = recognizer.Atn.NextTokens(s); if (nextTokens.Contains(la)) { nextTokensContext = null; nextTokensState = ATNState.InvalidStateNumber; return; } if (nextTokens.Contains(TokenConstants.EPSILON)) { if (nextTokensContext == null) { // It's possible the next token won't match; information tracked // by sync is restricted for performance. nextTokensContext = recognizer.Context; nextTokensState = recognizer.State; } return; } switch (s.StateType) { case StateType.BlockStart: case StateType.StarBlockStart: case StateType.PlusBlockStart: case StateType.StarLoopEntry: { // report error and recover if possible if (SingleTokenDeletion(recognizer) != null) { return; } throw new InputMismatchException(recognizer); } case StateType.PlusLoopBack: case StateType.StarLoopBack: { // System.err.println("at loop back: "+s.getClass().getSimpleName()); ReportUnwantedToken(recognizer); IntervalSet expecting = recognizer.GetExpectedTokens(); IntervalSet whatFollowsLoopIterationOrRule = expecting.Or(GetErrorRecoverySet(recognizer)); ConsumeUntil(recognizer, whatFollowsLoopIterationOrRule); break; } default: { // do nothing if we can't identify the exact kind of ATN state break; } } }
public bool next(char expect) { return(_input.LA(1) == expect); }
private void ProcessToken() { var tok = m_input.LT(1); int la1 = m_input.LA(1), la2 = m_input.LA(2); if (la1 == m_tokens.SELECT) { m_queryType = QueryType.SELECT; } if (la1 == m_tokens.UPDATE) { m_queryType = QueryType.UPDATE; } if (la1 == m_tokens.DELETE) { m_queryType = QueryType.DELETE; } if (la1 == m_tokens.INSERT) { m_queryType = QueryType.INSERT; } if (la1 == m_tokens.T_STRING) { var oldctx = m_context; SetPositionBegin(tok); SetContext(CodeContext.String); SetPositionEnd(tok); SetContext(oldctx); m_input.Consume(); return; } if (la1 == m_tokens.SELECT || la1 == m_tokens.WHERE || la1 == m_tokens.ON ) { SetPositionEnd(tok); SetContext(CodeContext.Column); m_input.Consume(); return; } if ((la1 == m_tokens.LPAREN && m_queryType == QueryType.INSERT) || la1 == m_tokens.SET ) { SetPositionEnd(tok); SetContext(CodeContext.ColumnWithoutQualifier); m_input.Consume(); return; } if (la1 == m_tokens.ORDER && la2 == m_tokens.BY || la1 == m_tokens.GROUP && la2 == m_tokens.BY) { SetPositionEnd(m_input.LT(2)); m_input.Consume(); m_input.Consume(); SetContext(CodeContext.Column); return; } if (la1 == m_tokens.FROM || la1 == m_tokens.JOIN || la1 == m_tokens.UPDATE || la1 == m_tokens.DELETE || la1 == m_tokens.INSERT ) { SetPositionEnd(tok); SetContext(CodeContext.Table); m_input.Consume(); return; } if (m_context == CodeContext.Table && m_tokens.IsIdent(la1)) { var name = new DepsName(); name.Components.Add(m_dialect.UnquoteName(tok.Text)); m_input.Consume(); while (m_input.LA(1) == m_tokens.DOT && m_tokens.IsIdent(m_input.LA(2))) { name.Components.Add(m_dialect.UnquoteName(m_input.LT(2).Text)); m_input.Consume(); m_input.Consume(); } var titem = new TableItem { Name = name }; if (m_tokens.IsIdent(m_input.LA(1))) { titem.Alias = m_dialect.UnquoteName(m_input.LT(1).Text); m_input.Consume(); } UsedTables.Add(titem); return; } // default token handling m_input.Consume(); SetPositionEnd(tok); }
protected void ThrowTemplateException(ITokenStream tokens, TemplateParser parser, RecognitionException re) { string msg = parser.GetErrorMessage(re, parser.TokenNames); //String hdr = parser.getErrorHeader(re); if (re.Token.Type == TemplateLexer.EOF_TYPE) { throw new TemplateException("premature EOF", re); } else if (re is NoViableAltException) { throw new TemplateException("'" + re.Token.Text + "' came as a complete surprise to me", re); } else if (tokens.Index == 0) { // couldn't parse anything throw new TemplateException("this doesn't look like a template: \"" + tokens + "\"", re); } else if (tokens.LA(1) == TemplateLexer.LDELIM) { // couldn't parse anything throw new TemplateException("doesn't look like an expression", re); } else { throw new TemplateException(msg, re); } }