/** <summary>Attempt to recover from a single missing or extra token.</summary> * * EXTRA TOKEN * * LA(1) is not what we are looking for. If LA(2) has the right token, * however, then assume LA(1) is some extra spurious token. Delete it * and LA(2) as if we were doing a normal match(), which advances the * input. * * MISSING TOKEN * * If current token is consistent with what could come after * ttype then it is ok to "insert" the missing token, else throw * exception For example, Input "i=(3;" is clearly missing the * ')'. When the parser returns from the nested call to expr, it * will have call chain: * * stat -> expr -> atom * * and it will be trying to match the ')' at this point in the * derivation: * * => ID '=' '(' INT ')' ('+' atom)* ';' * ^ * match() will see that ';' doesn't match ')' and report a * mismatched token error. To recover, it sees that LA(1)==';' * is in the set of tokens that can follow the ')' token * reference in rule atom. It can assume that you forgot the ')'. */ protected virtual object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; // if next token is what we are looking for then "delete" this token if (MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input, TokenNames); /* * System.err.println("recoverFromMismatchedToken deleting "+ * ((TokenStream)input).LT(1)+ * " since "+((TokenStream)input).LT(2)+" is what we want"); */ BeginResync(); input.Consume(); // simply delete extra token EndResync(); ReportError(e); // report after consuming so AW sees the token in the exception // we want to return the token we're actually matching object matchedSymbol = GetCurrentInputSymbol(input); input.Consume(); // move past ttype token as if all were ok return(matchedSymbol); } // can't recover with single token deletion, try insertion if (MismatchIsMissingToken(input, follow)) { object inserted = GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, inserted); ReportError(e); // report after inserting so AW sees the token in the exception return(inserted); } // even that didn't work; must throw the exception e = new MismatchedTokenException(ttype, input, TokenNames); throw e; }
protected virtual object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; if (this.MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input, this.TokenNames); this.BeginResync(); input.Consume(); this.EndResync(); this.ReportError(e); object currentInputSymbol = this.GetCurrentInputSymbol(input); input.Consume(); return(currentInputSymbol); } if (this.MismatchIsMissingToken(input, follow)) { object missingSymbol = this.GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, missingSymbol); this.ReportError(e); return(missingSymbol); } e = new MismatchedTokenException(ttype, input, this.TokenNames); throw e; }
public virtual void ConsumeUntil(IIntStream input, BitSet set) { for (int el = input.LA(1); el != -1 && !set.Member(el); el = input.LA(1)) { input.Consume(); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { for (int index = input.LA(1); index != -1 && index != tokenType; index = input.LA(1)) { input.Consume(); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int ttype = input.LA(1); while (ttype != Token.EOF && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
/// <summary>Consume tokens until one matches the given token set </summary> public virtual void ConsumeUntil(IIntStream input, BitSet set) { int ttype = input.LA(1); while (ttype != Token.EOF && !set.Member(ttype)) { input.Consume(); ttype = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, BitSet set) { int num = input.LA(1); while (num != -1 && !set.Member(num)) { input.Consume(); num = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int num = input.LA(1); while (num != -1 && num != tokenType) { input.Consume(); num = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { //System.out.println("consumeUntil "+tokenType); int ttype = input.LA(1); while (ttype != TokenTypes.EndOfFile && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
/** <summary>Consume tokens until one matches the given token set</summary> */ public virtual void ConsumeUntil(IIntStream input, BitSet set) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = input.LA(1); while (ttype != TokenTypes.EndOfFile && !set.Member(ttype)) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); input.Consume(); ttype = input.LA(1); } }
public virtual void Recover(IIntStream input, RecognitionException re) { if (this.state.lastErrorIndex == input.Index) { input.Consume(); } this.state.lastErrorIndex = input.Index; BitSet set = this.ComputeErrorRecoverySet(); this.BeginResync(); this.ConsumeUntil(input, set); this.EndResync(); }
public RecognitionException(string message, IIntStream input, int k, Exception innerException) : base(message, innerException) { this._input = input; this._k = k; if (input != null) { this._index = input.Index + k - 1; if (input is ITokenStream) { this._token = ((ITokenStream)input).LT(k); this._line = _token.Line; this._charPositionInLine = _token.CharPositionInLine; } ITreeNodeStream tns = input as ITreeNodeStream; if (tns != null) { ExtractInformationFromTreeNodeStream(tns, k); } else { ICharStream charStream = input as ICharStream; if (charStream != null) { int mark = input.Mark(); try { for (int i = 0; i < k - 1; i++) { input.Consume(); } this._c = input.LA(1); this._line = ((ICharStream)input).Line; this._charPositionInLine = ((ICharStream)input).CharPositionInLine; } finally { input.Rewind(mark); } } else { this._c = input.LA(k); } } } }
/** <summary> * Recover from an error found on the input stream. This is * for NoViableAlt and mismatched symbol exceptions. If you enable * single token insertion and deletion, this will usually not * handle mismatched symbol exceptions but there could be a mismatched * token that the match() routine could not recover from. * </summary> */ public virtual void Recover(IIntStream input, RecognitionException re) { if (state.lastErrorIndex == input.Index) { // uh oh, another error at same token index; must be a case // where LT(1) is in the recovery token set so nothing is // consumed; consume a single token so at least to prevent // an infinite loop; this is a failsafe. input.Consume(); } state.lastErrorIndex = input.Index; BitSet followSet = ComputeErrorRecoverySet(); BeginResync(); ConsumeUntil(input, followSet); EndResync(); }
/** Given an input stream, return the unique alternative predicted by * matching the input. Upon error, return NFA.INVALID_ALT_NUMBER * The first symbol of lookahead is presumed to be primed; that is, * input.lookahead(1) must point at the input symbol you want to start * predicting with. */ public int Predict(DFA dfa) { DFAState s = dfa.StartState; int c = input.LA(1); Transition eotTransition = null; dfaLoop: while (!s.IsAcceptState) { //Console.Out.WriteLine( "DFA.predict(" + s.stateNumber + ", " + dfa.nfa.Grammar.getTokenDisplayName( c ) + ")" ); // for each edge of s, look for intersection with current char for (int i = 0; i < s.NumberOfTransitions; i++) { Transition t = s.GetTransition(i); // special case: EOT matches any char if (t.Label.Matches(c)) { // take transition i s = (DFAState)t.Target; input.Consume(); c = input.LA(1); goto dfaLoop; } if (t.Label.Atom == Label.EOT) { eotTransition = t; } } if (eotTransition != null) { s = (DFAState)eotTransition.Target; goto dfaLoop; } /* * ErrorManager.error(ErrorManager.MSG_NO_VIABLE_DFA_ALT, * s, * dfa.nfa.Grammar.getTokenName(c)); */ return(NFA.INVALID_ALT_NUMBER); } // woohoo! We know which alt to predict // nothing emanates from a stop state; must terminate anyway //Console.Out.WriteLine( "DFA stop state " + s.stateNumber + " predicts " + s.getUniquelyPredictedAlt() ); return(s.GetUniquelyPredictedAlt()); }
public virtual object Match(IIntStream input, int ttype, BitSet follow) { object currentInputSymbol = this.GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); this.state.errorRecovery = false; this.state.failed = false; return(currentInputSymbol); } if (this.state.backtracking > 0) { this.state.failed = true; return(currentInputSymbol); } return(this.RecoverFromMismatchedToken(input, ttype, follow)); }
public override object Match(IIntStream input, int ttype, BitSet follow) { object currentInputSymbol = this.GetCurrentInputSymbol(input); DumpSymbol(currentInputSymbol); if (input.LA(1) == ttype) { input.Consume(); this.state.errorRecovery = false; this.state.failed = false; return currentInputSymbol; } else { if (this.state.backtracking <= 0) return this.RecoverFromMismatchedToken(input, ttype, follow); this.state.failed = true; return currentInputSymbol; } }
/** <summary> * Match current input symbol against ttype. Attempt * single token insertion or deletion error recovery. If * that fails, throw MismatchedTokenException. * </summary> * * <remarks> * To turn off single token insertion or deletion error * recovery, override recoverFromMismatchedToken() and have it * throw an exception. See TreeParser.recoverFromMismatchedToken(). * This way any error in a rule will cause an exception and * immediate exit from rule. Rule would recover by resynchronizing * to the set of symbols that can follow rule ref. * </remarks> */ public virtual object Match(IIntStream input, int ttype, BitSet follow) { //System.out.println("match "+((TokenStream)input).LT(1)); object matchedSymbol = GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return(matchedSymbol); } if (state.backtracking > 0) { state.failed = true; return(matchedSymbol); } matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return(matchedSymbol); }
public virtual int Predict(IIntStream input) { if (this.debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + (object)this.decisionNumber); } int marker = input.Mark(); int s1 = 0; try { char ch; while (true) { if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " state " + (object)s1 + " LA(1)=" + (object)(char)input.LA(1) + "(" + (object)input.LA(1) + "), index=" + (object)input.Index); } int s2 = (int)this.special[s1]; if (s2 >= 0) { if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " state " + (object)s1 + " is special state " + (object)s2); } s1 = this.SpecialStateTransition(this, s2, input); if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " returns from special state " + (object)s2 + " to " + (object)s1); } if (s1 != -1) { input.Consume(); } else { break; } } else if (this.accept[s1] < (short)1) { ch = (char)input.LA(1); if ((int)ch >= (int)this.min[s1] && (int)ch <= (int)this.max[s1]) { int num = (int)this.transition[s1][(int)ch - (int)this.min[s1]]; if (num < 0) { if (this.eot[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } s1 = (int)this.eot[s1]; input.Consume(); } else { goto label_23; } } else { s1 = num; input.Consume(); } } else if (this.eot[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } s1 = (int)this.eot[s1]; input.Consume(); } else { goto label_29; } } else { goto label_14; } } this.NoViableAlt(s1, input); return(0); label_14: if (this.debug) { Console.Error.WriteLine("accept; predict " + (object)this.accept[s1] + " from state " + (object)s1); } return((int)this.accept[s1]); label_23: this.NoViableAlt(s1, input); return(0); label_29: if (ch == char.MaxValue && this.eof[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("accept via EOF; predict " + (object)this.accept[(int)this.eof[s1]] + " from " + (object)this.eof[s1]); } return((int)this.accept[(int)this.eof[s1]]); } if (this.debug) { Console.Error.WriteLine("min[" + (object)s1 + "]=" + (object)this.min[s1]); Console.Error.WriteLine("max[" + (object)s1 + "]=" + (object)this.max[s1]); Console.Error.WriteLine("eot[" + (object)s1 + "]=" + (object)this.eot[s1]); Console.Error.WriteLine("eof[" + (object)s1 + "]=" + (object)this.eof[s1]); for (int index = 0; index < this.transition[s1].Length; ++index) { Console.Error.Write(((int)this.transition[s1][index]).ToString() + " "); } Console.Error.WriteLine(); } this.NoViableAlt(s1, input); return(0); } finally { input.Rewind(marker); } }
/** Fill a list of all NFA states visited during the parse */ protected virtual void ParseEngine( String startRule, NFAState start, NFAState stop, IIntStream input, Stack<object> ruleInvocationStack, IDebugEventListener actions, IList visitedStates ) { NFAState s = start; if ( actions != null ) { actions.EnterRule( s.nfa.grammar.FileName, start.enclosingRule.Name ); } int t = input.LA( 1 ); while ( s != stop ) { if ( visitedStates != null ) { visitedStates.Add( s ); } //Console.Out.WriteLine( "parse state " + s.stateNumber + " input=" + s.nfa.grammar.getTokenDisplayName( t ) ); // CASE 1: decision state if ( s.DecisionNumber > 0 && s.nfa.grammar.GetNumberOfAltsForDecisionNFA( s ) > 1 ) { // decision point, must predict and jump to alt DFA dfa = s.nfa.grammar.GetLookaheadDFA( s.DecisionNumber ); //if ( s.nfa.grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "decision: " + // dfa.getNFADecisionStartState().Description + // " input=" + s.nfa.grammar.getTokenDisplayName( t ) ); //} int m = input.Mark(); int predictedAlt = Predict( dfa ); if ( predictedAlt == NFA.INVALID_ALT_NUMBER ) { String description = dfa.NFADecisionStartState.Description; NoViableAltException nvae = new NoViableAltException( description, dfa.DecisionNumber, s.stateNumber, input ); if ( actions != null ) { actions.RecognitionException( nvae ); } input.Consume(); // recover throw nvae; } input.Rewind( m ); int parseAlt = s.TranslateDisplayAltToWalkAlt( predictedAlt ); //if ( s.nfa.grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "predicted alt " + predictedAlt + ", parseAlt " + parseAlt ); //} NFAState alt; if ( parseAlt > s.nfa.grammar.GetNumberOfAltsForDecisionNFA( s ) ) { // implied branch of loop etc... alt = s.nfa.grammar.nfa.GetState( s.endOfBlockStateNumber ); } else { alt = s.nfa.grammar.GetNFAStateForAltOfDecision( s, parseAlt ); } s = (NFAState)alt.transition[0].target; continue; } // CASE 2: finished matching a rule if ( s.IsAcceptState ) { // end of rule node if ( actions != null ) { actions.ExitRule( s.nfa.grammar.FileName, s.enclosingRule.Name ); } if ( ruleInvocationStack.Count == 0 ) { // done parsing. Hit the start state. //Console.Out.WriteLine( "stack empty in stop state for " + s.enclosingRule ); break; } // pop invoking state off the stack to know where to return to NFAState invokingState = (NFAState)ruleInvocationStack.Pop(); RuleClosureTransition invokingTransition = (RuleClosureTransition)invokingState.transition[0]; // move to node after state that invoked this rule s = invokingTransition.followState; continue; } Transition trans = s.transition[0]; Label label = trans.label; if ( label.IsSemanticPredicate ) { FailedPredicateException fpe = new FailedPredicateException( input, s.enclosingRule.Name, "can't deal with predicates yet" ); if ( actions != null ) { actions.RecognitionException( fpe ); } } // CASE 3: epsilon transition if ( label.IsEpsilon ) { // CASE 3a: rule invocation state if ( trans is RuleClosureTransition ) { ruleInvocationStack.Push( s ); s = (NFAState)trans.target; //Console.Out.WriteLine( "call " + s.enclosingRule.name + " from " + s.nfa.grammar.getFileName() ); if ( actions != null ) { actions.EnterRule( s.nfa.grammar.FileName, s.enclosingRule.Name ); } // could be jumping to new grammar, make sure DFA created if ( !s.nfa.grammar.AllDecisionDFAHaveBeenCreated ) { s.nfa.grammar.CreateLookaheadDFAs(); } } // CASE 3b: plain old epsilon transition, just move else { s = (NFAState)trans.target; } } // CASE 4: match label on transition else if ( label.Matches( t ) ) { if ( actions != null ) { if ( s.nfa.grammar.type == GrammarType.Parser || s.nfa.grammar.type == GrammarType.Combined ) { actions.ConsumeToken( ( (ITokenStream)input ).LT( 1 ) ); } } s = (NFAState)s.transition[0].target; input.Consume(); t = input.LA( 1 ); } // CASE 5: error condition; label is inconsistent with input else { if ( label.IsAtom ) { MismatchedTokenException mte = new MismatchedTokenException( label.Atom, input ); if ( actions != null ) { actions.RecognitionException( mte ); } input.Consume(); // recover throw mte; } else if ( label.IsSet ) { MismatchedSetException mse = new MismatchedSetException( ( (IntervalSet)label.Set ).ToRuntimeBitSet(), input ); if ( actions != null ) { actions.RecognitionException( mse ); } input.Consume(); // recover throw mse; } else if ( label.IsSemanticPredicate ) { FailedPredicateException fpe = new FailedPredicateException( input, s.enclosingRule.Name, label.SemanticContext.ToString() ); if ( actions != null ) { actions.RecognitionException( fpe ); } input.Consume(); // recover throw fpe; } else { throw new RecognitionException( input ); // unknown error } } } //Console.Out.WriteLine( "hit stop state for " + stop.enclosingRule ); if ( actions != null ) { actions.ExitRule( s.nfa.grammar.FileName, stop.enclosingRule.Name ); } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict( IIntStream input ) { if (input == null) throw new ArgumentNullException("input"); DfaDebugMessage("Enter DFA.Predict for decision {0}", decisionNumber); int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { DfaDebugMessage("DFA {0} state {1} LA(1)={2}({3}), index={4}", decisionNumber, s, (char)input.LA(1), input.LA(1), input.Index); int specialState = special[s]; if ( specialState >= 0 ) { DfaDebugMessage("DFA {0} state {1} is special state {2}", decisionNumber, s, specialState); s = SpecialStateTransition( this, specialState, input ); DfaDebugMessage("DFA {0} returns from special state {1} to {2}", decisionNumber, specialState, s); if ( s == -1 ) { NoViableAlt( s, input ); return 0; } input.Consume(); continue; } if ( accept[s] >= 1 ) { DfaDebugMessage("accept; predict {0} from state {1}", accept[s], s); return accept[s]; } // look for a normal char transition char c = (char)input.LA( 1 ); // -1 == \uFFFF, all tokens fit in 65000 space if ( c >= min[s] && c <= max[s] ) { int snext = transition[s][c - min[s]]; // move to next state if ( snext < 0 ) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if ( eot[s] >= 0 ) { // EOT Transition to accept state? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt( s, input ); return 0; } s = snext; input.Consume(); continue; } if ( eot[s] >= 0 ) { // EOT Transition? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); continue; } if ( c == unchecked( (char)TokenTypes.EndOfFile ) && eof[s] >= 0 ) { // EOF Transition to accept state? DfaDebugMessage("accept via EOF; predict {0} from {1}", accept[eof[s]], eof[s]); return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol DfaDebugInvalidSymbol(s); NoViableAlt( s, input ); return 0; } } finally { input.Rewind( mark ); } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict(IIntStream input) { if (input == null) { throw new ArgumentNullException("input"); } DfaDebugMessage("Enter DFA.Predict for decision {0}", decisionNumber); int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { DfaDebugMessage("DFA {0} state {1} LA(1)={2}({3}), index={4}", decisionNumber, s, (char)input.LA(1), input.LA(1), input.Index); int specialState = special[s]; if (specialState >= 0) { DfaDebugMessage("DFA {0} state {1} is special state {2}", decisionNumber, s, specialState); s = SpecialStateTransition(this, specialState, input); DfaDebugMessage("DFA {0} returns from special state {1} to {2}", decisionNumber, specialState, s); if (s == -1) { NoViableAlt(s, input); return(0); } input.Consume(); continue; } if (accept[s] >= 1) { DfaDebugMessage("accept; predict {0} from state {1}", accept[s], s); return(accept[s]); } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if (c >= min[s] && c <= max[s]) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) { // EOT Transition to accept state? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return(0); } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); continue; } if (c == unchecked ((char)TokenTypes.EndOfFile) && eof[s] >= 0) { // EOF Transition to accept state? DfaDebugMessage("accept via EOF; predict {0} from {1}", accept[eof[s]], eof[s]); return(accept[eof[s]]); } // not in range and not EOF/EOT, must be invalid symbol DfaDebugInvalidSymbol(s); NoViableAlt(s, input); return(0); } } finally { input.Rewind(mark); } }
/** <summary>Match the wildcard: in a symbol</summary> */ public virtual void MatchAny(IIntStream input) { state.errorRecovery = false; state.failed = false; input.Consume(); }
/// <summary> /// Match current input symbol against ttype. Attempt /// single token insertion or deletion error recovery. If /// that fails, throw MismatchedTokenException. /// </summary> /// <remarks> /// To turn off single token insertion or deletion error /// recovery, override RecoverFromMismatchedToken() and have it call /// pthrow an exception. See TreeParser.RecoverFromMismatchedToken(). /// This way any error in a rule will cause an exception and /// immediate exit from rule. Rule would recover by resynchronizing /// to the set of symbols that can follow rule ref. /// </remarks> public virtual object Match(IIntStream input, int ttype, BitSet follow) { object matchedSymbol = GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return matchedSymbol; } if (state.backtracking > 0) { state.failed = true; return matchedSymbol; } matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return matchedSymbol; }
public virtual int Predict(IIntStream input) { int num6; if (this.debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + this.decisionNumber); } int marker = input.Mark(); int index = 0; try { Label_0030: if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " state ", index, " LA(1)=", (char) input.LA(1), "(", input.LA(1), "), index=", input.Index })); } int s = this.special[index]; if (s >= 0) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " state ", index, " is special state ", s })); } index = this.SpecialStateTransition(this, s, input); if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " returns from special state ", s, " to ", index })); } if (index == -1) { this.NoViableAlt(index, input); return 0; } input.Consume(); goto Label_0030; } if (this.accept[index] >= 1) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "accept; predict ", this.accept[index], " from state ", index })); } return this.accept[index]; } char ch = (char) input.LA(1); if ((ch >= this.min[index]) && (ch <= this.max[index])) { int num4 = this.transition[index][ch - this.min[index]]; if (num4 < 0) { if (this.eot[index] < 0) { this.NoViableAlt(index, input); return 0; } if (this.debug) { Console.Error.WriteLine("EOT transition"); } index = this.eot[index]; input.Consume(); } else { index = num4; input.Consume(); } goto Label_0030; } if (this.eot[index] >= 0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } index = this.eot[index]; input.Consume(); goto Label_0030; } if ((ch == 0xffff) && (this.eof[index] >= 0)) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "accept via EOF; predict ", this.accept[this.eof[index]], " from ", this.eof[index] })); } return this.accept[this.eof[index]]; } if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "min[", index, "]=", this.min[index] })); Console.Error.WriteLine(string.Concat(new object[] { "max[", index, "]=", this.max[index] })); Console.Error.WriteLine(string.Concat(new object[] { "eot[", index, "]=", this.eot[index] })); Console.Error.WriteLine(string.Concat(new object[] { "eof[", index, "]=", this.eof[index] })); for (int i = 0; i < this.transition[index].Length; i++) { Console.Error.Write(this.transition[index][i] + " "); } Console.Error.WriteLine(); } this.NoViableAlt(index, input); num6 = 0; } finally { input.Rewind(marker); } return num6; }
protected virtual object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; if (this.MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input, this.TokenNames); this.BeginResync(); input.Consume(); this.EndResync(); this.ReportError(e); object currentInputSymbol = this.GetCurrentInputSymbol(input); input.Consume(); return currentInputSymbol; } if (this.MismatchIsMissingToken(input, follow)) { object inserted = this.GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, inserted); this.ReportError(e); return inserted; } e = new MismatchedTokenException(ttype, input, this.TokenNames); throw e; }
public virtual void ConsumeUntil(IIntStream input, BitSet set) { for (int i = input.LA(1); (i != -1) && !set.Member(i); i = input.LA(1)) { input.Consume(); } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict( IIntStream input ) { if ( debug ) { Console.Error.WriteLine( "Enter DFA.predict for decision " + decisionNumber ); } int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { for ( ; ; ) { if ( debug ) Console.Error.WriteLine( "DFA " + decisionNumber + " state " + s + " LA(1)=" + (char)input.LA( 1 ) + "(" + input.LA( 1 ) + "), index=" + input.Index ); int specialState = special[s]; if ( specialState >= 0 ) { if ( debug ) { Console.Error.WriteLine( "DFA " + decisionNumber + " state " + s + " is special state " + specialState ); } s = SpecialStateTransition( this, specialState, input ); if ( debug ) { Console.Error.WriteLine( "DFA " + decisionNumber + " returns from special state " + specialState + " to " + s ); } if ( s == -1 ) { NoViableAlt( s, input ); return 0; } input.Consume(); continue; } if ( accept[s] >= 1 ) { if ( debug ) Console.Error.WriteLine( "accept; predict " + accept[s] + " from state " + s ); return accept[s]; } // look for a normal char transition char c = (char)input.LA( 1 ); // -1 == \uFFFF, all tokens fit in 65000 space if ( c >= min[s] && c <= max[s] ) { int snext = transition[s][c - min[s]]; // move to next state if ( snext < 0 ) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if ( eot[s] >= 0 ) { // EOT Transition to accept state? if ( debug ) Console.Error.WriteLine( "EOT transition" ); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt( s, input ); return 0; } s = snext; input.Consume(); continue; } if ( eot[s] >= 0 ) { // EOT Transition? if ( debug ) Console.Error.WriteLine( "EOT transition" ); s = eot[s]; input.Consume(); continue; } if ( c == unchecked((char)TokenConstants.EOF) && eof[s] >= 0 ) { // EOF Transition to accept state? if ( debug ) Console.Error.WriteLine( "accept via EOF; predict " + accept[eof[s]] + " from " + eof[s] ); return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol if ( debug ) { Console.Error.WriteLine( "min[" + s + "]=" + min[s] ); Console.Error.WriteLine( "max[" + s + "]=" + max[s] ); Console.Error.WriteLine( "eot[" + s + "]=" + eot[s] ); Console.Error.WriteLine( "eof[" + s + "]=" + eof[s] ); for ( int p = 0; p < transition[s].Length; p++ ) { Console.Error.Write( transition[s][p] + " " ); } Console.Error.WriteLine(); } NoViableAlt( s, input ); return 0; } } finally { input.Rewind( mark ); } }
/// <summary> /// From the input stream, predict what alternative will succeed using this /// DFA (representing the covering regular approximation to the underlying CFL). /// </summary> /// <param name="input">Input stream</param> /// <returns>Return an alternative number 1..n. Throw an exception upon error.</returns> public int Predict(IIntStream input) { int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { int specialState = special[s]; if (specialState >= 0) { s = specialStateTransitionHandler(this, specialState, input); if (s == -1) { NoViableAlt(s, input); return 0; } input.Consume(); continue; } if (accept[s] >= 1) { return accept[s]; } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if ((c >= min[s]) && (c <= max[s])) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) // EOT Transition to accept state? { s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return 0; } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? s = eot[s]; input.Consume(); continue; } if ((c == (char)Token.EOF) && (eof[s] >= 0)) { // EOF Transition to accept state? return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol NoViableAlt(s, input); return 0; } } finally { input.Rewind(mark); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int ttype = input.LA(1); while (ttype != Token.EOF && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
/// <summary> /// Recover from an error found on the input stream. This is /// for NoViableAlt and mismatched symbol exceptions. If you enable /// single token insertion and deletion, this will usually not /// handle mismatched symbol exceptions but there could be a mismatched /// token that the Match() routine could not recover from. /// </summary> public virtual void Recover(IIntStream input, RecognitionException re) { if (state.lastErrorIndex == input.Index) { // uh oh, another error at same token index; must be a case // where LT(1) is in the recovery token set so nothing is // consumed; consume a single token so at least to prevent // an infinite loop; this is a failsafe. input.Consume(); } state.lastErrorIndex = input.Index; BitSet followSet = ComputeErrorRecoverySet(); BeginResync(); ConsumeUntil(input, followSet); EndResync(); }
public RecognitionException(string message, IIntStream input, int k, Exception innerException) : base(message, innerException) { this._input = input; this._k = k; if (input != null) { this._index = input.Index + k - 1; if (input is ITokenStream) { this._token = ((ITokenStream)input).LT(k); this._line = _token.Line; this._charPositionInLine = _token.CharPositionInLine; } ITreeNodeStream tns = input as ITreeNodeStream; if (tns != null) { ExtractInformationFromTreeNodeStream(tns, k); } else { ICharStream charStream = input as ICharStream; if (charStream != null) { int mark = input.Mark(); try { for (int i = 0; i < k - 1; i++) input.Consume(); this._c = input.LA(1); this._line = ((ICharStream)input).Line; this._charPositionInLine = ((ICharStream)input).CharPositionInLine; } finally { input.Rewind(mark); } } else { this._c = input.LA(k); } } } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict(IIntStream input) { if (debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + decisionNumber); } int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { for ( ; ;) { if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " state " + s + " LA(1)=" + (char)input.LA(1) + "(" + input.LA(1) + "), index=" + input.Index); } int specialState = special[s]; if (specialState >= 0) { if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " state " + s + " is special state " + specialState); } s = SpecialStateTransition(this, specialState, input); if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " returns from special state " + specialState + " to " + s); } if (s == -1) { NoViableAlt(s, input); return(0); } input.Consume(); continue; } if (accept[s] >= 1) { if (debug) { Console.Error.WriteLine("accept; predict " + accept[s] + " from state " + s); } return(accept[s]); } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if (c >= min[s] && c <= max[s]) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) { // EOT Transition to accept state? if (debug) { Console.Error.WriteLine("EOT transition"); } s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return(0); } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? if (debug) { Console.Error.WriteLine("EOT transition"); } s = eot[s]; input.Consume(); continue; } if (c == unchecked ((char)TokenTypes.EndOfFile) && eof[s] >= 0) { // EOF Transition to accept state? if (debug) { Console.Error.WriteLine("accept via EOF; predict " + accept[eof[s]] + " from " + eof[s]); } return(accept[eof[s]]); } // not in range and not EOF/EOT, must be invalid symbol if (debug) { Console.Error.WriteLine("min[" + s + "]=" + min[s]); Console.Error.WriteLine("max[" + s + "]=" + max[s]); Console.Error.WriteLine("eot[" + s + "]=" + eot[s]); Console.Error.WriteLine("eof[" + s + "]=" + eof[s]); for (int p = 0; p < transition[s].Length; p++) { Console.Error.Write(transition[s][p] + " "); } Console.Error.WriteLine(); } NoViableAlt(s, input); return(0); } } finally { input.Rewind(mark); } }
public virtual void Recover(IIntStream input, RecognitionException re) { if (this.state.lastErrorIndex == input.Index) { input.Consume(); } this.state.lastErrorIndex = input.Index; BitSet set = this.ComputeErrorRecoverySet(); this.BeginResync(); this.ConsumeUntil(input, set); this.EndResync(); }
public virtual void ConsumeUntil( IIntStream input, int tokenType ) { //System.out.println("consumeUntil "+tokenType); int ttype = input.LA( 1 ); while ( ttype != TokenTypes.EndOfFile && ttype != tokenType ) { input.Consume(); ttype = input.LA( 1 ); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { for (int i = input.LA(1); (i != -1) && (i != tokenType); i = input.LA(1)) { input.Consume(); } }
/// <summary> Match the wildcard: in a symbol</summary> public virtual void MatchAny(IIntStream input) { state.errorRecovery = false; state.failed = false; input.Consume(); }
/** <summary>Consume tokens until one matches the given token set</summary> */ public virtual void ConsumeUntil( IIntStream input, BitSet set ) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = input.LA( 1 ); while ( ttype != TokenTypes.EndOfFile && !set.Member( ttype ) ) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); input.Consume(); ttype = input.LA( 1 ); } }
/// <summary> /// Attempt to Recover from a single missing or extra token. /// </summary> /// <remarks> /// EXTRA TOKEN /// /// LA(1) is not what we are looking for. If LA(2) has the right token, /// however, then assume LA(1) is some extra spurious token. Delete it /// and LA(2) as if we were doing a normal Match(), which advances the /// input. /// /// MISSING TOKEN /// /// If current token is consistent with what could come after /// ttype then it is ok to "insert" the missing token, else throw /// exception For example, Input "i=(3;" is clearly missing the /// ')'. When the parser returns from the nested call to expr, it /// will have call chain: /// /// stat -> expr -> atom /// /// and it will be trying to Match the ')' at this point in the /// derivation: /// /// => ID '=' '(' INT ')' ('+' atom)* ';' /// ^ /// Match() will see that ';' doesn't Match ')' and report a /// mismatched token error. To Recover, it sees that LA(1)==';' /// is in the set of tokens that can follow the ')' token /// reference in rule atom. It can assume that you forgot the ')'. /// </remarks> protected internal virtual Object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; // if next token is what we are looking for then "delete" this token if (MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input); BeginResync(); input.Consume(); // simply delete extra token EndResync(); ReportError(e); // report after consuming so AW sees the token in the exception // we want to return the token we're actually matching object matchedSymbol = GetCurrentInputSymbol(input); input.Consume(); // move past ttype token as if all were ok return matchedSymbol; } // can't recover with single token deletion, try insertion if (MismatchIsMissingToken(input, follow)) { object inserted = GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, inserted); ReportError(e); // report after inserting so AW sees the token in the exception return inserted; } // even that didn't work; must throw the exception e = new MismatchedTokenException(ttype, input); throw e; }
/** Fill a list of all NFA states visited during the parse */ protected virtual void ParseEngine(string startRule, NFAState start, NFAState stop, IIntStream input, Stack <object> ruleInvocationStack, IDebugEventListener actions, IList <NFAState> visitedStates) { NFAState s = start; if (actions != null) { actions.EnterRule(s.nfa.Grammar.FileName, start.enclosingRule.Name); } int t = input.LA(1); while (s != stop) { if (visitedStates != null) { visitedStates.Add(s); } //Console.Out.WriteLine( "parse state " + s.stateNumber + " input=" + s.nfa.Grammar.getTokenDisplayName( t ) ); // CASE 1: decision state if (s.DecisionNumber > 0 && s.nfa.Grammar.GetNumberOfAltsForDecisionNFA(s) > 1) { // decision point, must predict and jump to alt DFA dfa = s.nfa.Grammar.GetLookaheadDFA(s.DecisionNumber); //if ( s.nfa.Grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "decision: " + // dfa.getNFADecisionStartState().Description + // " input=" + s.nfa.Grammar.getTokenDisplayName( t ) ); //} int m = input.Mark(); int predictedAlt = Predict(dfa); if (predictedAlt == NFA.INVALID_ALT_NUMBER) { string description = dfa.NFADecisionStartState.Description; NoViableAltException nvae = new NoViableAltException(description, dfa.NfaStartStateDecisionNumber, s.StateNumber, input); if (actions != null) { actions.RecognitionException(nvae); } input.Consume(); // recover throw nvae; } input.Rewind(m); int parseAlt = s.TranslateDisplayAltToWalkAlt(predictedAlt); //if ( s.nfa.Grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "predicted alt " + predictedAlt + ", parseAlt " + parseAlt ); //} NFAState alt; if (parseAlt > s.nfa.Grammar.GetNumberOfAltsForDecisionNFA(s)) { // implied branch of loop etc... alt = s.nfa.Grammar.nfa.GetState(s.endOfBlockStateNumber); } else { alt = s.nfa.Grammar.GetNFAStateForAltOfDecision(s, parseAlt); } s = (NFAState)alt.transition[0].Target; continue; } // CASE 2: finished matching a rule if (s.IsAcceptState) { // end of rule node if (actions != null) { actions.ExitRule(s.nfa.Grammar.FileName, s.enclosingRule.Name); } if (ruleInvocationStack.Count == 0) { // done parsing. Hit the start state. //Console.Out.WriteLine( "stack empty in stop state for " + s.enclosingRule ); break; } // pop invoking state off the stack to know where to return to NFAState invokingState = (NFAState)ruleInvocationStack.Pop(); RuleClosureTransition invokingTransition = (RuleClosureTransition)invokingState.transition[0]; // move to node after state that invoked this rule s = invokingTransition.FollowState; continue; } Transition trans = s.transition[0]; Label label = trans.Label; if (label.IsSemanticPredicate) { FailedPredicateException fpe = new FailedPredicateException(input, s.enclosingRule.Name, "can't deal with predicates yet"); if (actions != null) { actions.RecognitionException(fpe); } } // CASE 3: epsilon transition if (label.IsEpsilon) { // CASE 3a: rule invocation state if (trans is RuleClosureTransition) { ruleInvocationStack.Push(s); s = (NFAState)trans.Target; //Console.Out.WriteLine( "call " + s.enclosingRule.name + " from " + s.nfa.Grammar.getFileName() ); if (actions != null) { actions.EnterRule(s.nfa.Grammar.FileName, s.enclosingRule.Name); } // could be jumping to new grammar, make sure DFA created if (!s.nfa.Grammar.AllDecisionDFAHaveBeenCreated) { s.nfa.Grammar.CreateLookaheadDFAs(); } } // CASE 3b: plain old epsilon transition, just move else { s = (NFAState)trans.Target; } } // CASE 4: match label on transition else if (label.Matches(t)) { if (actions != null) { if (s.nfa.Grammar.type == GrammarType.Parser || s.nfa.Grammar.type == GrammarType.Combined) { actions.ConsumeToken(((ITokenStream)input).LT(1)); } } s = (NFAState)s.transition[0].Target; input.Consume(); t = input.LA(1); } // CASE 5: error condition; label is inconsistent with input else { if (label.IsAtom) { MismatchedTokenException mte = new MismatchedTokenException(label.Atom, input); if (actions != null) { actions.RecognitionException(mte); } input.Consume(); // recover throw mte; } else if (label.IsSet) { MismatchedSetException mse = new MismatchedSetException(((IntervalSet)label.Set).ToRuntimeBitSet(), input); if (actions != null) { actions.RecognitionException(mse); } input.Consume(); // recover throw mse; } else if (label.IsSemanticPredicate) { FailedPredicateException fpe = new FailedPredicateException(input, s.enclosingRule.Name, label.SemanticContext.ToString()); if (actions != null) { actions.RecognitionException(fpe); } input.Consume(); // recover throw fpe; } else { throw new RecognitionException(input); // unknown error } } } //Console.Out.WriteLine( "hit stop state for " + stop.enclosingRule ); if (actions != null) { actions.ExitRule(s.nfa.Grammar.FileName, stop.enclosingRule.Name); } }
/// <summary>Consume tokens until one matches the given token set </summary> public virtual void ConsumeUntil(IIntStream input, BitSet set) { int ttype = input.LA(1); while (ttype != Token.EOF && !set.Member(ttype)) { input.Consume(); ttype = input.LA(1); } }
/// <summary> /// From the input stream, predict what alternative will succeed using this /// DFA (representing the covering regular approximation to the underlying CFL). /// </summary> /// <param name="input">Input stream</param> /// <returns>Return an alternative number 1..n. Throw an exception upon error.</returns> public int Predict(IIntStream input) { int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { int specialState = special[s]; if (specialState >= 0) { s = specialStateTransitionHandler(this, specialState, input); if (s == -1) { NoViableAlt(s, input); return(0); } input.Consume(); continue; } if (accept[s] >= 1) { return(accept[s]); } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if ((c >= min[s]) && (c <= max[s])) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) // EOT Transition to accept state? { s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return(0); } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? s = eot[s]; input.Consume(); continue; } if ((c == (char)Token.EOF) && (eof[s] >= 0)) { // EOF Transition to accept state? return(accept[eof[s]]); } // not in range and not EOF/EOT, must be invalid symbol NoViableAlt(s, input); return(0); } } finally { input.Rewind(mark); } }