public virtual void ConsumeUntil(IIntStream input, BitSet set) { for (int el = input.LA(1); el != -1 && !set.Member(el); el = input.LA(1)) { input.Consume(); } }
public RecognitionException(string message, IIntStream input, Exception innerException) : base(message, innerException) { this._input = input; if (input != null) { this._index = input.Index; if (input is ITokenStream) { this._token = ((ITokenStream) input).LT(1); this._line = this._token.Line; this._charPositionInLine = this._token.CharPositionInLine; } ITreeNodeStream stream = input as ITreeNodeStream; if (stream != null) { this.ExtractInformationFromTreeNodeStream(stream); } else if (input is ICharStream) { this._c = input.LA(1); this._line = ((ICharStream) input).Line; this._charPositionInLine = ((ICharStream) input).CharPositionInLine; } else { this._c = input.LA(1); } } }
public RecognitionException(string message, IIntStream input, Exception innerException) : base(message, innerException) { this._input = input; if (input != null) { this._index = input.Index; if (input is ITokenStream) { this._token = ((ITokenStream)input).LT(1); this._line = _token.Line; this._charPositionInLine = _token.CharPositionInLine; } if (input is ITreeNodeStream) { ExtractInformationFromTreeNodeStream(input); } else if (input is ICharStream) { this._c = input.LA(1); this._line = ((ICharStream)input).Line; this._charPositionInLine = ((ICharStream)input).CharPositionInLine; } else { this._c = input.LA(1); } } }
private void Initialize(IIntStream input) { this.input = input; this.index = input.Index; if (input is ITokenStream) { this.token = ((ITokenStream)input).LT(1); this.line = token.Line; this.charPositionInLine = token.CharPositionInLine; } if (input is ITreeNodeStream) { ExtractInformationFromTreeNodeStream(input); } else if (input is ICharStream) { this.c = input.LA(1); this.line = ((ICharStream)input).Line; this.charPositionInLine = ((ICharStream)input).CharPositionInLine; } else { this.c = input.LA(1); } }
public RecognitionException(string message, Exception inner, IIntStream input) : base(message, inner) { this.input = input; this.index = input.Index(); if (input is ITokenStream) { this.token = ((ITokenStream)input).LT(1); this.line = token.Line; this.charPositionInLine = token.CharPositionInLine; } if (input is ITreeNodeStream) { ExtractInformationFromTreeNodeStream(input); } else if (input is ICharStream) { this.c = input.LA(1); this.line = ((ICharStream)input).Line; this.charPositionInLine = ((ICharStream)input).CharPositionInLine; } else { this.c = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { for (int index = input.LA(1); index != -1 && index != tokenType; index = input.LA(1)) { input.Consume(); } }
/// <summary>Consume tokens until one matches the given token set </summary> public virtual void ConsumeUntil(IIntStream input, BitSet set) { int ttype = input.LA(1); while (ttype != Token.EOF && !set.Member(ttype)) { input.Consume(); ttype = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, BitSet set) { int num = input.LA(1); while (num != -1 && !set.Member(num)) { input.Consume(); num = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int num = input.LA(1); while (num != -1 && num != tokenType) { input.Consume(); num = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int ttype = input.LA(1); while (ttype != Token.EOF && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { //System.out.println("consumeUntil "+tokenType); int ttype = input.LA(1); while (ttype != TokenTypes.EndOfFile && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
/** <summary>Consume tokens until one matches the given token set</summary> */ public virtual void ConsumeUntil(IIntStream input, BitSet set) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = input.LA(1); while (ttype != TokenTypes.EndOfFile && !set.Member(ttype)) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); input.Consume(); ttype = input.LA(1); } }
private int SpecialStateTransition2(DFA dfa, int s, IIntStream _input) { IIntStream input = _input; int _s = s; switch (s) { case 0: int LA2_8 = input.LA(1); s = -1; if (((LA2_8 >= '\u0000' && LA2_8 <= '\b') || (LA2_8 >= '\u000B' && LA2_8 <= '\f') || (LA2_8 >= '\u000E' && LA2_8 <= '\u001F') || (LA2_8 >= '!' && LA2_8 <= '\'') || (LA2_8 >= '*' && LA2_8 <= '\uFFFF'))) { s = 6; } else { s = 10; } if (s >= 0) { return(s); } break; case 1: int LA2_9 = input.LA(1); s = -1; if (((LA2_9 >= '\u0000' && LA2_9 <= '\b') || (LA2_9 >= '\u000B' && LA2_9 <= '\f') || (LA2_9 >= '\u000E' && LA2_9 <= '\u001F') || (LA2_9 >= '!' && LA2_9 <= '\'') || (LA2_9 >= '*' && LA2_9 <= '\uFFFF'))) { s = 6; } else { s = 11; } if (s >= 0) { return(s); } break; } NoViableAltException nvae = new NoViableAltException(dfa.Description, 2, _s, input); dfa.Error(nvae); throw nvae; }
public RecognitionException(string message, IIntStream input, int k, Exception innerException) : base(message, innerException) { this._input = input; this._k = k; if (input != null) { this._index = input.Index + k - 1; if (input is ITokenStream) { this._token = ((ITokenStream)input).LT(k); this._line = _token.Line; this._charPositionInLine = _token.CharPositionInLine; } ITreeNodeStream tns = input as ITreeNodeStream; if (tns != null) { ExtractInformationFromTreeNodeStream(tns, k); } else { ICharStream charStream = input as ICharStream; if (charStream != null) { int mark = input.Mark(); try { for (int i = 0; i < k - 1; i++) { input.Consume(); } this._c = input.LA(1); this._line = ((ICharStream)input).Line; this._charPositionInLine = ((ICharStream)input).CharPositionInLine; } finally { input.Rewind(mark); } } else { this._c = input.LA(k); } } } }
public virtual bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { // we have no information about the follow; we can only consume // a single token and hope for the best return(false); } // compute what can follow this grammar element reference if (follow.Member(TokenTypes.EndOfRule)) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(viableTokensFollowingThisRule); if (state._fsp >= 0) { // remove EOR if we're not the start symbol follow.Remove(TokenTypes.EndOfRule); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token //System.out.println("viable tokens="+follow.toString(getTokenNames())); //System.out.println("LT(1)="+((TokenStream)input).LT(1)); // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if (follow.Member(input.LA(1)) || follow.Member(TokenTypes.EndOfRule)) { //System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting..."); return(true); } return(false); }
protected internal int DFA8_SpecialStateTransition(DFA dfa, int s, IIntStream _input) //throws NoViableAltException { IIntStream input = _input; int _s = s; switch (s) { case 0: int LA8_18 = input.LA(1); s = -1; if (((LA8_18 >= '\u0000' && LA8_18 <= '\uFFFF'))) { s = 43; } else { s = 42; } if (s >= 0) { return(s); } break; } NoViableAltException nvae8 = new NoViableAltException(dfa.Description, 8, _s, input); dfa.Error(nvae8); throw nvae8; }
public bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { // we have no information about the follow; we can only consume // a single token and hope for the best return(false); } // compute what can follow this grammar element reference if (follow.Member(Token.EOR_TOKEN_TYPE)) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(viableTokensFollowingThisRule); if (state.followingStackPointer >= 0) { // remove EOR if we're not the start symbol follow.Remove(Token.EOR_TOKEN_TYPE); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if (follow.Member(input.LA(1)) || follow.Member(Token.EOR_TOKEN_TYPE)) { return(true); } return(false); }
public virtual object GetInputSymbol(int k) { if (input is ITokenStream) { return(((ITokenStream)input).LT(k)); } return((char)input.LA(k)); }
public virtual IToken NextToken() { if (grammar.type != GrammarType.Lexer) { return(null); } if (input.LA(1) == CharStreamConstants.EndOfFile) { return(new CommonToken((ICharStream)input, CharStreamConstants.EndOfFile, TokenChannels.Default, input.Index, input.Index)); } int start = input.Index; int charPos = ((ICharStream)input).CharPositionInLine; CommonToken token = null; while (input.LA(1) != CharStreamConstants.EndOfFile) { try { token = Scan(Grammar.ArtificialTokensRuleName, null); break; } catch (RecognitionException re) { // report a problem and try for another ReportScanError(re); continue; } } // the scan can only set type // we must set the line, and other junk here to make it a complete token int stop = input.Index - 1; if (token == null) { return(new CommonToken((ICharStream)input, CharStreamConstants.EndOfFile, TokenChannels.Default, start, start)); } token.Line = (((ICharStream)input).Line); token.StartIndex = start; token.StopIndex = stop; token.CharPositionInLine = charPos; return(token); }
/** <summary> * Match current input symbol against ttype. Attempt * single token insertion or deletion error recovery. If * that fails, throw MismatchedTokenException. * </summary> * * <remarks> * To turn off single token insertion or deletion error * recovery, override recoverFromMismatchedToken() and have it * throw an exception. See TreeParser.recoverFromMismatchedToken(). * This way any error in a rule will cause an exception and * immediate exit from rule. Rule would recover by resynchronizing * to the set of symbols that can follow rule ref. * </remarks> */ public virtual object Match(IIntStream input, int ttype, BitSet follow) { //System.out.println("match "+((TokenStream)input).LT(1)); object matchedSymbol = GetCurrentInputSymbol(input); //DEBUG int a = input.LA(1); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return(matchedSymbol); } if (state.backtracking > 0) { state.failed = true; return(matchedSymbol); } matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return(matchedSymbol); }
public static bool StartsWith(this IIntStream stream, int offset, string value) { if (stream.Index + offset + value.Length >= stream.Size) { return(false); } for (int i = 0; i < value.Length; i++) { if (value[i] != (char)stream.LA(offset + i + 1)) { return(false); } } return(true); }
public virtual bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { return(false); } if (follow.Member(1)) { BitSet sensitiveRuleFollow = this.ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(sensitiveRuleFollow); if (this.state._fsp >= 0) { follow.Remove(1); } } return(follow.Member(input.LA(1)) || follow.Member(1)); }
public virtual object Match(IIntStream input, int ttype, BitSet follow) { object currentInputSymbol = this.GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); this.state.errorRecovery = false; this.state.failed = false; return(currentInputSymbol); } if (this.state.backtracking > 0) { this.state.failed = true; return(currentInputSymbol); } return(this.RecoverFromMismatchedToken(input, ttype, follow)); }
public override object Match(IIntStream input, int ttype, BitSet follow) { object currentInputSymbol = this.GetCurrentInputSymbol(input); DumpSymbol(currentInputSymbol); if (input.LA(1) == ttype) { input.Consume(); this.state.errorRecovery = false; this.state.failed = false; return currentInputSymbol; } else { if (this.state.backtracking <= 0) return this.RecoverFromMismatchedToken(input, ttype, follow); this.state.failed = true; return currentInputSymbol; } }
/// <summary> /// Match current input symbol against ttype. Attempt /// single token insertion or deletion error recovery. If /// that fails, throw MismatchedTokenException. /// </summary> /// <remarks> /// To turn off single token insertion or deletion error /// recovery, override mismatchRecover() and have it call /// plain mismatch(), which does not recover. Then any error /// in a rule will cause an exception and immediate exit from /// rule. Rule would recover by resynchronizing to the set of /// symbols that can follow rule ref. /// </remarks> public virtual object Match(IIntStream input, int ttype, BitSet follow) { object matchedSymbol = GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return(matchedSymbol); } if (state.backtracking > 0) { state.failed = true; return(matchedSymbol); } Mismatch(input, ttype, follow); matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return(matchedSymbol); }
private int SpecialStateTransition9(DFA dfa, int s, IIntStream _input) { IIntStream input = _input; int _s = s; switch (s) { case 0: int LA9_0 = input.LA(1); s = -1; if ((LA9_0 == 'a')) { s = 1; } else if ((LA9_0 == 'b')) { s = 2; } else if ((LA9_0 == 'd')) { s = 3; } else if ((LA9_0 == 'e')) { s = 4; } else if ((LA9_0 == 'f')) { s = 5; } else if ((LA9_0 == 'i')) { s = 6; } else if ((LA9_0 == 'l')) { s = 7; } else if ((LA9_0 == 'n')) { s = 8; } else if ((LA9_0 == 'o')) { s = 9; } else if ((LA9_0 == 't')) { s = 10; } else if ((LA9_0 == 'v')) { s = 11; } else if ((LA9_0 == 'w')) { s = 12; } else if ((LA9_0 == 's')) { s = 13; } else if (((LA9_0 >= 'A' && LA9_0 <= 'Z') || LA9_0 == 'c' || (LA9_0 >= 'g' && LA9_0 <= 'h') || (LA9_0 >= 'j' && LA9_0 <= 'k') || LA9_0 == 'm' || (LA9_0 >= 'p' && LA9_0 <= 'r') || LA9_0 == 'u' || (LA9_0 >= 'x' && LA9_0 <= 'z'))) { s = 14; } else if (((LA9_0 >= '0' && LA9_0 <= '9'))) { s = 15; } else if ((LA9_0 == '\"')) { s = 16; } else if (((LA9_0 >= '\t' && LA9_0 <= '\n') || LA9_0 == '\r' || LA9_0 == ' ')) { s = 17; } else if ((LA9_0 == '/')) { s = 18; } else if (((LA9_0 >= '\u0000' && LA9_0 <= '\b') || (LA9_0 >= '\u000B' && LA9_0 <= '\f') || (LA9_0 >= '\u000E' && LA9_0 <= '\u001F') || LA9_0 == '!' || (LA9_0 >= '#' && LA9_0 <= '.') || (LA9_0 >= ':' && LA9_0 <= '@') || (LA9_0 >= '[' && LA9_0 <= '`') || (LA9_0 >= '{' && LA9_0 <= '\uFFFF'))) { s = 19; } if (s >= 0) { return(s); } break; case 1: int LA9_16 = input.LA(1); s = -1; if (((LA9_16 >= '\u0000' && LA9_16 <= '\uFFFF'))) { s = 40; } else { s = 19; } if (s >= 0) { return(s); } break; } NoViableAltException nvae = new NoViableAltException(dfa.Description, 9, _s, input); dfa.Error(nvae); throw nvae; }
/** Fill a list of all NFA states visited during the parse */ protected virtual void ParseEngine( String startRule, NFAState start, NFAState stop, IIntStream input, Stack<object> ruleInvocationStack, IDebugEventListener actions, IList visitedStates ) { NFAState s = start; if ( actions != null ) { actions.EnterRule( s.nfa.grammar.FileName, start.enclosingRule.Name ); } int t = input.LA( 1 ); while ( s != stop ) { if ( visitedStates != null ) { visitedStates.Add( s ); } //Console.Out.WriteLine( "parse state " + s.stateNumber + " input=" + s.nfa.grammar.getTokenDisplayName( t ) ); // CASE 1: decision state if ( s.DecisionNumber > 0 && s.nfa.grammar.GetNumberOfAltsForDecisionNFA( s ) > 1 ) { // decision point, must predict and jump to alt DFA dfa = s.nfa.grammar.GetLookaheadDFA( s.DecisionNumber ); //if ( s.nfa.grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "decision: " + // dfa.getNFADecisionStartState().Description + // " input=" + s.nfa.grammar.getTokenDisplayName( t ) ); //} int m = input.Mark(); int predictedAlt = Predict( dfa ); if ( predictedAlt == NFA.INVALID_ALT_NUMBER ) { String description = dfa.NFADecisionStartState.Description; NoViableAltException nvae = new NoViableAltException( description, dfa.DecisionNumber, s.stateNumber, input ); if ( actions != null ) { actions.RecognitionException( nvae ); } input.Consume(); // recover throw nvae; } input.Rewind( m ); int parseAlt = s.TranslateDisplayAltToWalkAlt( predictedAlt ); //if ( s.nfa.grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "predicted alt " + predictedAlt + ", parseAlt " + parseAlt ); //} NFAState alt; if ( parseAlt > s.nfa.grammar.GetNumberOfAltsForDecisionNFA( s ) ) { // implied branch of loop etc... alt = s.nfa.grammar.nfa.GetState( s.endOfBlockStateNumber ); } else { alt = s.nfa.grammar.GetNFAStateForAltOfDecision( s, parseAlt ); } s = (NFAState)alt.transition[0].target; continue; } // CASE 2: finished matching a rule if ( s.IsAcceptState ) { // end of rule node if ( actions != null ) { actions.ExitRule( s.nfa.grammar.FileName, s.enclosingRule.Name ); } if ( ruleInvocationStack.Count == 0 ) { // done parsing. Hit the start state. //Console.Out.WriteLine( "stack empty in stop state for " + s.enclosingRule ); break; } // pop invoking state off the stack to know where to return to NFAState invokingState = (NFAState)ruleInvocationStack.Pop(); RuleClosureTransition invokingTransition = (RuleClosureTransition)invokingState.transition[0]; // move to node after state that invoked this rule s = invokingTransition.followState; continue; } Transition trans = s.transition[0]; Label label = trans.label; if ( label.IsSemanticPredicate ) { FailedPredicateException fpe = new FailedPredicateException( input, s.enclosingRule.Name, "can't deal with predicates yet" ); if ( actions != null ) { actions.RecognitionException( fpe ); } } // CASE 3: epsilon transition if ( label.IsEpsilon ) { // CASE 3a: rule invocation state if ( trans is RuleClosureTransition ) { ruleInvocationStack.Push( s ); s = (NFAState)trans.target; //Console.Out.WriteLine( "call " + s.enclosingRule.name + " from " + s.nfa.grammar.getFileName() ); if ( actions != null ) { actions.EnterRule( s.nfa.grammar.FileName, s.enclosingRule.Name ); } // could be jumping to new grammar, make sure DFA created if ( !s.nfa.grammar.AllDecisionDFAHaveBeenCreated ) { s.nfa.grammar.CreateLookaheadDFAs(); } } // CASE 3b: plain old epsilon transition, just move else { s = (NFAState)trans.target; } } // CASE 4: match label on transition else if ( label.Matches( t ) ) { if ( actions != null ) { if ( s.nfa.grammar.type == GrammarType.Parser || s.nfa.grammar.type == GrammarType.Combined ) { actions.ConsumeToken( ( (ITokenStream)input ).LT( 1 ) ); } } s = (NFAState)s.transition[0].target; input.Consume(); t = input.LA( 1 ); } // CASE 5: error condition; label is inconsistent with input else { if ( label.IsAtom ) { MismatchedTokenException mte = new MismatchedTokenException( label.Atom, input ); if ( actions != null ) { actions.RecognitionException( mte ); } input.Consume(); // recover throw mte; } else if ( label.IsSet ) { MismatchedSetException mse = new MismatchedSetException( ( (IntervalSet)label.Set ).ToRuntimeBitSet(), input ); if ( actions != null ) { actions.RecognitionException( mse ); } input.Consume(); // recover throw mse; } else if ( label.IsSemanticPredicate ) { FailedPredicateException fpe = new FailedPredicateException( input, s.enclosingRule.Name, label.SemanticContext.ToString() ); if ( actions != null ) { actions.RecognitionException( fpe ); } input.Consume(); // recover throw fpe; } else { throw new RecognitionException( input ); // unknown error } } } //Console.Out.WriteLine( "hit stop state for " + stop.enclosingRule ); if ( actions != null ) { actions.ExitRule( s.nfa.grammar.FileName, stop.enclosingRule.Name ); } }
public virtual bool MismatchIsUnwantedToken(IIntStream input, int ttype) { return(input.LA(2) == ttype); }
public RecognitionException(string message, Exception inner, IIntStream input) : base(message, inner) { this.input = input; this.index = input.Index(); if (input is ITokenStream) { this.token = ((ITokenStream)input).LT(1); this.line = token.Line; this.charPositionInLine = token.CharPositionInLine; } //if (input is ITreeNodeStream) //{ // ExtractInformationFromTreeNodeStream(input); //} //else if (input is ICharStream) { this.c = input.LA(1); this.line = ((ICharStream)input).Line; this.charPositionInLine = ((ICharStream)input).CharPositionInLine; } else { this.c = input.LA(1); } }
/// <summary>Consume tokens until one matches the given token set </summary> public virtual void ConsumeUntil(IIntStream input, BitSet set) { int ttype = input.LA(1); while (ttype != Token.EOF && !set.Member(ttype)) { input.Consume(); ttype = input.LA(1); } }
private void Initialize(IIntStream input) { this.input = input; this.index = input.Index; if (input is ITokenStream) { this.token = ((ITokenStream)input).LT(1); this.line = token.Line; this.charPositionInLine = token.CharPositionInLine; } if (input is ITreeNodeStream) { ExtractInformationFromTreeNodeStream(input); } else if (input is ICharStream) { this.c = input.LA(1); this.line = ((ICharStream)input).Line; this.charPositionInLine = ((ICharStream)input).CharPositionInLine; } else { this.c = input.LA(1); } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict(IIntStream input) { if (debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + decisionNumber); } int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { for ( ; ;) { if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " state " + s + " LA(1)=" + (char)input.LA(1) + "(" + input.LA(1) + "), index=" + input.Index); } int specialState = special[s]; if (specialState >= 0) { if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " state " + s + " is special state " + specialState); } s = SpecialStateTransition(this, specialState, input); if (debug) { Console.Error.WriteLine("DFA " + decisionNumber + " returns from special state " + specialState + " to " + s); } if (s == -1) { NoViableAlt(s, input); return(0); } input.Consume(); continue; } if (accept[s] >= 1) { if (debug) { Console.Error.WriteLine("accept; predict " + accept[s] + " from state " + s); } return(accept[s]); } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if (c >= min[s] && c <= max[s]) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) { // EOT Transition to accept state? if (debug) { Console.Error.WriteLine("EOT transition"); } s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return(0); } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? if (debug) { Console.Error.WriteLine("EOT transition"); } s = eot[s]; input.Consume(); continue; } if (c == unchecked ((char)TokenTypes.EndOfFile) && eof[s] >= 0) { // EOF Transition to accept state? if (debug) { Console.Error.WriteLine("accept via EOF; predict " + accept[eof[s]] + " from " + eof[s]); } return(accept[eof[s]]); } // not in range and not EOF/EOT, must be invalid symbol if (debug) { Console.Error.WriteLine("min[" + s + "]=" + min[s]); Console.Error.WriteLine("max[" + s + "]=" + max[s]); Console.Error.WriteLine("eot[" + s + "]=" + eot[s]); Console.Error.WriteLine("eof[" + s + "]=" + eof[s]); for (int p = 0; p < transition[s].Length; p++) { Console.Error.Write(transition[s][p] + " "); } Console.Error.WriteLine(); } NoViableAlt(s, input); return(0); } } finally { input.Rewind(mark); } }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict(IIntStream input) { if (input == null) { throw new ArgumentNullException("input"); } DfaDebugMessage("Enter DFA.Predict for decision {0}", decisionNumber); int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { DfaDebugMessage("DFA {0} state {1} LA(1)={2}({3}), index={4}", decisionNumber, s, (char)input.LA(1), input.LA(1), input.Index); int specialState = special[s]; if (specialState >= 0) { DfaDebugMessage("DFA {0} state {1} is special state {2}", decisionNumber, s, specialState); s = SpecialStateTransition(this, specialState, input); DfaDebugMessage("DFA {0} returns from special state {1} to {2}", decisionNumber, specialState, s); if (s == -1) { NoViableAlt(s, input); return(0); } input.Consume(); continue; } if (accept[s] >= 1) { DfaDebugMessage("accept; predict {0} from state {1}", accept[s], s); return(accept[s]); } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if (c >= min[s] && c <= max[s]) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) { // EOT Transition to accept state? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return(0); } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); continue; } if (c == unchecked ((char)TokenTypes.EndOfFile) && eof[s] >= 0) { // EOF Transition to accept state? DfaDebugMessage("accept via EOF; predict {0} from {1}", accept[eof[s]], eof[s]); return(accept[eof[s]]); } // not in range and not EOF/EOT, must be invalid symbol DfaDebugInvalidSymbol(s); NoViableAlt(s, input); return(0); } } finally { input.Rewind(mark); } }
public virtual bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { return false; } if (follow.Member(1)) { BitSet a = this.ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(a); if (this.state._fsp >= 0) { follow.Remove(1); } } if (!follow.Member(input.LA(1)) && !follow.Member(1)) { return false; } return true; }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict( IIntStream input ) { if ( debug ) { Console.Error.WriteLine( "Enter DFA.predict for decision " + decisionNumber ); } int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { for ( ; ; ) { if ( debug ) Console.Error.WriteLine( "DFA " + decisionNumber + " state " + s + " LA(1)=" + (char)input.LA( 1 ) + "(" + input.LA( 1 ) + "), index=" + input.Index ); int specialState = special[s]; if ( specialState >= 0 ) { if ( debug ) { Console.Error.WriteLine( "DFA " + decisionNumber + " state " + s + " is special state " + specialState ); } s = SpecialStateTransition( this, specialState, input ); if ( debug ) { Console.Error.WriteLine( "DFA " + decisionNumber + " returns from special state " + specialState + " to " + s ); } if ( s == -1 ) { NoViableAlt( s, input ); return 0; } input.Consume(); continue; } if ( accept[s] >= 1 ) { if ( debug ) Console.Error.WriteLine( "accept; predict " + accept[s] + " from state " + s ); return accept[s]; } // look for a normal char transition char c = (char)input.LA( 1 ); // -1 == \uFFFF, all tokens fit in 65000 space if ( c >= min[s] && c <= max[s] ) { int snext = transition[s][c - min[s]]; // move to next state if ( snext < 0 ) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if ( eot[s] >= 0 ) { // EOT Transition to accept state? if ( debug ) Console.Error.WriteLine( "EOT transition" ); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt( s, input ); return 0; } s = snext; input.Consume(); continue; } if ( eot[s] >= 0 ) { // EOT Transition? if ( debug ) Console.Error.WriteLine( "EOT transition" ); s = eot[s]; input.Consume(); continue; } if ( c == unchecked((char)TokenConstants.EOF) && eof[s] >= 0 ) { // EOF Transition to accept state? if ( debug ) Console.Error.WriteLine( "accept via EOF; predict " + accept[eof[s]] + " from " + eof[s] ); return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol if ( debug ) { Console.Error.WriteLine( "min[" + s + "]=" + min[s] ); Console.Error.WriteLine( "max[" + s + "]=" + max[s] ); Console.Error.WriteLine( "eot[" + s + "]=" + eot[s] ); Console.Error.WriteLine( "eof[" + s + "]=" + eof[s] ); for ( int p = 0; p < transition[s].Length; p++ ) { Console.Error.Write( transition[s][p] + " " ); } Console.Error.WriteLine(); } NoViableAlt( s, input ); return 0; } } finally { input.Rewind( mark ); } }
public virtual bool MismatchIsUnwantedToken(IIntStream input, int ttype) { return (input.LA(2) == ttype); }
/** Fill a list of all NFA states visited during the parse */ protected virtual void ParseEngine(string startRule, NFAState start, NFAState stop, IIntStream input, Stack <object> ruleInvocationStack, IDebugEventListener actions, IList <NFAState> visitedStates) { NFAState s = start; if (actions != null) { actions.EnterRule(s.nfa.Grammar.FileName, start.enclosingRule.Name); } int t = input.LA(1); while (s != stop) { if (visitedStates != null) { visitedStates.Add(s); } //Console.Out.WriteLine( "parse state " + s.stateNumber + " input=" + s.nfa.Grammar.getTokenDisplayName( t ) ); // CASE 1: decision state if (s.DecisionNumber > 0 && s.nfa.Grammar.GetNumberOfAltsForDecisionNFA(s) > 1) { // decision point, must predict and jump to alt DFA dfa = s.nfa.Grammar.GetLookaheadDFA(s.DecisionNumber); //if ( s.nfa.Grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "decision: " + // dfa.getNFADecisionStartState().Description + // " input=" + s.nfa.Grammar.getTokenDisplayName( t ) ); //} int m = input.Mark(); int predictedAlt = Predict(dfa); if (predictedAlt == NFA.INVALID_ALT_NUMBER) { string description = dfa.NFADecisionStartState.Description; NoViableAltException nvae = new NoViableAltException(description, dfa.NfaStartStateDecisionNumber, s.StateNumber, input); if (actions != null) { actions.RecognitionException(nvae); } input.Consume(); // recover throw nvae; } input.Rewind(m); int parseAlt = s.TranslateDisplayAltToWalkAlt(predictedAlt); //if ( s.nfa.Grammar.type != GrammarType.Lexer ) //{ // Console.Out.WriteLine( "predicted alt " + predictedAlt + ", parseAlt " + parseAlt ); //} NFAState alt; if (parseAlt > s.nfa.Grammar.GetNumberOfAltsForDecisionNFA(s)) { // implied branch of loop etc... alt = s.nfa.Grammar.nfa.GetState(s.endOfBlockStateNumber); } else { alt = s.nfa.Grammar.GetNFAStateForAltOfDecision(s, parseAlt); } s = (NFAState)alt.transition[0].Target; continue; } // CASE 2: finished matching a rule if (s.IsAcceptState) { // end of rule node if (actions != null) { actions.ExitRule(s.nfa.Grammar.FileName, s.enclosingRule.Name); } if (ruleInvocationStack.Count == 0) { // done parsing. Hit the start state. //Console.Out.WriteLine( "stack empty in stop state for " + s.enclosingRule ); break; } // pop invoking state off the stack to know where to return to NFAState invokingState = (NFAState)ruleInvocationStack.Pop(); RuleClosureTransition invokingTransition = (RuleClosureTransition)invokingState.transition[0]; // move to node after state that invoked this rule s = invokingTransition.FollowState; continue; } Transition trans = s.transition[0]; Label label = trans.Label; if (label.IsSemanticPredicate) { FailedPredicateException fpe = new FailedPredicateException(input, s.enclosingRule.Name, "can't deal with predicates yet"); if (actions != null) { actions.RecognitionException(fpe); } } // CASE 3: epsilon transition if (label.IsEpsilon) { // CASE 3a: rule invocation state if (trans is RuleClosureTransition) { ruleInvocationStack.Push(s); s = (NFAState)trans.Target; //Console.Out.WriteLine( "call " + s.enclosingRule.name + " from " + s.nfa.Grammar.getFileName() ); if (actions != null) { actions.EnterRule(s.nfa.Grammar.FileName, s.enclosingRule.Name); } // could be jumping to new grammar, make sure DFA created if (!s.nfa.Grammar.AllDecisionDFAHaveBeenCreated) { s.nfa.Grammar.CreateLookaheadDFAs(); } } // CASE 3b: plain old epsilon transition, just move else { s = (NFAState)trans.Target; } } // CASE 4: match label on transition else if (label.Matches(t)) { if (actions != null) { if (s.nfa.Grammar.type == GrammarType.Parser || s.nfa.Grammar.type == GrammarType.Combined) { actions.ConsumeToken(((ITokenStream)input).LT(1)); } } s = (NFAState)s.transition[0].Target; input.Consume(); t = input.LA(1); } // CASE 5: error condition; label is inconsistent with input else { if (label.IsAtom) { MismatchedTokenException mte = new MismatchedTokenException(label.Atom, input); if (actions != null) { actions.RecognitionException(mte); } input.Consume(); // recover throw mte; } else if (label.IsSet) { MismatchedSetException mse = new MismatchedSetException(((IntervalSet)label.Set).ToRuntimeBitSet(), input); if (actions != null) { actions.RecognitionException(mse); } input.Consume(); // recover throw mse; } else if (label.IsSemanticPredicate) { FailedPredicateException fpe = new FailedPredicateException(input, s.enclosingRule.Name, label.SemanticContext.ToString()); if (actions != null) { actions.RecognitionException(fpe); } input.Consume(); // recover throw fpe; } else { throw new RecognitionException(input); // unknown error } } } //Console.Out.WriteLine( "hit stop state for " + stop.enclosingRule ); if (actions != null) { actions.ExitRule(s.nfa.Grammar.FileName, stop.enclosingRule.Name); } }
public virtual void ConsumeUntil(IIntStream input, BitSet set) { for (int i = input.LA(1); (i != -1) && !set.Member(i); i = input.LA(1)) { input.Consume(); } }
/// <summary> /// Match current input symbol against ttype. Attempt /// single token insertion or deletion error recovery. If /// that fails, throw MismatchedTokenException. /// </summary> /// <remarks> /// To turn off single token insertion or deletion error /// recovery, override RecoverFromMismatchedToken() and have it call /// pthrow an exception. See TreeParser.RecoverFromMismatchedToken(). /// This way any error in a rule will cause an exception and /// immediate exit from rule. Rule would recover by resynchronizing /// to the set of symbols that can follow rule ref. /// </remarks> public virtual object Match(IIntStream input, int ttype, BitSet follow) { object matchedSymbol = GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return matchedSymbol; } if (state.backtracking > 0) { state.failed = true; return matchedSymbol; } matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return matchedSymbol; }
public virtual int Predict(IIntStream input) { if (this.debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + (object)this.decisionNumber); } int marker = input.Mark(); int s1 = 0; try { char ch; while (true) { if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " state " + (object)s1 + " LA(1)=" + (object)(char)input.LA(1) + "(" + (object)input.LA(1) + "), index=" + (object)input.Index); } int s2 = (int)this.special[s1]; if (s2 >= 0) { if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " state " + (object)s1 + " is special state " + (object)s2); } s1 = this.SpecialStateTransition(this, s2, input); if (this.debug) { Console.Error.WriteLine("DFA " + (object)this.decisionNumber + " returns from special state " + (object)s2 + " to " + (object)s1); } if (s1 != -1) { input.Consume(); } else { break; } } else if (this.accept[s1] < (short)1) { ch = (char)input.LA(1); if ((int)ch >= (int)this.min[s1] && (int)ch <= (int)this.max[s1]) { int num = (int)this.transition[s1][(int)ch - (int)this.min[s1]]; if (num < 0) { if (this.eot[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } s1 = (int)this.eot[s1]; input.Consume(); } else { goto label_23; } } else { s1 = num; input.Consume(); } } else if (this.eot[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } s1 = (int)this.eot[s1]; input.Consume(); } else { goto label_29; } } else { goto label_14; } } this.NoViableAlt(s1, input); return(0); label_14: if (this.debug) { Console.Error.WriteLine("accept; predict " + (object)this.accept[s1] + " from state " + (object)s1); } return((int)this.accept[s1]); label_23: this.NoViableAlt(s1, input); return(0); label_29: if (ch == char.MaxValue && this.eof[s1] >= (short)0) { if (this.debug) { Console.Error.WriteLine("accept via EOF; predict " + (object)this.accept[(int)this.eof[s1]] + " from " + (object)this.eof[s1]); } return((int)this.accept[(int)this.eof[s1]]); } if (this.debug) { Console.Error.WriteLine("min[" + (object)s1 + "]=" + (object)this.min[s1]); Console.Error.WriteLine("max[" + (object)s1 + "]=" + (object)this.max[s1]); Console.Error.WriteLine("eot[" + (object)s1 + "]=" + (object)this.eot[s1]); Console.Error.WriteLine("eof[" + (object)s1 + "]=" + (object)this.eof[s1]); for (int index = 0; index < this.transition[s1].Length; ++index) { Console.Error.Write(((int)this.transition[s1][index]).ToString() + " "); } Console.Error.WriteLine(); } this.NoViableAlt(s1, input); return(0); } finally { input.Rewind(marker); } }
public RecognitionException(string message, IIntStream input, int k, Exception innerException) : base(message, innerException) { this._input = input; this._k = k; if (input != null) { this._index = input.Index + k - 1; if (input is ITokenStream) { this._token = ((ITokenStream)input).LT(k); this._line = _token.Line; this._charPositionInLine = _token.CharPositionInLine; } ITreeNodeStream tns = input as ITreeNodeStream; if (tns != null) { ExtractInformationFromTreeNodeStream(tns, k); } else { ICharStream charStream = input as ICharStream; if (charStream != null) { int mark = input.Mark(); try { for (int i = 0; i < k - 1; i++) input.Consume(); this._c = input.LA(1); this._line = ((ICharStream)input).Line; this._charPositionInLine = ((ICharStream)input).CharPositionInLine; } finally { input.Rewind(mark); } } else { this._c = input.LA(k); } } } }
public bool CheckQuote(string what_rule) { var lexer_state = this.State; var parser_state = Parser.State; var ts = this.RealTokenStream; //System.Console.WriteLine("Rule " + what_rule); //System.Console.WriteLine("lexer state " + lexer_state); //System.Console.WriteLine("parser state " + parser_state); //System.Console.Write("prior tokens: "); //for (int i = magic; i > 0; --i) //{ // try // { // IToken cur = ts.LT(-i); // if (cur != null) // System.Console.WriteLine(cur.ToString()); // } // catch // { // } //} IToken last = ts.LT(-1); var stop = last.StopIndex; CommonToken last_ct = last as CommonToken; var s2 = last_ct.InputStream.Index; IIntStream istr = this.InputStream; var num_chars = s2 - stop - 1; //System.Console.Write("prior characters to last token: "); //for (int i = num_chars; i > 0; --i) //{ // try // { // var cur = istr.LA(-i); // if (cur > 0) // System.Console.Write((char) cur); // } // catch // { // } //} //System.Console.WriteLine(); //System.Console.WriteLine("Parser context:"); //var ctx = Parser.Context; //while (ctx != null) //{ // System.Console.WriteLine("c = " + ctx.GetType().Name); // ctx = ctx.Parent as ParserRuleContext; //} // Heuristic decision making. bool result = false; int quote_count = 0; int last_quote = 0; for (int i = 0; i < num_chars; ++i) { try { var cur = istr.LA(-i); if (cur > 0) { if (cur == (int)'\'') { quote_count++; last_quote = i; } } } catch { } } bool is_operator = true; for (int i = last_quote; i < num_chars; ++i) { try { var cur = istr.LA(-i); if (cur > 0) { var c = (char)cur; if (i == last_quote && c == '\'') { ; } else if (c == ' ' || c == '\t') { ; } else if (Char.IsLetterOrDigit(c)) { is_operator = true; break; } else { is_operator = false; break; } } } catch { } } if (what_rule == "SQ") { result = quote_count == 1 && is_operator; //System.Console.WriteLine(result ? ("yes, it's a " + what_rule + " operator.") : ("no, it's not a " + what_rule + " operator.")); } else { result = (quote_count > 1 && !is_operator); //System.Console.WriteLine(result ? ("yes, it's a " + what_rule) : ("no, it's not a " + what_rule)); } //System.Console.WriteLine(); return(result); }
public virtual int Predict(IIntStream input) { int num6; if (this.debug) { Console.Error.WriteLine("Enter DFA.predict for decision " + this.decisionNumber); } int marker = input.Mark(); int index = 0; try { Label_0030: if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " state ", index, " LA(1)=", (char) input.LA(1), "(", input.LA(1), "), index=", input.Index })); } int s = this.special[index]; if (s >= 0) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " state ", index, " is special state ", s })); } index = this.SpecialStateTransition(this, s, input); if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "DFA ", this.decisionNumber, " returns from special state ", s, " to ", index })); } if (index == -1) { this.NoViableAlt(index, input); return 0; } input.Consume(); goto Label_0030; } if (this.accept[index] >= 1) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "accept; predict ", this.accept[index], " from state ", index })); } return this.accept[index]; } char ch = (char) input.LA(1); if ((ch >= this.min[index]) && (ch <= this.max[index])) { int num4 = this.transition[index][ch - this.min[index]]; if (num4 < 0) { if (this.eot[index] < 0) { this.NoViableAlt(index, input); return 0; } if (this.debug) { Console.Error.WriteLine("EOT transition"); } index = this.eot[index]; input.Consume(); } else { index = num4; input.Consume(); } goto Label_0030; } if (this.eot[index] >= 0) { if (this.debug) { Console.Error.WriteLine("EOT transition"); } index = this.eot[index]; input.Consume(); goto Label_0030; } if ((ch == 0xffff) && (this.eof[index] >= 0)) { if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "accept via EOF; predict ", this.accept[this.eof[index]], " from ", this.eof[index] })); } return this.accept[this.eof[index]]; } if (this.debug) { Console.Error.WriteLine(string.Concat(new object[] { "min[", index, "]=", this.min[index] })); Console.Error.WriteLine(string.Concat(new object[] { "max[", index, "]=", this.max[index] })); Console.Error.WriteLine(string.Concat(new object[] { "eot[", index, "]=", this.eot[index] })); Console.Error.WriteLine(string.Concat(new object[] { "eof[", index, "]=", this.eof[index] })); for (int i = 0; i < this.transition[index].Length; i++) { Console.Error.Write(this.transition[index][i] + " "); } Console.Error.WriteLine(); } this.NoViableAlt(index, input); num6 = 0; } finally { input.Rewind(marker); } return num6; }
/** <summary> * From the input stream, predict what alternative will succeed * using this DFA (representing the covering regular approximation * to the underlying CFL). Return an alternative number 1..n. Throw * an exception upon error. * </summary> */ public virtual int Predict( IIntStream input ) { if (input == null) throw new ArgumentNullException("input"); DfaDebugMessage("Enter DFA.Predict for decision {0}", decisionNumber); int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { DfaDebugMessage("DFA {0} state {1} LA(1)={2}({3}), index={4}", decisionNumber, s, (char)input.LA(1), input.LA(1), input.Index); int specialState = special[s]; if ( specialState >= 0 ) { DfaDebugMessage("DFA {0} state {1} is special state {2}", decisionNumber, s, specialState); s = SpecialStateTransition( this, specialState, input ); DfaDebugMessage("DFA {0} returns from special state {1} to {2}", decisionNumber, specialState, s); if ( s == -1 ) { NoViableAlt( s, input ); return 0; } input.Consume(); continue; } if ( accept[s] >= 1 ) { DfaDebugMessage("accept; predict {0} from state {1}", accept[s], s); return accept[s]; } // look for a normal char transition char c = (char)input.LA( 1 ); // -1 == \uFFFF, all tokens fit in 65000 space if ( c >= min[s] && c <= max[s] ) { int snext = transition[s][c - min[s]]; // move to next state if ( snext < 0 ) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if ( eot[s] >= 0 ) { // EOT Transition to accept state? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt( s, input ); return 0; } s = snext; input.Consume(); continue; } if ( eot[s] >= 0 ) { // EOT Transition? DfaDebugMessage("EOT transition"); s = eot[s]; input.Consume(); continue; } if ( c == unchecked( (char)TokenTypes.EndOfFile ) && eof[s] >= 0 ) { // EOF Transition to accept state? DfaDebugMessage("accept via EOF; predict {0} from {1}", accept[eof[s]], eof[s]); return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol DfaDebugInvalidSymbol(s); NoViableAlt( s, input ); return 0; } } finally { input.Rewind( mark ); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { for (int i = input.LA(1); (i != -1) && (i != tokenType); i = input.LA(1)) { input.Consume(); } }
public bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { // we have no information about the follow; we can only consume // a single token and hope for the best return false; } // compute what can follow this grammar element reference if (follow.Member(Token.EOR_TOKEN_TYPE)) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(viableTokensFollowingThisRule); if (state.followingStackPointer >= 0) { // remove EOR if we're not the start symbol follow.Remove(Token.EOR_TOKEN_TYPE); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if ( follow.Member(input.LA(1)) || follow.Member(Token.EOR_TOKEN_TYPE) ) { return true; } return false; }
public bool MismatchIsUnwantedToken(IIntStream input, int ttype) { return input.LA(2) == ttype; }
public virtual void ConsumeUntil( IIntStream input, int tokenType ) { //System.out.println("consumeUntil "+tokenType); int ttype = input.LA( 1 ); while ( ttype != TokenTypes.EndOfFile && ttype != tokenType ) { input.Consume(); ttype = input.LA( 1 ); } }
public virtual void ConsumeUntil(IIntStream input, int tokenType) { int ttype = input.LA(1); while (ttype != Token.EOF && ttype != tokenType) { input.Consume(); ttype = input.LA(1); } }
/** <summary>Consume tokens until one matches the given token set</summary> */ public virtual void ConsumeUntil( IIntStream input, BitSet set ) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = input.LA( 1 ); while ( ttype != TokenTypes.EndOfFile && !set.Member( ttype ) ) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); input.Consume(); ttype = input.LA( 1 ); } }
public virtual bool MismatchIsMissingToken( IIntStream input, BitSet follow ) { if ( follow == null ) { // we have no information about the follow; we can only consume // a single token and hope for the best return false; } // compute what can follow this grammar element reference if ( follow.Member( TokenTypes.EndOfRule ) ) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or( viableTokensFollowingThisRule ); if ( state._fsp >= 0 ) { // remove EOR if we're not the start symbol follow.Remove( TokenTypes.EndOfRule ); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token //System.out.println("viable tokens="+follow.toString(getTokenNames())); //System.out.println("LT(1)="+((TokenStream)input).LT(1)); // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if ( follow.Member( input.LA( 1 ) ) || follow.Member( TokenTypes.EndOfRule ) ) { //System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting..."); return true; } return false; }
/// <summary> /// From the input stream, predict what alternative will succeed using this /// DFA (representing the covering regular approximation to the underlying CFL). /// </summary> /// <param name="input">Input stream</param> /// <returns>Return an alternative number 1..n. Throw an exception upon error.</returns> public int Predict(IIntStream input) { int mark = input.Mark(); // remember where decision started in input int s = 0; // we always start at s0 try { while (true) { int specialState = special[s]; if (specialState >= 0) { s = specialStateTransitionHandler(this, specialState, input); if (s == -1) { NoViableAlt(s, input); return 0; } input.Consume(); continue; } if (accept[s] >= 1) { return accept[s]; } // look for a normal char transition char c = (char)input.LA(1); // -1 == \uFFFF, all tokens fit in 65000 space if ((c >= min[s]) && (c <= max[s])) { int snext = transition[s][c - min[s]]; // move to next state if (snext < 0) { // was in range but not a normal transition // must check EOT, which is like the else clause. // eot[s]>=0 indicates that an EOT edge goes to another // state. if (eot[s] >= 0) // EOT Transition to accept state? { s = eot[s]; input.Consume(); // TODO: I had this as return accept[eot[s]] // which assumed here that the EOT edge always // went to an accept...faster to do this, but // what about predicated edges coming from EOT // target? continue; } NoViableAlt(s, input); return 0; } s = snext; input.Consume(); continue; } if (eot[s] >= 0) { // EOT Transition? s = eot[s]; input.Consume(); continue; } if ((c == (char)Token.EOF) && (eof[s] >= 0)) { // EOF Transition to accept state? return accept[eof[s]]; } // not in range and not EOF/EOT, must be invalid symbol NoViableAlt(s, input); return 0; } } finally { input.Rewind(mark); } }