protected MismatchedSetException(SerializationInfo info, StreamingContext context) : base(info, context) { if (info == null) throw new ArgumentNullException("info"); this._expecting = (BitSet)info.GetValue("Expecting", typeof(BitSet)); }
protected override object GetMissingSymbol(IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { String tokenText = null; if ( expectedTokenType==Token.EOF ) tokenText = "<missing EOF>"; else tokenText = "<missing " + TokenNames[expectedTokenType] + ">"; CommonToken t = new CommonToken(expectedTokenType, tokenText); IToken current = ((ITokenStream)input).LT(1); if (current.Type == Token.EOF) { current = ((ITokenStream)input).LT(-1); } t.line = current.Line; t.CharPositionInLine = current.CharPositionInLine; t.Channel = DEFAULT_TOKEN_CHANNEL; return t; }
public override object Match(IIntStream input, int ttype, BitSet follow) { object currentInputSymbol = this.GetCurrentInputSymbol(input); DumpSymbol(currentInputSymbol); if (input.LA(1) == ttype) { input.Consume(); this.state.errorRecovery = false; this.state.failed = false; return currentInputSymbol; } else { if (this.state.backtracking <= 0) return this.RecoverFromMismatchedToken(input, ttype, follow); this.state.failed = true; return currentInputSymbol; } }
protected virtual BitSet CombineFollows(bool exact) { int num = this.state._fsp; BitSet set = new BitSet(); for (int i = num; i >= 0; i--) { BitSet a = this.state.following[i]; set.OrInPlace(a); if (exact) { if (!a.Member(1)) { return set; } if (i > 0) { set.Remove(1); } } } return set; }
protected override object GetMissingSymbol(IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { string text = null; if (expectedTokenType == -1) { text = "<missing EOF>"; } else { text = "<missing " + this.TokenNames[expectedTokenType] + ">"; } CommonToken token = new CommonToken(expectedTokenType, text); IToken token2 = ((ITokenStream) input).LT(1); if (token2.Type == -1) { token2 = ((ITokenStream) input).LT(-1); } token.Line = token2.Line; token.CharPositionInLine = token2.CharPositionInLine; token.Channel = 0; return token; }
protected override object GetMissingSymbol( IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow ) { string tokenText = null; if ( expectedTokenType == TokenTypes.EndOfFile ) tokenText = "<missing EOF>"; else tokenText = "<missing " + TokenNames[expectedTokenType] + ">"; CommonToken t = new CommonToken( expectedTokenType, tokenText ); IToken current = ( (ITokenStream)input ).LT( 1 ); if ( current.Type == TokenTypes.EndOfFile ) { current = ( (ITokenStream)input ).LT( -1 ); } t.Line = current.Line; t.CharPositionInLine = current.CharPositionInLine; t.Channel = DefaultTokenChannel; t.InputStream = current.InputStream; return t; }
protected internal virtual BitSet CombineFollows(bool exact) { int top = state.followingStackPointer; BitSet followSet = new BitSet(); for (int i = top; i >= 0; i--) { BitSet localFollowSet = (BitSet)state.following[i]; followSet.OrInPlace(localFollowSet); if (exact) { // can we see end of rule? if (localFollowSet.Member(Token.EOR_TOKEN_TYPE)) { // Only leave EOR in set if at top (start rule); this lets // us know if have to include follow(start rule); i.e., EOF if (i > 0) { followSet.Remove(Token.EOR_TOKEN_TYPE); } } else { // can't see end of rule, quit break; } } } return followSet; }
/** Not currently used */ public virtual object RecoverFromMismatchedSet(IIntStream input, RecognitionException e, BitSet follow) { if (MismatchIsMissingToken(input, follow)) { ReportError(e); // we don't know how to conjure up a token for sets yet return GetMissingSymbol(input, e, Token.INVALID_TOKEN_TYPE, follow); } // TODO do single token deletion like above for Token mismatch throw e; }
public bool MismatchIsMissingToken(IIntStream input, BitSet follow) { if (follow == null) { // we have no information about the follow; we can only consume // a single token and hope for the best return false; } // compute what can follow this grammar element reference if (follow.Member(Token.EOR_TOKEN_TYPE)) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or(viableTokensFollowingThisRule); if (state.followingStackPointer >= 0) { // remove EOR if we're not the start symbol follow.Remove(Token.EOR_TOKEN_TYPE); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if ( follow.Member(input.LA(1)) || follow.Member(Token.EOR_TOKEN_TYPE) ) { return true; } return false; }
public MismatchedNotSetException(BitSet expecting, IIntStream input) : base(expecting, input) { }
/** <summary> * Given a start and stop index, return a List of all tokens in * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. * </summary> */ public virtual IList<IToken> GetTokens( int start, int stop, BitSet types ) { if ( p == -1 ) { FillBuffer(); } if ( stop >= tokens.Count ) { stop = tokens.Count - 1; } if ( start < 0 ) { start = 0; } if ( start > stop ) { return null; } // list = tokens[start:stop]:{Token t, t.getType() in types} IList<IToken> filteredTokens = new List<IToken>(); for ( int i = start; i <= stop; i++ ) { IToken t = tokens[i]; if ( types == null || types.Member( t.Type ) ) { filteredTokens.Add( t ); } } if ( filteredTokens.Count == 0 ) { filteredTokens = null; } return filteredTokens; }
public void OrInPlace( BitSet a ) { if ( a == null ) { return; } // If this is smaller than a, grow this first if ( a._bits.Length > _bits.Length ) { SetSize( a._bits.Length ); } int min = Math.Min( _bits.Length, a._bits.Length ); for ( int i = min - 1; i >= 0; i-- ) { _bits[i] |= a._bits[i]; } }
/** <summary>Attempt to recover from a single missing or extra token.</summary> * * EXTRA TOKEN * * LA(1) is not what we are looking for. If LA(2) has the right token, * however, then assume LA(1) is some extra spurious token. Delete it * and LA(2) as if we were doing a normal match(), which advances the * input. * * MISSING TOKEN * * If current token is consistent with what could come after * ttype then it is ok to "insert" the missing token, else throw * exception For example, Input "i=(3;" is clearly missing the * ')'. When the parser returns from the nested call to expr, it * will have call chain: * * stat -> expr -> atom * * and it will be trying to match the ')' at this point in the * derivation: * * => ID '=' '(' INT ')' ('+' atom)* ';' * ^ * match() will see that ';' doesn't match ')' and report a * mismatched token error. To recover, it sees that LA(1)==';' * is in the set of tokens that can follow the ')' token * reference in rule atom. It can assume that you forgot the ')'. */ protected virtual object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; // if next token is what we are looking for then "delete" this token if (MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input, TokenNames); /* * System.err.println("recoverFromMismatchedToken deleting "+ * ((TokenStream)input).LT(1)+ * " since "+((TokenStream)input).LT(2)+" is what we want"); */ BeginResync(); input.Consume(); // simply delete extra token EndResync(); ReportError(e); // report after consuming so AW sees the token in the exception // we want to return the token we're actually matching object matchedSymbol = GetCurrentInputSymbol(input); input.Consume(); // move past ttype token as if all were ok return(matchedSymbol); } // can't recover with single token deletion, try insertion if (MismatchIsMissingToken(input, follow)) { object inserted = GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, inserted); ReportError(e); // report after inserting so AW sees the token in the exception return(inserted); } // even that didn't work; must throw the exception e = new MismatchedTokenException(ttype, input, TokenNames); throw e; }
public MismatchedNotSetException(string message, BitSet expecting, IIntStream input, Exception innerException) : base(message, expecting, input, innerException) { }
/// <summary> /// Push a rule's follow set using our own hardcoded stack /// </summary> /// <param name="fset"></param> protected void PushFollow(BitSet fset) { if ((state.followingStackPointer + 1) >= state.following.Length) { BitSet[] f = new BitSet[state.following.Length * 2]; Array.Copy(state.following, 0, f, 0, state.following.Length); state.following = f; } state.following[++state.followingStackPointer] = fset; }
/** Given a start and stop index, return a List of all tokens in * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. */ public virtual List<IToken> GetTokens(int start, int stop, BitSet types) { if (_p == -1) Setup(); if (stop >= _tokens.Count) stop = _tokens.Count - 1; if (start < 0) start = 0; if (start > stop) return null; // list = tokens[start:stop]:{Token t, t.getType() in types} List<IToken> filteredTokens = new List<IToken>(); for (int i = start; i <= stop; i++) { IToken t = _tokens[i]; if (types == null || types.Member(t.Type)) { filteredTokens.Add(t); } } if (filteredTokens.Count == 0) { filteredTokens = null; } return filteredTokens; }
public virtual List<IToken> GetTokens(int start, int stop, BitSet types) { if (this._p == -1) { this.Setup(); } if (stop >= this._tokens.Count) { stop = this._tokens.Count - 1; } if (start < 0) { start = 0; } if (start > stop) { return null; } List<IToken> list = new List<IToken>(); for (int i = start; i <= stop; i++) { IToken item = this._tokens[i]; if ((types == null) || types.Member(item.Type)) { list.Add(item); } } if (list.Count == 0) { list = null; } return list; }
public static BitSet Of( int el ) { BitSet s = new BitSet( el + 1 ); s.Add( el ); return s; }
/// <summary>We have DOWN/UP nodes in the stream that have no line info; override. /// plus we want to alter the exception type. Don't try to recover /// from tree parser errors inline... /// </summary> protected internal override object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { throw new MismatchedTreeNodeException(ttype, (ITreeNodeStream)input); }
public override object RecoverFromMismatchedSet(IIntStream input, RecognitionException e, BitSet follow) { throw e; }
protected override object GetMissingSymbol(IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { string tokenText = "<missing " + TokenNames[expectedTokenType] + ">"; return new CommonTree(new CommonToken(expectedTokenType, tokenText)); }
public new void RecoverFromMismatchedSet(IIntStream input, RecognitionException e, BitSet follow) { throw e; }
/// <summary> /// Match current input symbol against ttype. Attempt /// single token insertion or deletion error recovery. If /// that fails, throw MismatchedTokenException. /// </summary> /// <remarks> /// To turn off single token insertion or deletion error /// recovery, override RecoverFromMismatchedToken() and have it call /// pthrow an exception. See TreeParser.RecoverFromMismatchedToken(). /// This way any error in a rule will cause an exception and /// immediate exit from rule. Rule would recover by resynchronizing /// to the set of symbols that can follow rule ref. /// </remarks> public virtual object Match(IIntStream input, int ttype, BitSet follow) { object matchedSymbol = GetCurrentInputSymbol(input); if (input.LA(1) == ttype) { input.Consume(); state.errorRecovery = false; state.failed = false; return matchedSymbol; } if (state.backtracking > 0) { state.failed = true; return matchedSymbol; } matchedSymbol = RecoverFromMismatchedToken(input, ttype, follow); return matchedSymbol; }
protected override object GetMissingSymbol(IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { object o = base.GetMissingSymbol(input, e, expectedTokenType, follow); dbg.ConsumeNode(o); return o; }
/// <summary> /// Attempt to Recover from a single missing or extra token. /// </summary> /// <remarks> /// EXTRA TOKEN /// /// LA(1) is not what we are looking for. If LA(2) has the right token, /// however, then assume LA(1) is some extra spurious token. Delete it /// and LA(2) as if we were doing a normal Match(), which advances the /// input. /// /// MISSING TOKEN /// /// If current token is consistent with what could come after /// ttype then it is ok to "insert" the missing token, else throw /// exception For example, Input "i=(3;" is clearly missing the /// ')'. When the parser returns from the nested call to expr, it /// will have call chain: /// /// stat -> expr -> atom /// /// and it will be trying to Match the ')' at this point in the /// derivation: /// /// => ID '=' '(' INT ')' ('+' atom)* ';' /// ^ /// Match() will see that ';' doesn't Match ')' and report a /// mismatched token error. To Recover, it sees that LA(1)==';' /// is in the set of tokens that can follow the ')' token /// reference in rule atom. It can assume that you forgot the ')'. /// </remarks> protected internal virtual Object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { RecognitionException e = null; // if next token is what we are looking for then "delete" this token if (MismatchIsUnwantedToken(input, ttype)) { e = new UnwantedTokenException(ttype, input); BeginResync(); input.Consume(); // simply delete extra token EndResync(); ReportError(e); // report after consuming so AW sees the token in the exception // we want to return the token we're actually matching object matchedSymbol = GetCurrentInputSymbol(input); input.Consume(); // move past ttype token as if all were ok return matchedSymbol; } // can't recover with single token deletion, try insertion if (MismatchIsMissingToken(input, follow)) { object inserted = GetMissingSymbol(input, e, ttype, follow); e = new MissingTokenException(ttype, input, inserted); ReportError(e); // report after inserting so AW sees the token in the exception return inserted; } // even that didn't work; must throw the exception e = new MismatchedTokenException(ttype, input); throw e; }
public static BitSet Of( int a, int b ) { BitSet s = new BitSet( Math.Max( a, b ) + 1 ); s.Add( a ); s.Add( b ); return s; }
/// <summary>Consume tokens until one matches the given token set </summary> public virtual void ConsumeUntil(IIntStream input, BitSet set) { int ttype = input.LA(1); while (ttype != Token.EOF && !set.Member(ttype)) { input.Consume(); ttype = input.LA(1); } }
public static BitSet Of( int a, int b, int c, int d ) { BitSet s = new BitSet(); s.Add( a ); s.Add( b ); s.Add( c ); s.Add( d ); return s; }
/// <summary> /// Conjure up a missing token during error recovery. /// </summary> /// <remarks> /// The recognizer attempts to recover from single missing /// symbols. But, actions might refer to that missing symbol. /// For example, x=ID {f($x);}. The action clearly assumes /// that there has been an identifier matched previously and that /// $x points at that token. If that token is missing, but /// the next token in the stream is what we want we assume that /// this token is missing and we keep going. Because we /// have to return some token to replace the missing token, /// we have to conjure one up. This method gives the user control /// over the tokens returned for missing tokens. Mostly, /// you will want to create something special for identifier /// tokens. For literals such as '{' and ',', the default /// action in the parser or tree parser works. It simply creates /// a CommonToken of the appropriate type. The text will be the token. /// If you change what tokens must be created by the lexer, /// override this method to create the appropriate tokens. /// </remarks> protected virtual object GetMissingSymbol(IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { return null; }
/** <summary>return this | a in a new set</summary> */ public BitSet Or( BitSet a ) { if ( a == null ) { return this; } BitSet s = (BitSet)this.Clone(); s.OrInPlace( a ); return s; }
protected override object RecoverFromMismatchedToken(IIntStream input, int ttype, BitSet follow) { throw new MismatchedTokenException(ttype, input); }
public virtual List <IToken> GetTokens(int start, int stop, int ttype) { return(GetTokens(start, stop, BitSet.Of(ttype))); }