/** Given a start and stop index, return a List of all tokens in * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. */ public virtual List<IToken> GetTokens(int start, int stop, BitSet types) { if (_p == -1) Setup(); if (stop >= _tokens.Count) stop = _tokens.Count - 1; if (start < 0) start = 0; if (start > stop) return null; // list = tokens[start:stop]:{Token t, t.getType() in types} List<IToken> filteredTokens = new List<IToken>(); for (int i = start; i <= stop; i++) { IToken t = _tokens[i]; if (types == null || types.Member(t.Type)) { filteredTokens.Add(t); } } if (filteredTokens.Count == 0) { filteredTokens = null; } return filteredTokens; }
protected override object GetMissingSymbol( IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { string tokenText = null; if ( expectedTokenType == TokenTypes.EndOfFile ) tokenText = "<missing EOF>"; else tokenText = "<missing " + TokenNames[expectedTokenType] + ">"; CommonToken t = new CommonToken( expectedTokenType, tokenText ); IToken current = ( (ITokenStream)input ).LT( 1 ); if ( current.Type == TokenTypes.EndOfFile ) { current = ( (ITokenStream)input ).LT( -1 ); } t.Line = current.Line; t.CharPositionInLine = current.CharPositionInLine; t.Channel = DefaultTokenChannel; t.InputStream = current.InputStream; return t; }
public MismatchedNotSetException(BitSet expecting, IIntStream input) : base(expecting, input) { }
public MismatchedNotSetException(string message, BitSet expecting, IIntStream input, Exception innerException) : base(message, expecting, input, innerException) { }
/** <summary>return this | a in a new set</summary> */ public BitSet Or( BitSet a ) { if ( a == null ) { return this; } BitSet s = (BitSet)this.Clone(); s.OrInPlace( a ); return s; }
public MismatchedSetException(string message, BitSet expecting, IIntStream input, Exception innerException) : base(message, input, innerException) { this._expecting = expecting; }
/** <summary> * Match current input symbol against ttype. Attempt * single token insertion or deletion error recovery. If * that fails, throw MismatchedTokenException. * </summary> * * <remarks> * To turn off single token insertion or deletion error * recovery, override recoverFromMismatchedToken() and have it * throw an exception. See TreeParser.recoverFromMismatchedToken(). * This way any error in a rule will cause an exception and * immediate exit from rule. Rule would recover by resynchronizing * to the set of symbols that can follow rule ref. * </remarks> */ public virtual object Match( IIntStream input, int ttype, BitSet follow ) { //System.out.println("match "+((TokenStream)input).LT(1)); object matchedSymbol = GetCurrentInputSymbol( input ); if ( input.LA( 1 ) == ttype ) { input.Consume(); state.errorRecovery = false; state.failed = false; return matchedSymbol; } if ( state.backtracking > 0 ) { state.failed = true; return matchedSymbol; } matchedSymbol = RecoverFromMismatchedToken( input, ttype, follow ); return matchedSymbol; }
public static BitSet Of( int a, int b ) { BitSet s = new BitSet( Math.Max( a, b ) + 1 ); s.Add( a ); s.Add( b ); return s; }
/** <summary>Attempt to recover from a single missing or extra token.</summary> * * EXTRA TOKEN * * LA(1) is not what we are looking for. If LA(2) has the right token, * however, then assume LA(1) is some extra spurious token. Delete it * and LA(2) as if we were doing a normal match(), which advances the * input. * * MISSING TOKEN * * If current token is consistent with what could come after * ttype then it is ok to "insert" the missing token, else throw * exception For example, Input "i=(3;" is clearly missing the * ')'. When the parser returns from the nested call to expr, it * will have call chain: * * stat -> expr -> atom * * and it will be trying to match the ')' at this point in the * derivation: * * => ID '=' '(' INT ')' ('+' atom)* ';' * ^ * match() will see that ';' doesn't match ')' and report a * mismatched token error. To recover, it sees that LA(1)==';' * is in the set of tokens that can follow the ')' token * reference in rule atom. It can assume that you forgot the ')'. */ protected virtual object RecoverFromMismatchedToken( IIntStream input, int ttype, BitSet follow ) { RecognitionException e = null; // if next token is what we are looking for then "delete" this token if ( MismatchIsUnwantedToken( input, ttype ) ) { e = new UnwantedTokenException( ttype, input, TokenNames ); /* System.err.println("recoverFromMismatchedToken deleting "+ ((TokenStream)input).LT(1)+ " since "+((TokenStream)input).LT(2)+" is what we want"); */ BeginResync(); input.Consume(); // simply delete extra token EndResync(); ReportError( e ); // report after consuming so AW sees the token in the exception // we want to return the token we're actually matching object matchedSymbol = GetCurrentInputSymbol( input ); input.Consume(); // move past ttype token as if all were ok return matchedSymbol; } // can't recover with single token deletion, try insertion if ( MismatchIsMissingToken( input, follow ) ) { object inserted = GetMissingSymbol( input, e, ttype, follow ); e = new MissingTokenException( ttype, input, inserted ); ReportError( e ); // report after inserting so AW sees the token in the exception return inserted; } // even that didn't work; must throw the exception e = new MismatchedTokenException(ttype, input, TokenNames); throw e; }
/** <summary>Consume tokens until one matches the given token set</summary> */ public virtual void ConsumeUntil( IIntStream input, BitSet set ) { //System.out.println("consumeUntil("+set.toString(getTokenNames())+")"); int ttype = input.LA( 1 ); while ( ttype != TokenTypes.EndOfFile && !set.Member( ttype ) ) { //System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]); input.Consume(); ttype = input.LA( 1 ); } }
/** <summary>Push a rule's follow set using our own hardcoded stack</summary> */ protected void PushFollow( BitSet fset ) { if ( ( state._fsp + 1 ) >= state.following.Length ) { Array.Resize(ref state.following, state.following.Length * 2); } state.following[++state._fsp] = fset; }
/** <summary>Conjure up a missing token during error recovery.</summary> * * <remarks> * The recognizer attempts to recover from single missing * symbols. But, actions might refer to that missing symbol. * For example, x=ID {f($x);}. The action clearly assumes * that there has been an identifier matched previously and that * $x points at that token. If that token is missing, but * the next token in the stream is what we want we assume that * this token is missing and we keep going. Because we * have to return some token to replace the missing token, * we have to conjure one up. This method gives the user control * over the tokens returned for missing tokens. Mostly, * you will want to create something special for identifier * tokens. For literals such as '{' and ',', the default * action in the parser or tree parser works. It simply creates * a CommonToken of the appropriate type. The text will be the token. * If you change what tokens must be created by the lexer, * override this method to create the appropriate tokens. * </remarks> */ protected virtual object GetMissingSymbol( IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { return null; }
protected MismatchedSetException(SerializationInfo info, StreamingContext context) : base(info, context) { if (info == null) throw new ArgumentNullException("info"); this._expecting = (BitSet)info.GetValue("Expecting", typeof(BitSet)); }
/** <summary> * Given a start and stop index, return a List of all tokens in * the token type BitSet. Return null if no tokens were found. This * method looks at both on and off channel tokens. * </summary> */ public virtual IList<IToken> GetTokens( int start, int stop, BitSet types ) { if ( p == -1 ) { FillBuffer(); } if ( stop >= tokens.Count ) { stop = tokens.Count - 1; } if ( start < 0 ) { start = 0; } if ( start > stop ) { return null; } // list = tokens[start:stop]:{Token t, t.getType() in types} IList<IToken> filteredTokens = new List<IToken>(); for ( int i = start; i <= stop; i++ ) { IToken t = tokens[i]; if ( types == null || types.Member( t.Type ) ) { filteredTokens.Add( t ); } } if ( filteredTokens.Count == 0 ) { filteredTokens = null; } return filteredTokens; }
public virtual bool MismatchIsMissingToken( IIntStream input, BitSet follow ) { if ( follow == null ) { // we have no information about the follow; we can only consume // a single token and hope for the best return false; } // compute what can follow this grammar element reference if ( follow.Member( TokenTypes.EndOfRule ) ) { BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW(); follow = follow.Or( viableTokensFollowingThisRule ); if ( state._fsp >= 0 ) { // remove EOR if we're not the start symbol follow.Remove( TokenTypes.EndOfRule ); } } // if current token is consistent with what could come after set // then we know we're missing a token; error recovery is free to // "insert" the missing token //System.out.println("viable tokens="+follow.toString(getTokenNames())); //System.out.println("LT(1)="+((TokenStream)input).LT(1)); // BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR // in follow set to indicate that the fall of the start symbol is // in the set (EOF can follow). if ( follow.Member( input.LA( 1 ) ) || follow.Member( TokenTypes.EndOfRule ) ) { //System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting..."); return true; } return false; }
public static BitSet Of( int el ) { BitSet s = new BitSet( el + 1 ); s.Add( el ); return s; }
/** Not currently used */ public virtual object RecoverFromMismatchedSet( IIntStream input, RecognitionException e, BitSet follow) { if ( MismatchIsMissingToken( input, follow ) ) { // System.out.println("missing token"); ReportError( e ); // we don't know how to conjure up a token for sets yet return GetMissingSymbol( input, e, TokenTypes.Invalid, follow ); } // TODO do single token deletion like above for Token mismatch throw e; }
public static BitSet Of( int a, int b, int c, int d ) { BitSet s = new BitSet(); s.Add( a ); s.Add( b ); s.Add( c ); s.Add( d ); return s; }
// what is exact? it seems to only add sets from above on stack // if EOR is in set i. When it sees a set w/o EOR, it stops adding. // Why would we ever want them all? Maybe no viable alt instead of // mismatched token? protected virtual BitSet CombineFollows(bool exact) { int top = state._fsp; BitSet followSet = new BitSet(); for ( int i = top; i >= 0; i-- ) { BitSet localFollowSet = (BitSet)state.following[i]; /* System.out.println("local follow depth "+i+"="+ localFollowSet.toString(getTokenNames())+")"); */ followSet.OrInPlace( localFollowSet ); if ( exact ) { // can we see end of rule? if ( localFollowSet.Member( TokenTypes.EndOfRule ) ) { // Only leave EOR in set if at top (start rule); this lets // us know if have to include follow(start rule); i.e., EOF if ( i > 0 ) { followSet.Remove( TokenTypes.EndOfRule ); } } else { // can't see end of rule, quit break; } } } return followSet; }
public void OrInPlace( BitSet a ) { if ( a == null ) { return; } // If this is smaller than a, grow this first if ( a._bits.Length > _bits.Length ) { SetSize( a._bits.Length ); } int min = Math.Min( _bits.Length, a._bits.Length ); for ( int i = min - 1; i >= 0; i-- ) { _bits[i] |= a._bits[i]; } }
public MismatchedSetException( BitSet expecting, IIntStream input ) : base(input) { this._expecting = expecting; }