/** Given a start and stop index, return a List of all tokens in
* the token type BitSet. Return null if no tokens were found. This
* method looks at both on and off channel tokens.
*/
public virtual List<IToken> GetTokens(int start, int stop, BitSet types)
{
if (_p == -1)
Setup();
if (stop >= _tokens.Count)
stop = _tokens.Count - 1;
if (start < 0)
start = 0;
if (start > stop)
return null;
// list = tokens[start:stop]:{Token t, t.getType() in types}
List<IToken> filteredTokens = new List<IToken>();
for (int i = start; i <= stop; i++)
{
IToken t = _tokens[i];
if (types == null || types.Member(t.Type))
{
filteredTokens.Add(t);
}
}
if (filteredTokens.Count == 0)
{
filteredTokens = null;
}
return filteredTokens;
}
protected override object GetMissingSymbol( IIntStream input,
RecognitionException e,
int expectedTokenType,
BitSet follow)
{
string tokenText = null;
if ( expectedTokenType == TokenTypes.EndOfFile )
tokenText = "<missing EOF>";
else
tokenText = "<missing " + TokenNames[expectedTokenType] + ">";
CommonToken t = new CommonToken( expectedTokenType, tokenText );
IToken current = ( (ITokenStream)input ).LT( 1 );
if ( current.Type == TokenTypes.EndOfFile )
{
current = ( (ITokenStream)input ).LT( -1 );
}
t.Line = current.Line;
t.CharPositionInLine = current.CharPositionInLine;
t.Channel = DefaultTokenChannel;
t.InputStream = current.InputStream;
return t;
}
public MismatchedNotSetException(BitSet expecting, IIntStream input)
: base(expecting, input)
{
}
public MismatchedNotSetException(string message, BitSet expecting, IIntStream input, Exception innerException)
: base(message, expecting, input, innerException)
{
}
/** <summary>return this | a in a new set</summary> */
public BitSet Or( BitSet a )
{
if ( a == null )
{
return this;
}
BitSet s = (BitSet)this.Clone();
s.OrInPlace( a );
return s;
}
public MismatchedSetException(string message, BitSet expecting, IIntStream input, Exception innerException)
: base(message, input, innerException)
{
this._expecting = expecting;
}
/** <summary>
* Match current input symbol against ttype. Attempt
* single token insertion or deletion error recovery. If
* that fails, throw MismatchedTokenException.
* </summary>
*
* <remarks>
* To turn off single token insertion or deletion error
* recovery, override recoverFromMismatchedToken() and have it
* throw an exception. See TreeParser.recoverFromMismatchedToken().
* This way any error in a rule will cause an exception and
* immediate exit from rule. Rule would recover by resynchronizing
* to the set of symbols that can follow rule ref.
* </remarks>
*/
public virtual object Match( IIntStream input, int ttype, BitSet follow )
{
//System.out.println("match "+((TokenStream)input).LT(1));
object matchedSymbol = GetCurrentInputSymbol( input );
if ( input.LA( 1 ) == ttype )
{
input.Consume();
state.errorRecovery = false;
state.failed = false;
return matchedSymbol;
}
if ( state.backtracking > 0 )
{
state.failed = true;
return matchedSymbol;
}
matchedSymbol = RecoverFromMismatchedToken( input, ttype, follow );
return matchedSymbol;
}
public static BitSet Of( int a, int b )
{
BitSet s = new BitSet( Math.Max( a, b ) + 1 );
s.Add( a );
s.Add( b );
return s;
}
/** <summary>Attempt to recover from a single missing or extra token.</summary>
*
* EXTRA TOKEN
*
* LA(1) is not what we are looking for. If LA(2) has the right token,
* however, then assume LA(1) is some extra spurious token. Delete it
* and LA(2) as if we were doing a normal match(), which advances the
* input.
*
* MISSING TOKEN
*
* If current token is consistent with what could come after
* ttype then it is ok to "insert" the missing token, else throw
* exception For example, Input "i=(3;" is clearly missing the
* ')'. When the parser returns from the nested call to expr, it
* will have call chain:
*
* stat -> expr -> atom
*
* and it will be trying to match the ')' at this point in the
* derivation:
*
* => ID '=' '(' INT ')' ('+' atom)* ';'
* ^
* match() will see that ';' doesn't match ')' and report a
* mismatched token error. To recover, it sees that LA(1)==';'
* is in the set of tokens that can follow the ')' token
* reference in rule atom. It can assume that you forgot the ')'.
*/
protected virtual object RecoverFromMismatchedToken( IIntStream input, int ttype, BitSet follow )
{
RecognitionException e = null;
// if next token is what we are looking for then "delete" this token
if ( MismatchIsUnwantedToken( input, ttype ) )
{
e = new UnwantedTokenException( ttype, input, TokenNames );
/*
System.err.println("recoverFromMismatchedToken deleting "+
((TokenStream)input).LT(1)+
" since "+((TokenStream)input).LT(2)+" is what we want");
*/
BeginResync();
input.Consume(); // simply delete extra token
EndResync();
ReportError( e ); // report after consuming so AW sees the token in the exception
// we want to return the token we're actually matching
object matchedSymbol = GetCurrentInputSymbol( input );
input.Consume(); // move past ttype token as if all were ok
return matchedSymbol;
}
// can't recover with single token deletion, try insertion
if ( MismatchIsMissingToken( input, follow ) )
{
object inserted = GetMissingSymbol( input, e, ttype, follow );
e = new MissingTokenException( ttype, input, inserted );
ReportError( e ); // report after inserting so AW sees the token in the exception
return inserted;
}
// even that didn't work; must throw the exception
e = new MismatchedTokenException(ttype, input, TokenNames);
throw e;
}
/** <summary>Consume tokens until one matches the given token set</summary> */
public virtual void ConsumeUntil( IIntStream input, BitSet set )
{
//System.out.println("consumeUntil("+set.toString(getTokenNames())+")");
int ttype = input.LA( 1 );
while ( ttype != TokenTypes.EndOfFile && !set.Member( ttype ) )
{
//System.out.println("consume during recover LA(1)="+getTokenNames()[input.LA(1)]);
input.Consume();
ttype = input.LA( 1 );
}
}
/** <summary>Push a rule's follow set using our own hardcoded stack</summary> */
protected void PushFollow( BitSet fset )
{
if ( ( state._fsp + 1 ) >= state.following.Length )
{
Array.Resize(ref state.following, state.following.Length * 2);
}
state.following[++state._fsp] = fset;
}
/** <summary>Conjure up a missing token during error recovery.</summary>
*
* <remarks>
* The recognizer attempts to recover from single missing
* symbols. But, actions might refer to that missing symbol.
* For example, x=ID {f($x);}. The action clearly assumes
* that there has been an identifier matched previously and that
* $x points at that token. If that token is missing, but
* the next token in the stream is what we want we assume that
* this token is missing and we keep going. Because we
* have to return some token to replace the missing token,
* we have to conjure one up. This method gives the user control
* over the tokens returned for missing tokens. Mostly,
* you will want to create something special for identifier
* tokens. For literals such as '{' and ',', the default
* action in the parser or tree parser works. It simply creates
* a CommonToken of the appropriate type. The text will be the token.
* If you change what tokens must be created by the lexer,
* override this method to create the appropriate tokens.
* </remarks>
*/
protected virtual object GetMissingSymbol( IIntStream input,
RecognitionException e,
int expectedTokenType,
BitSet follow)
{
return null;
}
protected MismatchedSetException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
if (info == null)
throw new ArgumentNullException("info");
this._expecting = (BitSet)info.GetValue("Expecting", typeof(BitSet));
}
/** <summary>
* Given a start and stop index, return a List of all tokens in
* the token type BitSet. Return null if no tokens were found. This
* method looks at both on and off channel tokens.
* </summary>
*/
public virtual IList<IToken> GetTokens( int start, int stop, BitSet types )
{
if ( p == -1 )
{
FillBuffer();
}
if ( stop >= tokens.Count )
{
stop = tokens.Count - 1;
}
if ( start < 0 )
{
start = 0;
}
if ( start > stop )
{
return null;
}
// list = tokens[start:stop]:{Token t, t.getType() in types}
IList<IToken> filteredTokens = new List<IToken>();
for ( int i = start; i <= stop; i++ )
{
IToken t = tokens[i];
if ( types == null || types.Member( t.Type ) )
{
filteredTokens.Add( t );
}
}
if ( filteredTokens.Count == 0 )
{
filteredTokens = null;
}
return filteredTokens;
}
public virtual bool MismatchIsMissingToken( IIntStream input, BitSet follow )
{
if ( follow == null )
{
// we have no information about the follow; we can only consume
// a single token and hope for the best
return false;
}
// compute what can follow this grammar element reference
if ( follow.Member( TokenTypes.EndOfRule ) )
{
BitSet viableTokensFollowingThisRule = ComputeContextSensitiveRuleFOLLOW();
follow = follow.Or( viableTokensFollowingThisRule );
if ( state._fsp >= 0 )
{ // remove EOR if we're not the start symbol
follow.Remove( TokenTypes.EndOfRule );
}
}
// if current token is consistent with what could come after set
// then we know we're missing a token; error recovery is free to
// "insert" the missing token
//System.out.println("viable tokens="+follow.toString(getTokenNames()));
//System.out.println("LT(1)="+((TokenStream)input).LT(1));
// BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
// in follow set to indicate that the fall of the start symbol is
// in the set (EOF can follow).
if ( follow.Member( input.LA( 1 ) ) || follow.Member( TokenTypes.EndOfRule ) )
{
//System.out.println("LT(1)=="+((TokenStream)input).LT(1)+" is consistent with what follows; inserting...");
return true;
}
return false;
}
public static BitSet Of( int el )
{
BitSet s = new BitSet( el + 1 );
s.Add( el );
return s;
}
/** Not currently used */
public virtual object RecoverFromMismatchedSet( IIntStream input,
RecognitionException e,
BitSet follow)
{
if ( MismatchIsMissingToken( input, follow ) )
{
// System.out.println("missing token");
ReportError( e );
// we don't know how to conjure up a token for sets yet
return GetMissingSymbol( input, e, TokenTypes.Invalid, follow );
}
// TODO do single token deletion like above for Token mismatch
throw e;
}
public static BitSet Of( int a, int b, int c, int d )
{
BitSet s = new BitSet();
s.Add( a );
s.Add( b );
s.Add( c );
s.Add( d );
return s;
}
// what is exact? it seems to only add sets from above on stack
// if EOR is in set i. When it sees a set w/o EOR, it stops adding.
// Why would we ever want them all? Maybe no viable alt instead of
// mismatched token?
protected virtual BitSet CombineFollows(bool exact)
{
int top = state._fsp;
BitSet followSet = new BitSet();
for ( int i = top; i >= 0; i-- )
{
BitSet localFollowSet = (BitSet)state.following[i];
/*
System.out.println("local follow depth "+i+"="+
localFollowSet.toString(getTokenNames())+")");
*/
followSet.OrInPlace( localFollowSet );
if ( exact )
{
// can we see end of rule?
if ( localFollowSet.Member( TokenTypes.EndOfRule ) )
{
// Only leave EOR in set if at top (start rule); this lets
// us know if have to include follow(start rule); i.e., EOF
if ( i > 0 )
{
followSet.Remove( TokenTypes.EndOfRule );
}
}
else
{ // can't see end of rule, quit
break;
}
}
}
return followSet;
}
public void OrInPlace( BitSet a )
{
if ( a == null )
{
return;
}
// If this is smaller than a, grow this first
if ( a._bits.Length > _bits.Length )
{
SetSize( a._bits.Length );
}
int min = Math.Min( _bits.Length, a._bits.Length );
for ( int i = min - 1; i >= 0; i-- )
{
_bits[i] |= a._bits[i];
}
}
public MismatchedSetException( BitSet expecting, IIntStream input )
: base(input)
{
this._expecting = expecting;
}
