/** <summary> * The standard method called to automatically emit a token at the * outermost lexical rule. The token object should point into the * char buffer start..stop. If there is a text override in 'text', * use that to set the token's text. Override this method to emit * custom Token objects. * </summary> * * <remarks> * If you are building trees, then you should also override * Parser or TreeParser.getMissingSymbol(). * </remarks> */ public virtual IToken Emit() { IToken t = new CommonToken( input, state.type, state.channel, state.tokenStartCharIndex, CharIndex - 1 ); t.Line = state.tokenStartLine; t.Text = state.text; t.CharPositionInLine = state.tokenStartCharPositionInLine; Emit( t ); return t; }
/** <summary>Return a token from this source; i.e., match a token on the char stream.</summary> */ public virtual IToken NextToken() { for ( ; ; ) { state.token = null; state.channel = TokenChannels.Default; state.tokenStartCharIndex = input.Index; state.tokenStartCharPositionInLine = input.CharPositionInLine; state.tokenStartLine = input.Line; state.text = null; if ( input.LA( 1 ) == CharStreamConstants.EndOfFile ) { IToken eof = new CommonToken((ICharStream)input, CharStreamConstants.EndOfFile, TokenChannels.Default, input.Index, input.Index); eof.Line = Line; eof.CharPositionInLine = CharPositionInLine; return eof; } try { ParseNextToken(); if ( state.token == null ) { Emit(); } else if ( state.token == Tokens.Skip ) { continue; } return state.token; } catch (MismatchedRangeException mre) { ReportError(mre); // MatchRange() routine has already called recover() } catch (MismatchedTokenException mte) { ReportError(mte); // Match() routine has already called recover() } catch ( RecognitionException re ) { ReportError( re ); Recover( re ); // throw out current char and try again } } }
protected override object GetMissingSymbol( IIntStream input, RecognitionException e, int expectedTokenType, BitSet follow) { string tokenText = null; if ( expectedTokenType == TokenTypes.EndOfFile ) tokenText = "<missing EOF>"; else tokenText = "<missing " + TokenNames[expectedTokenType] + ">"; CommonToken t = new CommonToken( expectedTokenType, tokenText ); IToken current = ( (ITokenStream)input ).LT( 1 ); if ( current.Type == TokenTypes.EndOfFile ) { current = ( (ITokenStream)input ).LT( -1 ); } t.Line = current.Line; t.CharPositionInLine = current.CharPositionInLine; t.Channel = DefaultTokenChannel; t.InputStream = current.InputStream; return t; }