public void SetEndPosition(Lexer lexer) { EndLine = lexer[0, false].Line; LexerToken token = lexer[0, false]; EndLinePos = token.LinePos + (token.Token == null ? 0 : token.Token.Length); }
/// <summary> Gets the symbol the specified number of tokens ahead without advancing the current token. </summary> /// <remarks> If the token is not a symbol, returns an empty string. </remarks> public LexerToken PeekToken(int count) { LexerToken token = this[count]; if (token.Type == TokenType.Error) { throw token.Error; } return(token); }
/// <summary> The token a specific number of tokens ahead of the current token. </summary> public LexerToken this[int index, bool checkActive] { get { System.Diagnostics.Debug.Assert(index < LookAheadCount, "Lexer look ahead attempt exceeds maximum"); LexerToken token = _tokens[(_currentIndex + index) % LookAheadCount]; if (checkActive && (token.Type == TokenType.Unknown)) { throw new LexerException(LexerException.Codes.NoActiveToken); } return(token); } }
/// <remarks> It is an error to access the current TokenType until <see cref="NextToken"/> has been called. </remarks> public Lexer(string input) { _tokenizer = new Tokenizer(input); for (int i = 0; i < LookAheadCount; i++) { _tokens[i] = new LexerToken(); } for (int i = 0; i < (LookAheadCount - 1); i++) { if (!ReadNext(i)) { break; } } }
/// <summary>Advances the current token.</summary> /// <returns>Returns the now active token.</returns> public LexerToken NextToken() { ReadNext(_currentIndex); _currentIndex = (_currentIndex + 1) % LookAheadCount; LexerToken token = _tokens[_currentIndex]; if (token.Type == TokenType.EOF) { throw new LexerException(LexerException.Codes.UnexpectedEOF); } if (token.Type == TokenType.Error) { throw token.Error; } return(token); }
/// <summary> Gets the symbol the specified number of tokens ahead without advancing the current token. </summary> /// <remarks> If the token is not a symbol, returns an empty string. </remarks> public string PeekTokenSymbol(int count) { LexerToken token = this[count]; if (token.Type == TokenType.Symbol) { return(token.Token); } else if (token.Type == TokenType.Error) { throw token.Error; } else { return(String.Empty); } }
/// <summary> Reads the next token into the specified location within the buffer. </summary> /// <returns> True if the read token type is not EOF. </returns> private bool ReadNext(int index) { LexerToken token = _tokens[index]; try { token.Type = _tokenizer.NextToken(); token.Token = _tokenizer.Token; } catch (Exception exception) { token.Type = TokenType.Error; token.Error = exception; } token.Line = _tokenizer.Line; token.LinePos = _tokenizer.LinePos; return((token.Type != TokenType.EOF) && (token.Type != TokenType.Error)); }