/// <summary> /// Peek one token ahead of the current position /// </summary> /// <param name="token">The peeked token</param> /// <returns>True if there is one more token after the current position, otherwise false</returns> public bool Peek(out LexerToken token) { // read but don't advance // implementation: actually advance the lexer // but make sure Peek/Next call returns the same token if (this.peekToken != null) { // we've already peeked token = this.peekToken; return true; } // store current token LexerToken temp = this.currentToken; // advance parser if (this.Next()) { this.peekToken = this.currentToken; token = this.currentToken; // restore current state this.currentToken = temp; return true; } else { this.peekToken = null; token = null; // restore current state this.currentToken = temp; return false; } }
/// <summary> /// Peek one token ahead of the current position /// </summary> /// <param name="token">The peeked token</param> /// <returns>True if there is one more token after the current position, otherwise false</returns> public bool Peek(out LexerToken token) { // read but don't advance // implementation: actually advance the lexer // but make sure Peek/Next call returns the same token if (this.peekToken != null) { // we've already peeked token = this.peekToken; return(true); } // store current token LexerToken temp = this.currentToken; // advance parser if (this.Next()) { this.peekToken = this.currentToken; token = this.currentToken; // restore current state this.currentToken = temp; return(true); } else { this.peekToken = null; token = null; // restore current state this.currentToken = temp; return(false); } }
public bool Next() { int num; if (this.peekToken != null) { this.currentToken = this.peekToken; this.peekToken = null; return(true); } LexerToken currentToken = this.CurrentToken; int? currentType = null; StringBuilder builder = null; bool flag = false; while (!flag && ((num = this.reader.Peek()) >= 0)) { int num2; char nextChar = (char)num; flag = this.MatchTokenType(nextChar, currentType, out num2); if (!currentType.HasValue) { currentType = new int?(num2); builder = new StringBuilder(); builder.Append(nextChar); this.reader.Read(); } else { int?nullable2 = currentType; int num3 = num2; if ((nullable2.GetValueOrDefault() == num3) && nullable2.HasValue) { builder.Append(nextChar); this.reader.Read(); continue; } flag = true; } } if (currentType.HasValue) { LexerToken token2 = new LexerToken { Text = builder.ToString(), Type = currentType.Value }; this.currentToken = token2; } return(currentToken != this.currentToken); }
public bool Next() { int num; if (this.peekToken != null) { this.currentToken = this.peekToken; this.peekToken = null; return true; } LexerToken currentToken = this.CurrentToken; int? currentType = null; StringBuilder builder = null; bool flag = false; while (!flag && ((num = this.reader.Peek()) >= 0)) { int num2; char nextChar = (char) num; flag = this.MatchTokenType(nextChar, currentType, out num2); if (!currentType.HasValue) { currentType = new int?(num2); builder = new StringBuilder(); builder.Append(nextChar); this.reader.Read(); } else { int? nullable2 = currentType; int num3 = num2; if ((nullable2.GetValueOrDefault() == num3) && nullable2.HasValue) { builder.Append(nextChar); this.reader.Read(); continue; } flag = true; } } if (currentType.HasValue) { LexerToken token2 = new LexerToken { Text = builder.ToString(), Type = currentType.Value }; this.currentToken = token2; } return (currentToken != this.currentToken); }
public bool Peek(out LexerToken token) { if (this.peekToken != null) { token = this.peekToken; return true; } LexerToken currentToken = this.currentToken; if (this.Next()) { this.peekToken = this.currentToken; token = this.currentToken; this.currentToken = currentToken; return true; } this.peekToken = null; token = null; this.currentToken = currentToken; return false; }
public bool Peek(out LexerToken token) { if (this.peekToken != null) { token = this.peekToken; return(true); } LexerToken currentToken = this.currentToken; if (this.Next()) { this.peekToken = this.currentToken; token = this.currentToken; this.currentToken = currentToken; return(true); } this.peekToken = null; token = null; this.currentToken = currentToken; return(false); }
/// <summary> /// Move to the next token /// </summary> /// <returns>True if lexer has moved, otherwise false</returns> public bool Next() { // DEVNOTE(pqian): // This function is called a very large number of times during text parsing. // For example, experiments show that during simple WKT parsing, this function accounts for over 70% // of the workload. Thus, efforts should be taken to keep this function as light weight as possible. // Token Accumulation Logic: // we'll accumulate whenever we are: // 1a. starting fresh, hence currentType is null. // 1b. we are inside a token that's building, and the new character is still part of the same token. // we should break out of the loop if: // 2a. MatchTokenType tell us to (terminate = true) // 2b. We are inside a token and we've encountered a new token (currentType and peek type is different) // It follows that most of the time we will either accumulate the token or break out of the loop // to return the token, except in one case, where we are starting fresh AND the first character is a terminal // character (delimeter). In this case we will both accumulate that char and break out of the loop. if (this.peekToken != null) { this.currentToken = this.peekToken; this.peekToken = null; return(true); } LexerToken originalToken = this.CurrentToken; int? currentTokenType = null; int textValue; StringBuilder nextTokenText = null; bool isDelimiter = false; while (!isDelimiter && (textValue = this.reader.Peek()) >= 0) { char currentChar = (char)textValue; int newTokenType; isDelimiter = this.MatchTokenType(currentChar, currentTokenType, out newTokenType); if (!currentTokenType.HasValue) { // fresh token currentTokenType = newTokenType; nextTokenText = new StringBuilder(); nextTokenText.Append(currentChar); this.reader.Read(); } else { // existing token if (currentTokenType == newTokenType) { // continuation of the current token nextTokenText.Append(currentChar); this.reader.Read(); } else { // starting a new token isDelimiter = true; } } } // we got here due to end of stream, could still have unprocessed tokens if (currentTokenType.HasValue) { Debug.Assert(nextTokenText != null, "Token text should not be null if current Token type has value"); this.currentToken = new LexerToken() { Text = nextTokenText.ToString(), Type = currentTokenType.Value }; } return(originalToken != this.currentToken); }
private void TestTokenEqual(LexerToken token, LexerToken other) { Assert.Equal(token.Text, other.Text); Assert.Equal(token.Type, other.Type); }
/// <summary> /// Move to the next token /// </summary> /// <returns>True if lexer has moved, otherwise false</returns> public bool Next() { // DEVNOTE(pqian): // This function is called a very large number of times during text parsing. // For example, experiments show that during simple WKT parsing, this function accounts for over 70% // of the workload. Thus, efforts should be taken to keep this function as light weight as possible. // Token Accumulation Logic: // we'll accumulate whenever we are: // 1a. starting fresh, hence currentType is null. // 1b. we are inside a token that's building, and the new character is still part of the same token. // we should break out of the loop if: // 2a. MatchTokenType tell us to (terminate = true) // 2b. We are inside a token and we've encountered a new token (currentType and peek type is different) // It follows that most of the time we will either accumulate the token or break out of the loop // to return the token, except in one case, where we are starting fresh AND the first character is a terminal // character (delimeter). In this case we will both accumulate that char and break out of the loop. if (this.peekToken != null) { this.currentToken = this.peekToken; this.peekToken = null; return true; } LexerToken originalToken = this.CurrentToken; int? currentTokenType = null; int textValue; StringBuilder nextTokenText = null; bool isDelimiter = false; while (!isDelimiter && (textValue = this.reader.Peek()) >= 0) { char currentChar = (char)textValue; int newTokenType; isDelimiter = this.MatchTokenType(currentChar, currentTokenType, out newTokenType); if (!currentTokenType.HasValue) { // fresh token currentTokenType = newTokenType; nextTokenText = new StringBuilder(); nextTokenText.Append(currentChar); this.reader.Read(); } else { // existing token if (currentTokenType == newTokenType) { // continuation of the current token nextTokenText.Append(currentChar); this.reader.Read(); } else { // starting a new token isDelimiter = true; } } } // we got here due to end of stream, could still have unprocessed tokens if (currentTokenType.HasValue) { Debug.Assert(nextTokenText != null, "Token text should not be null if current Token type has value"); this.currentToken = new LexerToken() { Text = nextTokenText.ToString(), Type = currentTokenType.Value }; } return originalToken != this.currentToken; }