public TokenInfo ReadToken(PeekBuffer <char> buffer) { bool seenDecimal = false; StringBuilder number = new StringBuilder(); while (buffer.TryPeek(out var nextChar)) { if (char.IsDigit(nextChar)) { number.Append(nextChar); buffer.TryRead(out _); } else if (!seenDecimal && nextChar == '.') { seenDecimal = true; number.Append(nextChar); buffer.TryRead(out _); } else { break; } } return(new TokenInfo(TokenType.Number, double.Parse(number.ToString()))); }
public void PeekBeforeRead() { var enumerator = "1234".GetEnumerator(); PeekBuffer <char> buffer = new PeekBuffer <char>(enumerator); Assert.IsTrue(buffer.TryPeek(out var item)); Assert.AreEqual('1', item); Assert.IsTrue(buffer.TryPeek(1, out item)); Assert.AreEqual('2', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('1', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('2', item); Assert.IsTrue(buffer.TryPeek(out item)); Assert.AreEqual('3', item); Assert.IsTrue(buffer.TryPeek(1, out item)); Assert.AreEqual('4', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('3', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('4', item); }
public void TryRead_Sequential() { var enumerator = "123".GetEnumerator(); PeekBuffer <char> buffer = new PeekBuffer <char>(enumerator); Assert.IsTrue(buffer.TryRead(out var item)); Assert.AreEqual('1', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('2', item); Assert.IsTrue(buffer.TryRead(out item)); Assert.AreEqual('3', item); }
public ProductionInfo ReadProduction(PeekBuffer <TokenInfo> tokenStream) { StringBuilder data = new StringBuilder(); tokenStream.TryRead(out var nameToken); tokenStream.TryRead(out _); data.Append(nameToken.Data.ToString()); data.Append("("); while (tokenStream.TryPeek(out var next)) { if (next.Type == TokenType.RightParen) { tokenStream.TryRead(out _); data.Append(")"); return(new ProductionInfo(data.ToString())); } // Read next token if (!ExpressionReader.FirstToken(next)) { return(null); } if (!ExpressionReader.IsProduction(tokenStream)) { return(null); } var expression = ExpressionReader.ReadProduction(tokenStream); if (expression == null) { return(null); } data.Append(expression.Data); // The token after an expression should be comma or right-paren if (!tokenStream.TryPeek(out next)) { return(null); } if (next.Type == TokenType.RightParen) { continue; } if (next.Type == TokenType.Comma) { tokenStream.TryRead(out _); } } return(null); }
public TokenInfo ReadToken(PeekBuffer <char> buffer) { buffer.TryRead(out _); buffer.TryRead(out _); bool nextLineSeen = false; while (buffer.TryPeek(out var nextChar)) { if (nextChar == '\r' || nextChar == '\n') { nextLineSeen = true; } else if (nextLineSeen) { break; } buffer.TryRead(out _); } return(null); }
public TokenInfo ReadToken(PeekBuffer <char> buffer) { while (buffer.TryPeek(out var nextChar)) { if (char.IsWhiteSpace(nextChar)) { buffer.TryRead(out _); } else { break; } } return(null); }
public TokenInfo ReadToken(PeekBuffer <char> buffer) { ReadTokenCalled = CallOrder++; for (int i = 0; i < CharsToRead; i++) { buffer.TryRead(out _); } if (ReturnTokenInfo) { if (Data != null) { return(new TokenInfo(Type, Data)); } return(new TokenInfo(Type)); } return(null); }
public TokenInfo ReadToken(PeekBuffer <char> buffer) { buffer.TryRead(out _); return(new TokenInfo(type)); }