public TokenInfo ReadToken(PeekBuffer <char> buffer)
{
bool seenDecimal = false;
StringBuilder number = new StringBuilder();
while (buffer.TryPeek(out var nextChar))
{
if (char.IsDigit(nextChar))
{
number.Append(nextChar);
buffer.TryRead(out _);
}
else if (!seenDecimal && nextChar == '.')
{
seenDecimal = true;
number.Append(nextChar);
buffer.TryRead(out _);
}
else
{
break;
}
}
return(new TokenInfo(TokenType.Number, double.Parse(number.ToString())));
}
public void PeekBeforeRead()
{
var enumerator = "1234".GetEnumerator();
PeekBuffer <char> buffer = new PeekBuffer <char>(enumerator);
Assert.IsTrue(buffer.TryPeek(out var item));
Assert.AreEqual('1', item);
Assert.IsTrue(buffer.TryPeek(1, out item));
Assert.AreEqual('2', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('1', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('2', item);
Assert.IsTrue(buffer.TryPeek(out item));
Assert.AreEqual('3', item);
Assert.IsTrue(buffer.TryPeek(1, out item));
Assert.AreEqual('4', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('3', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('4', item);
}
public void TryRead_Sequential()
{
var enumerator = "123".GetEnumerator();
PeekBuffer <char> buffer = new PeekBuffer <char>(enumerator);
Assert.IsTrue(buffer.TryRead(out var item));
Assert.AreEqual('1', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('2', item);
Assert.IsTrue(buffer.TryRead(out item));
Assert.AreEqual('3', item);
}
public ProductionInfo ReadProduction(PeekBuffer <TokenInfo> tokenStream)
{
StringBuilder data = new StringBuilder();
tokenStream.TryRead(out var nameToken);
tokenStream.TryRead(out _);
data.Append(nameToken.Data.ToString());
data.Append("(");
while (tokenStream.TryPeek(out var next))
{
if (next.Type == TokenType.RightParen)
{
tokenStream.TryRead(out _);
data.Append(")");
return(new ProductionInfo(data.ToString()));
}
// Read next token
if (!ExpressionReader.FirstToken(next))
{
return(null);
}
if (!ExpressionReader.IsProduction(tokenStream))
{
return(null);
}
var expression = ExpressionReader.ReadProduction(tokenStream);
if (expression == null)
{
return(null);
}
data.Append(expression.Data);
// The token after an expression should be comma or right-paren
if (!tokenStream.TryPeek(out next))
{
return(null);
}
if (next.Type == TokenType.RightParen)
{
continue;
}
if (next.Type == TokenType.Comma)
{
tokenStream.TryRead(out _);
}
}
return(null);
}
public TokenInfo ReadToken(PeekBuffer <char> buffer)
{
buffer.TryRead(out _);
buffer.TryRead(out _);
bool nextLineSeen = false;
while (buffer.TryPeek(out var nextChar))
{
if (nextChar == '\r' || nextChar == '\n')
{
nextLineSeen = true;
}
else if (nextLineSeen)
{
break;
}
buffer.TryRead(out _);
}
return(null);
}
public TokenInfo ReadToken(PeekBuffer <char> buffer)
{
while (buffer.TryPeek(out var nextChar))
{
if (char.IsWhiteSpace(nextChar))
{
buffer.TryRead(out _);
}
else
{
break;
}
}
return(null);
}
public TokenInfo ReadToken(PeekBuffer <char> buffer)
{
ReadTokenCalled = CallOrder++;
for (int i = 0; i < CharsToRead; i++)
{
buffer.TryRead(out _);
}
if (ReturnTokenInfo)
{
if (Data != null)
{
return(new TokenInfo(Type, Data));
}
return(new TokenInfo(Type));
}
return(null);
}
public TokenInfo ReadToken(PeekBuffer <char> buffer)
{
buffer.TryRead(out _);
return(new TokenInfo(type));
}