public static void Consume(this Peekable <Token> tokens, TokenType type, string value)
{
var token = tokens.Read();
token.Assert(type, value);
tokens.Consume();
}
public static bool ConsumeIf(this Peekable <Token> tokens, TokenType type, string value)
{
if (tokens.Peek(out var token) && token.Is(type, value))
{
tokens.Consume();
return(true);
}
return(false);
}
public static string Consume(this Peekable <Token> tokens, TokenType type)
{
var token = tokens.Read();
token.Assert(type);
string s = token.Value;
tokens.Consume();
return(s);
}
public static bool SkipToEndStatementOrObject(this Peekable <Token> tokens)
{
while (tokens.Peek(out var token))
{
tokens.Consume();
if (token.Is(TokenType.Symbol, "}") || token.Is(TokenType.Symbol, ";"))
{
return(true);
}
}
return(false);
}
private static bool SkipToSymbol(this Peekable <Token> tokens, string symbol)
{
while (tokens.Peek(out var token))
{
tokens.Consume();
if (token.Is(TokenType.Symbol, symbol))
{
return(true);
}
}
return(false);
}
internal static int ConsumeInt32(this Peekable <Token> tokens, int?max = null)
{
var token = tokens.Read();
token.Assert(TokenType.AlphaNumeric);
tokens.Consume();
if (TryParseInt32(token.Value, out int val, max))
{
return(val);
}
throw token.Throw(ErrorCode.InvalidInteger, "Unable to parse integer");
}
public static bool SkipToEndOptions(this Peekable <Token> tokens)
{
while (tokens.Peek(out var token))
{
if (token.Is(TokenType.Symbol, ";") || token.Is(TokenType.Symbol, "}"))
{
return(true); // but don't consume
}
tokens.Consume();
if (token.Is(TokenType.Symbol, "]"))
{
return(true);
}
}
return(false);
}
internal static bool ConsumeBoolean(this Peekable <Token> tokens)
{
var token = tokens.Read();
token.Assert(TokenType.AlphaNumeric);
tokens.Consume();
if (string.Equals("true", token.Value, StringComparison.OrdinalIgnoreCase))
{
return(true);
}
if (string.Equals("false", token.Value, StringComparison.OrdinalIgnoreCase))
{
return(false);
}
throw token.Throw("Unable to parse boolean");
}
public static bool SkipToEndStatement(this Peekable <Token> tokens)
{
Token token;
while (tokens.Peek(out token))
{
if (token.Is(TokenType.Symbol, "}"))
{
return(true); // but don't consume
}
tokens.Consume();
if (token.Is(TokenType.Symbol, ";"))
{
return(true);
}
}
return(false);
}
internal static string ConsumeString(this Peekable <Token> tokens, bool asBytes = false)
{
var token = tokens.Read();
switch (token.Type)
{
case TokenType.StringLiteral:
MemoryStream ms = null;
do
{
ReadStringBytes(ref ms, token.Value);
tokens.Consume();
} while (tokens.Peek(out token) && token.Type == TokenType.StringLiteral); // literal concat is a thing
if (ms == null)
{
return("");
}
if (!asBytes)
{
#if NETSTANDARD1_3
string s = ms.TryGetBuffer(out var segment)
? Encoding.UTF8.GetString(segment.Array, segment.Offset, segment.Count)
: Encoding.UTF8.GetString(ms.ToArray());
#else
string s = Encoding.UTF8.GetString(ms.GetBuffer(), 0, (int)ms.Length);
#endif
return(s.Replace("\\", @"\\")
.Replace("\'", @"\'")
.Replace("\"", @"\""")
.Replace("\r", @"\r")
.Replace("\n", @"\n")
.Replace("\t", @"\t"));
}
var sb = new StringBuilder((int)ms.Length);
int b;
ms.Position = 0;
while ((b = ms.ReadByte()) >= 0)
{
switch (b)
{
case '\n': sb.Append(@"\n"); break;
case '\r': sb.Append(@"\r"); break;
case '\t': sb.Append(@"\t"); break;
case '\'': sb.Append(@"\'"); break;
case '\"': sb.Append(@"\"""); break;
case '\\': sb.Append(@"\\"); break;
default:
if (b >= 32 && b < 127)
{
sb.Append((char)b);
}
else
{
// encode as 3-part octal
sb.Append('\\')
.Append((char)(((b >> 6) & 7) + (int)'0'))
.Append((char)(((b >> 3) & 7) + (int)'0'))
.Append((char)(((b >> 0) & 7) + (int)'0'));
}
break;
}
}
return(sb.ToString());
case TokenType.AlphaNumeric:
tokens.Consume();
return(token.Value);
default:
throw token.Throw(ErrorCode.InvalidString);
}
}