private TokenType SkipNumber(char ch, StringBuilder buffer, ref int hash)
{
StringHasher hasher = StringHasher.Create();
if (ch == '-')
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
}
if (!Tokenizer.IsDigit(ch))
{
return(TokenType.Error);
}
ch = this.SkipDigits(ch, buffer, ref hasher);
if (ch == '.')
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
if (!Tokenizer.IsDigit(ch))
{
return(TokenType.Error);
}
ch = this.SkipDigits(ch, buffer, ref hasher);
}
if (ch == 'e' || ch == 'E')
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
if (ch == '-' || ch == '+')
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
}
if (!Tokenizer.IsDigit(ch))
{
return(TokenType.Error);
}
this.SkipDigits(ch, buffer, ref hasher);
}
hash = hasher.HashValue;
Debug.Assert(buffer == null || StringHasher.HashSubstring(buffer.ToString(), 0, buffer.Length) == hash);
return(TokenType.Number);
}
private char SkipDigits(char ch, StringBuilder buffer, ref StringHasher hasher)
{
do
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
}while (Tokenizer.IsDigit(ch));
return(ch);
}
private TokenType SkipString(StringBuilder buffer, ref int hash)
{
TokenType type = TokenType.String;
StringHasher hasher = StringHasher.Create();
char ch = this.NextChar();
while (true)
{
if (ch == '\"')
{
_ = this.NextChar();
break;
}
else if (ch == '\\')
{
type = TokenType.EncodedString;
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
switch (ch)
{
case '\"':
case '\\':
case '/':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
break;
case 'u':
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
if (!Tokenizer.IsHexDigit(ch))
{
return(TokenType.Error);
}
hasher.AddChar(ch); // 1
buffer?.Append(ch);
ch = this.NextChar();
if (!Tokenizer.IsHexDigit(ch))
{
return(TokenType.Error);
}
hasher.AddChar(ch); // 2
buffer?.Append(ch);
ch = this.NextChar();
if (!Tokenizer.IsHexDigit(ch))
{
return(TokenType.Error);
}
hasher.AddChar(ch); // 3
buffer?.Append(ch);
ch = this.NextChar();
if (!Tokenizer.IsHexDigit(ch))
{
return(TokenType.Error);
}
hasher.AddChar(ch); // 4
buffer?.Append(ch);
ch = this.NextChar();
break;
default:
return(TokenType.Error);
}
}
else if (ch < ' ')
{
return(TokenType.Error);
}
else
{
hasher.AddChar(ch);
buffer?.Append(ch);
ch = this.NextChar();
}
}
hash = hasher.HashValue;
Debug.Assert(buffer == null || StringHasher.HashSubstring(buffer.ToString(), 0, buffer.Length) == hash);
return(type);
}