public virtual bool ReadToken(out Token token)
{
token = new Token();
var tokenType = TokenType.None;
if (SkipWhitespace())
tokenType = TokenType.EndLine;
else
{
var c = PopChar();
string str = null;
double d = 0;
ulong i = 0;
CharToken charToken;
if (twoChar.TryGetValue(c, out charToken))
{
var c2 = PopChar();
if (c2 == charToken.SecondChar)
tokenType = charToken.TokenType;
else
PushChar(c);
}
if (tokenType == TokenType.None)
singleChar.TryGetValue(c, out tokenType);
if (tokenType == TokenType.None)
{
sb.Remove(0, sb.Length);
if (IsIdentifier(c, true))
{
do
{
sb.Append(c);
} while (IsIdentifier(c = PopChar(), false));
PushChar(c);
str = sb.ToString();
if (!keywords.TryGetValue(sb.ToString(), out tokenType))
tokenType = TokenType.Identifier;
else
str = null;
}
else if (char.IsDigit(c))
{
sb.Append(c);
bool isFloat = false;
bool validChar;
do
{
validChar = false;
sb.Append(c = PopChar());
if (c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')
{
isFloat = true;
validChar = true;
}
} while (validChar || char.IsDigit(c));
PushChar(c);
str = sb.ToString();
if (isFloat)
d = double.Parse(str);
else
i = ulong.Parse(str);
tokenType = isFloat ? TokenType.Float : TokenType.Integer;
}
}
if (tokenType == TokenType.String)
{
sb.Remove(0, sb.Length);
char c2;
while ((c2 = PopChar()) != c)
{
if (c2 == '\\')
{
c2 = PopChar();
switch (c2)
{
case 'n':
c2 = '\n';
break;
case 'r':
c2 = '\r';
break;
case '0':
c2 = '\0';
break;
case 't':
c2 = '\t';
break;
case 'f':
c2 = '\f';
break;
case '\\':
c2 = '\\';
break;
case '"':
c2 = '"';
break;
}
}
sb.Append(c2);
}
PushChar(c2);
str = sb.ToString();
}
token.Float = d;
token.Integer = i;
token.String = str;
}
token.TokenType = tokenType;
return true;
}
void ProcessToken(Token token)
{
}