public Token NextToken()
{
if (this.lastToken != null)
{
Token t = this.lastToken;
this.lastToken = null;
return t;
}
char ch;
try
{
ch = this.NextCharSkipBlanks();
}
catch (EndOfInputException)
{
return null;
}
if (char.IsDigit(ch))
{
return this.NextInteger(ch);
}
if (char.IsLetter(ch) || ch == '_')
{
return this.NextName(ch);
}
if (ch == StringChar)
{
return this.NextString();
}
if (ch == QuotedStringChar)
{
return this.NextQuotedString();
}
if (Separators.Contains(ch))
{
return this.NextSeparator(ch);
}
if (Operators.Contains(ch))
{
return this.NextOperator(ch);
}
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Unknown input '{0}'", ch));
}
public UnexpectedTokenException(Token token)
: base(string.Format(CultureInfo.CurrentCulture, "Unexpected '{0}'", token.Value))
{
}
private Token NextString()
{
StringBuilder sb = new StringBuilder();
char ch;
char lastChar = (char)0;
ch = this.NextChar();
while (ch != StringChar || lastChar == '\\')
{
if (lastChar == '\\')
{
switch (ch)
{
case 't':
sb.Length--;
sb.Append('\t');
break;
case 'a':
sb.Length--;
sb.Append('\a');
break;
case 'b':
sb.Length--;
sb.Append('\b');
break;
case 'e':
sb.Length--;
sb.Append((char)27);
break;
case 'f':
sb.Length--;
sb.Append('\f');
break;
case 'n':
sb.Length--;
sb.Append('\n');
break;
case 'r':
sb.Length--;
sb.Append('\r');
break;
case 'v':
sb.Length--;
sb.Append('\v');
break;
case '\\':
break;
default:
sb.Length--;
sb.Append(ch);
break;
}
lastChar = (char)0;
}
else
{
sb.Append(ch);
lastChar = ch;
}
ch = this.NextChar();
}
Token token = new Token();
token.Value = sb.ToString();
token.TokenType = TokenType.String;
return token;
}
private Token NextQuotedString()
{
StringBuilder sb = new StringBuilder();
char ch;
char lastChar = (char)0;
ch = this.NextChar();
while (ch != QuotedStringChar)
{
sb.Append(ch);
lastChar = ch;
ch = this.NextChar();
}
Token token = new Token();
token.Value = sb.ToString();
token.TokenType = TokenType.String;
return token;
}
private Token NextReal(string integerPart)
{
string real = integerPart + ".";
char ch;
try
{
ch = this.NextChar();
while (char.IsDigit(ch))
{
real += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = real;
token.TokenType = TokenType.Real;
return token;
}
private Token NextName(char ch)
{
string name = ch.ToString();
try
{
ch = this.NextChar();
while (char.IsLetterOrDigit(ch) || ch == '_')
{
name += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = name;
token.TokenType = TokenType.Name;
if (name == "true" || name == "false")
token.TokenType = TokenType.Boolean;
if (name == "@" || name == "@@")
throw new InvalidInputException(name);
return token;
}
private Token NextInteger(char ch)
{
string integer = ch.ToString();
try
{
ch = this.NextChar();
while (char.IsDigit(ch))
{
integer += ch;
ch = this.NextChar();
}
if (ch == '.')
{
return this.NextReal(integer);
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = integer;
token.TokenType = TokenType.Integer;
return token;
}
internal void PushToken(Token token)
{
if (this.lastToken != null)
{
throw new InvalidOperationException();
}
this.lastToken = token;
}
private static bool IsToken(Token token, string value, TokenType type)
{
if (token == null)
return false;
if (token.TokenType != type)
return false;
if (type == TokenType.Name)
return token.Value.Equals(value, StringComparison.InvariantCultureIgnoreCase);
return token.Value.Equals(value);
}
private static bool IsName(Token token, string value)
{
return IsToken(token, value, TokenType.Name);
}
