public Token NextToken()
{
if (this.lastTokens.Count > 0)
return this.lastTokens.Pop();
char ch;
try
{
ch = this.NextCharSkipBlanks();
}
catch (EndOfInputException)
{
return null;
}
if (ch == '\r')
{
Token token = new Token() { TokenType = TokenType.EndOfLine, Value = "\r\n" };
try
{
char ch2 = this.NextCharSkipBlanks();
if (ch2 != '\n')
this.PushChar(ch2);
}
catch (EndOfInputException)
{
return null;
}
return token;
}
if (ch == '\n')
return new Token() { TokenType = TokenType.EndOfLine, Value = "\r\n" };
if (char.IsDigit(ch))
return this.NextInteger(ch);
if (char.IsLetter(ch) || ch == '_' || ch == '@' || ch == '$')
return this.NextName(ch);
if (ch == StringChar)
return this.NextString();
if (ch == QuotedStringChar)
return this.NextQuotedString();
if (Separators.Contains(ch))
return this.NextSeparator(ch);
if (Operators.Contains(ch))
return this.NextOperator(ch);
throw new InvalidDataException(string.Format(CultureInfo.InvariantCulture, "Unknown input '{0}'", ch));
}
private static bool IsToken(Token token, string value, TokenType type)
{
if (token == null)
return false;
if (token.TokenType != type)
return false;
if (type == TokenType.Name)
return token.Value.Equals(value, StringComparison.InvariantCultureIgnoreCase);
return token.Value.Equals(value);
}
private static bool IsName(Token token, string value)
{
return IsToken(token, value, TokenType.Name);
}
private Token NextReal(string integerPart)
{
string real = integerPart + ".";
char ch;
try
{
ch = this.NextChar();
while (char.IsDigit(ch))
{
real += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = real;
token.TokenType = TokenType.Real;
return token;
}
private Token NextString()
{
StringBuilder sb = new StringBuilder();
char ch;
char lastChar = (char)0;
ch = this.NextChar();
while (ch != StringChar || lastChar == '\\')
{
if (lastChar == '\\')
{
switch (ch)
{
case '\'':
sb.Length--;
sb.Append(ch);
break;
case 't':
sb.Length--;
sb.Append('\t');
break;
case 'a':
sb.Length--;
sb.Append('\a');
break;
case 'b':
sb.Length--;
sb.Append('\b');
break;
case 'e':
sb.Length--;
sb.Append((char) 27);
break;
case 'f':
sb.Length--;
sb.Append('\f');
break;
case 'n':
sb.Length--;
sb.Append('\n');
break;
case 'r':
sb.Length--;
sb.Append('\r');
break;
case 's':
sb.Length--;
sb.Append(' ');
break;
case 'v':
sb.Length--;
sb.Append('\v');
break;
case '\\':
break;
default:
sb.Append(ch);
break;
}
lastChar = (char)0;
}
else
{
sb.Append(ch);
lastChar = ch;
}
ch = this.NextChar();
}
Token token = new Token();
token.Value = sb.ToString();
token.TokenType = TokenType.String;
return token;
}
private Token NextQuotedString()
{
StringBuilder sb = new StringBuilder();
char ch;
char lastChar = (char)0;
ch = this.NextChar();
while (ch != QuotedStringChar || lastChar == '\\')
{
if (lastChar == '\\')
{
if (ch == '\'')
{
sb.Length--;
sb.Append(ch);
}
else if (ch != '\\')
sb.Append(ch);
lastChar = (char)0;
}
else
{
sb.Append(ch);
lastChar = ch;
}
ch = this.NextChar();
}
Token token = new Token();
token.Value = sb.ToString();
token.TokenType = TokenType.String;
return token;
}
private Token NextName(char ch)
{
string name = ch.ToString();
try
{
ch = this.NextChar();
while (char.IsLetterOrDigit(ch) || ch == '_' || (ch == '@' && name == "@"))
{
name += ch;
ch = this.NextChar();
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = name;
token.TokenType = TokenType.Name;
if (name == "true" || name == "false")
token.TokenType = TokenType.Boolean;
if (name == "@" || name == "@@")
throw new InvalidInputException(name);
return token;
}
private Token NextInteger(char ch)
{
string integer = ch.ToString();
try
{
ch = this.NextChar();
while (char.IsDigit(ch))
{
integer += ch;
ch = this.NextChar();
}
if (ch == '.')
{
return this.NextReal(integer);
}
this.PushChar(ch);
}
catch (EndOfInputException)
{
}
Token token = new Token();
token.Value = integer;
token.TokenType = TokenType.Integer;
return token;
}
internal void PushToken(Token token)
{
this.lastTokens.Push(token);
}
public UnexpectedTokenException(Token token)
: base(string.Format(CultureInfo.CurrentCulture, "Unexpected '{0}'", token.Value))
{
}