private Token _GetNextToken()
{
int c;
Token tok = null;
TokenType tok_type;
builder.Length = 0;
while ((c = ReadChar ()) != -1) {
if (in_code_block && (c == '\'' || c == '\"')) {
string str = ReadQuotedString (c);
Console.WriteLine ("READ QUOTED STRING: {0}", str);
if (str == null)
return new Token (line, col, TokenType.TOKEN_EOF, String.Empty);
tok = new Token (line, col, TokenType.TOKEN_QUOTED_STRING, str);
return tok;
}
int d = reader.Peek ();
if (in_code_block && IsNumberStartChar (c) && IsNumberChar (d)) {
object number = ReadNumber (c, d, out tok_type);
tok = new Token (line, col, tok_type, number.ToString (), number);
return tok;
}
string two_chars = String.Concat ((char) c, (char) d);
if (token_ops.DoubleCharOps.TryGetValue (two_chars, out tok_type)) {
tok = new Token (line, col, tok_type, two_chars);
reader.Read ();
UpdateInCodeBlock (tok_type);
return tok;
}
if (token_ops.SingleCharOps.TryGetValue (c, out tok_type)) {
tok = new Token (line, col, tok_type, "" + (char) c);
return tok;
}
if (in_code_block) {
if (Char.IsWhiteSpace ((char) c)) {
tok = new Token (line, col, TokenType.TOKEN_WHITESPACE, "" + (char) c);
return tok;
}
if (IsNameStartChar (c)) {
do {
builder.Append ((char) c);
c = ReadChar ();
} while (IsNameChar (c));
PutbackChar (c);
tok = new Token (line, col, TokenType.TOKEN_NAME, builder.ToString ());
return tok;
}
}
tok = new Token (line, col, TokenType.TOKEN_DATA, "" + (char) c);
return tok;
}
return new Token (line, col, TokenType.TOKEN_EOF, "");
}
public Token GetNextToken()
{
current = _GetNextToken ();
return current;
}
