private Token GetNextToken()
{
// First, we handle the whitespace.
// We first make sure the initial whitespace is eaten if it exists:
if (IsFirstToken)
EatWhitespace();
// Now save this whitespace and clear the buffer, we'll check for more whitespace after the token.
string preWhitespace = whitespace.ToString();
whitespace.Clear();
// Get the token
Token result = null;
switch (currentChar)
{
case '.':
Read();
if (currentChar == '.')
{
Read();
result = new Token(Token.Type.VARGS, new Location(file, line, col - 2, line, col));
}
else result = new Token(Token.Type.DOT, new Location(file, line, col - 1, line, col));
break;
case ',': Read(); result = new Token(Token.Type.COMMA, new Location(file, line, col - 1, line, col)); break;
case ':': Read(); result = new Token(Token.Type.COLON, new Location(file, line, col - 1, line, col)); break;
case '(': Read(); result = new Token(Token.Type.OPEN_BRACE, new Location(file, line, col - 1, line, col)); break;
case ')': Read(); result = new Token(Token.Type.CLOSE_BRACE, new Location(file, line, col - 1, line, col)); break;
case '[': Read(); result = new Token(Token.Type.OPEN_SQUARE_BRACE, new Location(file, line, col - 1, line, col)); break;
case ']': Read(); result = new Token(Token.Type.CLOSE_SQUARE_BRACE, new Location(file, line, col - 1, line, col)); break;
case '{': Read(); result = new Token(Token.Type.OPEN_CURLY_BRACE, new Location(file, line, col - 1, line, col)); break;
case '}': Read(); result = new Token(Token.Type.CLOSE_CURLY_BRACE, new Location(file, line, col - 1, line, col)); break;
case '`': ReadLineComment(); result = null; break;
case '~': case '!': case '@': case '#':
case '%': case '^': case '&': case '*':
case '-': case '=': case '+': case '\\':
case '|': case '<': case '>': case '/':
case '?': case ';': result = ReadOperator(); break;
case '"': result = ReadString(); break;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
result = ReadNumber(); break;
case '$': result = ReadParamIndex(); break;
case '\'':
var loc = new Location(file, line, col);
Read();
var sym = ReadOthers();
if (sym.type == Token.Type.WILDCARD)
log.Error(sym.location, "The wildcard character cannot be used as a symbol literal.");
else if (sym.type != Token.Type.IDENTIFIER)
log.Error(sym.location, "Keyword '{0}' cannot be used as a symbol literal.", sym.image);
result = new Token(Token.Type.SYMBOL, loc, sym.image);
break;
default:
if (currentChar == '_' || char.IsLetter((char)currentChar))
result = ReadOthers();
else
{
// TODO error, skip the character in attempt to recover.
log.Error(new Location(file, line, col),
"Unexpected character '{0}'. This character should only exist in a string, are you missing quotes?",
char.ConvertFromUtf32(currentChar));
Read();
return GetNextToken();
}
break;
}
EatWhitespaceSansNewline();
if (result != null)
{
// Add the whitespace to the token.
result.PreWhitespace = preWhitespace;
result.PostWhitespace = whitespace.ToString();
}
// Once the token has its whitespace, make sure we've handled newlines now.
// If the tokens are on the same line, this does nothing so that's good.
EatWhitespace();
// That's it!
return result;
}
/// <summary>
/// Checks if the current token is of the given type.
/// If it is, the stream advances and this method returns true.
/// Otherwise a message is added to the error list if one was given, then this method returns false.
/// </summary>
/// <param name="type"></param>
/// <param name="failMessage"></param>
/// <returns></returns>
internal bool Expect(Token.Type type, string failMessage)
{
if (failMessage == null)
throw new ArgumentNullException("failMessage");
if (!IsOver && Current.type == type)
{
Advance();
return true;
}
// We make this optional because the parser could provide much more detailed information
// on its own after some thought, we don't HAVE to expect that error message up front.
if (failMessage != null)
LogError(failMessage);
return false;
}