public static Token ReadToken(LocatedTextReaderWrapper input)
{
input.SkipWhiteSpace();
int nextCharacterCode = input.Peek();
if(nextCharacterCode < 0) {
return null;
}
char nextCharacter = (char)nextCharacterCode;
var startLocation = input.Location.Clone();
Token token;
if(IsStringStart(nextCharacter)) {
token = ReadString(input);
} else if(IsNumberChar(nextCharacter)) {
token = ReadNumber(input);
} else if(IsIdentifierStart(nextCharacter)) {
token = ReadIdentifier(input);
} else if(IsSymbolChar(nextCharacter)) {
token = ReadSymbol(input);
} else {
throw new BadDataException("Unknown token starting with " + nextCharacter, startLocation);
}
token.Location = startLocation;
return token;
}
private static Token ReadNumber(LocatedTextReaderWrapper input)
{
var startLocation = input.Location.Clone();
var numberString = new StringBuilder();
while(IsNumberChar((char)input.Peek())) {
numberString.Append((char)input.Read());
}
double number;
if(!double.TryParse(numberString.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowLeadingSign, Parser.DefaultCulture, out number)) {
throw new BadDataException("Number", startLocation);
}
return new Token(TokenType.Number, number);
}
private static Token ReadSymbol(LocatedTextReaderWrapper input)
{
var curToken = "";
while(input.Peek() >= 0) {
char nextCharacter = (char)input.Peek();
if(!IsSymbolChar(nextCharacter)) {
break;
}
string newToken = curToken + nextCharacter;
if(!IsSymbolStart(newToken)) {
break;
}
curToken = newToken;
input.Read(); // Discard.
}
return new Token(TokenType.Symbol, curToken);
}
private static Token ReadIdentifier(LocatedTextReaderWrapper input)
{
var token = new StringBuilder();
while(input.Peek() >= 0) {
char nextCharacter = (char)input.Peek();
if(!IsIdentifierChar(nextCharacter)) {
break;
}
token.Append(nextCharacter);
input.Read(); // Discard.
}
return new Token(TokenType.Identifier, token.ToString());
}