public static Token ReadToken(LocatedTextReaderWrapper input) { input.SkipWhiteSpace(); int nextCharacterCode = input.Peek(); if(nextCharacterCode < 0) { return null; } char nextCharacter = (char)nextCharacterCode; var startLocation = input.Location.Clone(); Token token; if(IsStringStart(nextCharacter)) { token = ReadString(input); } else if(IsNumberChar(nextCharacter)) { token = ReadNumber(input); } else if(IsIdentifierStart(nextCharacter)) { token = ReadIdentifier(input); } else if(IsSymbolChar(nextCharacter)) { token = ReadSymbol(input); } else { throw new BadDataException("Unknown token starting with " + nextCharacter, startLocation); } token.Location = startLocation; return token; }
private static Token ReadNumber(LocatedTextReaderWrapper input) { var startLocation = input.Location.Clone(); var numberString = new StringBuilder(); while(IsNumberChar((char)input.Peek())) { numberString.Append((char)input.Read()); } double number; if(!double.TryParse(numberString.ToString(), NumberStyles.AllowDecimalPoint | NumberStyles.AllowLeadingSign, Parser.DefaultCulture, out number)) { throw new BadDataException("Number", startLocation); } return new Token(TokenType.Number, number); }
private static Token ReadSymbol(LocatedTextReaderWrapper input) { var curToken = ""; while(input.Peek() >= 0) { char nextCharacter = (char)input.Peek(); if(!IsSymbolChar(nextCharacter)) { break; } string newToken = curToken + nextCharacter; if(!IsSymbolStart(newToken)) { break; } curToken = newToken; input.Read(); // Discard. } return new Token(TokenType.Symbol, curToken); }
private static Token ReadIdentifier(LocatedTextReaderWrapper input) { var token = new StringBuilder(); while(input.Peek() >= 0) { char nextCharacter = (char)input.Peek(); if(!IsIdentifierChar(nextCharacter)) { break; } token.Append(nextCharacter); input.Read(); // Discard. } return new Token(TokenType.Identifier, token.ToString()); }