/// <summary> /// Gets an identifier from the reader such as a variable reference, null, true, or false. /// Follows C# rules, non-qouted string starting with a letter or "_" followed by letters digits or "_" /// </summary> /// <param name="start">the starting character</param> /// <param name="buffer">a buffer to hold input</param> /// <returns>identifier token</returns> private Token GetIdentifier(char start, FastStringBuilder buffer) { int startLine = _lineNumber; int startPos = _position; buffer.Append(start); int c; char ch; while ((c = Peek()) != -1) { ch = (char)c; if (char.IsLetterOrDigit(ch) || ch == '_') { buffer.Append(ch); } else { return(new Token(TokenType.Identifier, buffer.ToString(), startLine, startPos)); } Read(); } return(new Token(TokenType.Identifier, buffer.ToString(), startLine, startPos)); }
/// <summary> /// Gets a single or double qouted string from the reader, handling and escape characters /// </summary> /// <param name="start">the starting character</param> /// <param name="buffer">buffer for input</param> /// <returns>string token</returns> private Token GetQuotedString(char start, FastStringBuilder buffer) { int startLine = _lineNumber; int startPos = _position; char quoteChar = start; bool escape = false; char ch; int c; while ((c = Read()) != -1) { ch = (char)c; if (escape) { switch (ch) { case 't': // horizantal tab buffer.Append('\t'); break; case 'n': // newline buffer.Append('\n'); break; case '\\': // reverse solidus buffer.Append('\\'); break; case '/': // solidus buffer.Append('/'); break; case 'b': // backspace buffer.Append('\b'); break; case 'f': // formfeed buffer.Append('\f'); break; case 'r': // carriage return buffer.Append('\r'); break; case 'u': // unicode escape sequence \unnnn { char[] ucodeChar = new char[4]; int nRead = Read(ucodeChar, 0, 4); if (nRead != 4) { throw new ParseException("Invalid unicode escape sequence, expecting \"\\unnnn\", but got " + (new string(ucodeChar, 0, nRead))); } buffer.Append((char)uint.Parse(new string(ucodeChar), System.Globalization.NumberStyles.HexNumber)); } break; default: buffer.Append(ch); break; } escape = false; } else { if (ch == '\\') { escape = true; } else if (ch == quoteChar) { return(new Token(quoteChar == '"' ? TokenType.DoubleQuotedString : TokenType.SingleQuotedString, buffer.ToString(), startLine, startPos)); } else { buffer.Append(ch); } } } throw new ParseException("Unterminated string constant"); }
/// <summary> /// Gets a number from the reader, which can be integer, floating point or scientific notation /// Examples: 123343, -123232, 12.345, -45.3434, 3.45E+10 /// </summary> /// <param name="start">the starting character</param> /// <param name="buffer">buffer to hold input</param> /// <returns>number token</returns> private Token GetNumber(char start, FastStringBuilder buffer) { int startLine = _lineNumber; int startPos = _position; char ch = start; buffer.Append(ch); int i = (start == '.') ? 1 : 0; while (i < 3) { switch (i) { case 0: // first part of integer GetIntegerPart(buffer); ch = (char)Peek(); if (ch == '.') { i = 1; // try to read fractional now buffer.Append((char)Read()); } else if (ch == 'e' || ch == 'E') { i = 2; // try to read exponent now buffer.Append((char)Read()); } else { i = 4; //break out break; } break; case 1: // fractional part GetIntegerPart(buffer); ch = (char)Peek(); if (ch == '.') { throw new ParseException("Invalid number exception"); } else if (ch == 'e' || ch == 'E') { i = 2; // read exponent buffer.Append((char)Read()); } else { i = 3; // break out } break; case 2: // scientific notation ch = (char)Peek(); //check for an optional sign if (ch == '+' || ch == '-') { buffer.Append((char)Read()); } GetIntegerPart(buffer); ch = (char)Peek(); if (ch == '.') { throw new ParseException("Invalid number exception"); } else { i = 3; // break out } break; } } return(new Token(TokenType.Number, buffer.ToString(), startLine, startPos)); }