/// <summary>
/// Create an instance of the token stream to read from the given reader.
/// </summary>
/// <param name="reader"></param>
public TokenStream(TextReader reader)
{
_reader = reader;
//_capacity = 16;
//_tokens = new Queue<Token>(_capacity);
_symbols = "[]<>():,{}.$".ToCharArray();
Array.Sort <char>(_symbols);
_buffer = new FastStringBuilder();
}
/// <summary>
/// Reads a token from the text reader and returns it
/// </summary>
/// <returns></returns>
private Token ReadTokenFromReader()
{
FastStringBuilder buffer = _buffer;
buffer.Length = 0;
int c;
char ch;
if (!_isEmpty)
{
while ((c = Read()) != -1)
{
ch = (char)c;
if (IsQuoteStart(ch))
{
return(GetQuotedString(ch, buffer));
}
else if (IsNumberStart(ch))
{
return(GetNumber(ch, buffer));
}
else if (char.IsWhiteSpace(ch))
{
// nothing
}
else if (IsIdentifierStart(ch))
{
return(GetIdentifier(ch, buffer));
}
else if (IsLineCommentStart(ch))
{
ReadLineComment(ch);
}
else if (IsMultilineCommentStart(ch))
{
ReadMultilineComment(ch);
}
else if (IsSymbolStart(ch))
{
return(GetSymbol(ch));
}
else
{
throw new ParseException("Invalid character");
}
buffer.Length = 0;
}
_isEmpty = true;
}
return(Token.Empty);
}
/// <summary>
/// Gets an integer portion of a number, stopping at a "." or the start of an exponent "e" or "E"
/// </summary>
/// <param name="buffer">buffer to store input</param>
private void GetIntegerPart(FastStringBuilder buffer)
{
int c;
char ch;
while ((c = Peek()) != -1)
{
ch = (char)c;
if (char.IsNumber(ch))
{
buffer.Append(ch);
}
else if (ch == '.' || ch == 'e' || ch == 'E' || IsSymbolStart(ch) || char.IsWhiteSpace(ch))
{
break;
}
else
{
throw new ParseException("Invalid number, unexpected character: " + ch);
}
Read();
}
}
/// <summary>
/// Gets an identifier from the reader such as a variable reference, null, true, or false.
/// Follows C# rules, non-qouted string starting with a letter or "_" followed by letters digits or "_"
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">a buffer to hold input</param>
/// <returns>identifier token</returns>
private Token GetIdentifier(char start, FastStringBuilder buffer)
{
int startLine = _lineNumber;
int startPos = _position;
buffer.Append(start);
int c;
char ch;
while ((c = Peek()) != -1)
{
ch = (char)c;
if (char.IsLetterOrDigit(ch) || ch == '_')
{
buffer.Append(ch);
}
else
{
return(new Token(TokenType.Identifier, buffer.ToString(), startLine, startPos));
}
Read();
}
return(new Token(TokenType.Identifier, buffer.ToString(), startLine, startPos));
}
/// <summary>
/// Gets a single or double qouted string from the reader, handling and escape characters
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer for input</param>
/// <returns>string token</returns>
private Token GetQuotedString(char start, FastStringBuilder buffer)
{
int startLine = _lineNumber;
int startPos = _position;
char quoteChar = start;
bool escape = false;
char ch;
int c;
while ((c = Read()) != -1)
{
ch = (char)c;
if (escape)
{
switch (ch)
{
case 't': // horizantal tab
buffer.Append('\t');
break;
case 'n': // newline
buffer.Append('\n');
break;
case '\\': // reverse solidus
buffer.Append('\\');
break;
case '/': // solidus
buffer.Append('/');
break;
case 'b': // backspace
buffer.Append('\b');
break;
case 'f': // formfeed
buffer.Append('\f');
break;
case 'r': // carriage return
buffer.Append('\r');
break;
case 'u': // unicode escape sequence \unnnn
{
char[] ucodeChar = new char[4];
int nRead = Read(ucodeChar, 0, 4);
if (nRead != 4)
{
throw new ParseException("Invalid unicode escape sequence, expecting \"\\unnnn\", but got " + (new string(ucodeChar, 0, nRead)));
}
buffer.Append((char)uint.Parse(new string(ucodeChar), System.Globalization.NumberStyles.HexNumber));
}
break;
default:
buffer.Append(ch);
break;
}
escape = false;
}
else
{
if (ch == '\\')
{
escape = true;
}
else if (ch == quoteChar)
{
return(new Token(quoteChar == '"' ? TokenType.DoubleQuotedString : TokenType.SingleQuotedString, buffer.ToString(), startLine, startPos));
}
else
{
buffer.Append(ch);
}
}
}
throw new ParseException("Unterminated string constant");
}
/// <summary>
/// Gets a number from the reader, which can be integer, floating point or scientific notation
/// Examples: 123343, -123232, 12.345, -45.3434, 3.45E+10
/// </summary>
/// <param name="start">the starting character</param>
/// <param name="buffer">buffer to hold input</param>
/// <returns>number token</returns>
private Token GetNumber(char start, FastStringBuilder buffer)
{
int startLine = _lineNumber;
int startPos = _position;
char ch = start;
buffer.Append(ch);
int i = (start == '.') ? 1 : 0;
while (i < 3)
{
switch (i)
{
case 0: // first part of integer
GetIntegerPart(buffer);
ch = (char)Peek();
if (ch == '.')
{
i = 1; // try to read fractional now
buffer.Append((char)Read());
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // try to read exponent now
buffer.Append((char)Read());
}
else
{
i = 4; //break out
break;
}
break;
case 1: // fractional part
GetIntegerPart(buffer);
ch = (char)Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else if (ch == 'e' || ch == 'E')
{
i = 2; // read exponent
buffer.Append((char)Read());
}
else
{
i = 3; // break out
}
break;
case 2: // scientific notation
ch = (char)Peek();
//check for an optional sign
if (ch == '+' || ch == '-')
{
buffer.Append((char)Read());
}
GetIntegerPart(buffer);
ch = (char)Peek();
if (ch == '.')
{
throw new ParseException("Invalid number exception");
}
else
{
i = 3; // break out
}
break;
}
}
return(new Token(TokenType.Number, buffer.ToString(), startLine, startPos));
}
