private Token ReadNew()
{
CommentMode comment = CommentMode.None;
int readInt;
Token tmpToken;
while ((readInt = _source.Read()) >= 0)
{
// Handle comment mode
if (comment == CommentMode.Line)
{
if (readInt == '\n' || readInt == '\r')
{
comment = CommentMode.None;
}
continue;
}
else if (comment == CommentMode.Block)
{
if (readInt == '*')
{
if ((readInt = _source.Read()) == '/')
{
comment = CommentMode.None;
}
else if (readInt >= 0)
{
_source.PushBack(readInt);
}
}
continue;
}
// Skip whitespaces
if (char.IsWhiteSpace((char)readInt))
{
continue;
}
long position = _source.Position;
switch (readInt)
{
case '+':
switch (readInt = _source.Read())
{
case '+': return(new Token(TokenType.OperatorIncrement, position));
case '=': return(new Token(TokenType.OperatorAssignPlus, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorPlus, position));
}
case '-':
switch (readInt = _source.Read())
{
case '-': return(new Token(TokenType.OperatorDecrement, position));
case '=': return(new Token(TokenType.OperatorAssignMinus, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorMinus, position));
}
case '*':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorAssignTimes, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorTimes, position));
}
case '/':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorAssignDivide, position));
case '/':
comment = CommentMode.Line;
break;
case '*':
comment = CommentMode.Block;
break;
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorDivide, position));
}
break;
case '%':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorAssignModulo, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorModulo, position));
}
case '(': return(new Token(TokenType.BracketLeft, position));
case ')': return(new Token(TokenType.BracketRight, position));
case '[': return(new Token(TokenType.SquareBracketLeft, position));
case ']': return(new Token(TokenType.SquareBracketRight, position));
case '{': return(new Token(TokenType.CurlyBracketLeft, position));
case '}': return(new Token(TokenType.CurlyBracketRight, position));
case ';': return(new Token(TokenType.EndOfInstruction, position));
case ',': return(new Token(TokenType.Comma, position));
case '.':
// It might be a floating point number (in fact, iff it is follows by a digit)
int nextInt = _source.Read();
if (nextInt >= 0)
{
_source.PushBack(nextInt);
if (char.IsDigit((char)nextInt))
{
// Since the next character is a digit, ProcessNumber_Float() cannot return null
return(ProcessNumber_Float(position, new StringBuilder(), readInt));
}
}
return(new Token(TokenType.Period, position));
case ':':
switch (readInt = _source.Read())
{
case ':': return(new Token(TokenType.DoubleColon, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.Colon, position));
}
case '<':
switch (readInt = _source.Read())
{
case '<':
if ((readInt = _source.Read()) == '=')
{
return(new Token(TokenType.OperatorAssignLeftShift, position));
}
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorLeftShift, position));
case '=': return(new Token(TokenType.OperatorLessThanEqual, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorLessThan, position));
}
case '>':
switch (readInt = _source.Read())
{
case '>':
if ((readInt = _source.Read()) == '=')
{
return(new Token(TokenType.OperatorAssignRightShift, position));
}
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorRightShift, position));
case '=': return(new Token(TokenType.OperatorMoreThanEqual, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorMoreThan, position));
}
case '!':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorNotEqual, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorLogicalNot, position));
}
case '~': return(new Token(TokenType.OperatorBitwiseNot, position));
case '&':
switch (readInt = _source.Read())
{
case '&': return(new Token(TokenType.OperatorLogicalAnd, position));
case '=': return(new Token(TokenType.OperatorAssignBitwiseAnd, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorBitwiseAnd, position));
}
case '|':
switch (readInt = _source.Read())
{
case '|': return(new Token(TokenType.OperatorLogicalOr, position));
case '=': return(new Token(TokenType.OperatorAssignBitwiseOr, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorBitwiseOr, position));
}
case '^':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorAssignBitwiseXor, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorBitwiseXor, position));
}
case '=':
switch (readInt = _source.Read())
{
case '=': return(new Token(TokenType.OperatorEqual, position));
default:
if (readInt >= 0)
{
_source.PushBack(readInt);
}
return(new Token(TokenType.OperatorAssign, position));
}
case '\'':
switch (readInt = _source.Read())
{
case '\'':
throw new SyntaxException(_source.Position, "Empty character");
case '\\':
switch (readInt = _source.Read())
{
case '0': readInt = 0; break;
case '\'': break;
case '\\': break;
case 'n': readInt = '\n'; break;
case 'r': readInt = '\r'; break;
case 't': readInt = '\t'; break;
default:
throw new SyntaxException(_source.Position, "Unrecognized escape sequence");
}
break;
}
if (_source.Read() != '\'')
{
throw new SyntaxException(_source.Position, "Too many characters");
}
return(new ValuedToken <string>(TokenType.DecInt, position, readInt.ToString()));
default:
if ((tmpToken = ProcessNumber(readInt)) != null)
{
return(tmpToken);
}
if ((tmpToken = ProcessName(readInt)) != null)
{
return(tmpToken);
}
if ((tmpToken = ProcessString(readInt)) != null)
{
return(tmpToken);
}
throw new SyntaxException(_source.Position, "Unrecognized character: " + (char)readInt);
}
}
// We reached the end
return(new Token(TokenType.EndOfDocument, _source.Position + 1));
}
