public void ParseName()
{
Tokenizer tokenizer = new Tokenizer("name");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.Name, token.Type);
Assert.AreEqual("name", token.Value);
Assert.IsNull(tokenizer.NextToken());
}
public void ParseIntegerWithSpaces()
{
Tokenizer tokenizer = new Tokenizer(" 1234 ");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.Integer, token.Type);
Assert.AreEqual("1234", token.Value);
Assert.IsNull(tokenizer.NextToken());
}
public void ParseCarriageReturnNewLineAsEndOfLine()
{
Tokenizer tokenizer = new Tokenizer("\r\n");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.EndOfLine, token.Type);
Assert.AreEqual("\n", token.Value);
Assert.IsNull(tokenizer.NextToken());
}
public CommandReader(TextReader reader, bool islinereader)
{
this.tokenizer = new Tokenizer(reader);
this.islinereader = islinereader;
}
public void RaiseIfInvalidCharacter()
{
Tokenizer tokenizer = new Tokenizer("[]");
tokenizer.NextToken();
}
public void ParseTwoStrings()
{
Tokenizer tokenizer = new Tokenizer("\"Adam\" \"Smith\"");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.String, token.Type);
Assert.AreEqual("Adam", token.Value);
token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.String, token.Type);
Assert.AreEqual("Smith", token.Value);
Assert.IsNull(tokenizer.NextToken());
}
public void ParseSimpleStringWithUnicodeOtherSymbol()
{
char othersymbol = (char)0xfffd;
Assert.IsTrue(char.GetUnicodeCategory(othersymbol) == UnicodeCategory.OtherSymbol);
Tokenizer tokenizer = new Tokenizer(othersymbol + "\"Adam\"");
Token token = tokenizer.NextToken();
Assert.IsNotNull(token);
Assert.AreEqual(TokenType.String, token.Type);
Assert.AreEqual("Adam", token.Value);
Assert.IsNull(tokenizer.NextToken());
}