public void TestCase()
{
var text = "This is {test} \"te\\\"xt\".";
var tokenizer = new TextTokenizer(text,
WordRule.Default, WhitespaceRule.Default, StringRule.Default,
new SingleCharRule('{'), new SingleCharRule('}'), new SingleCharRule('.'));
var tokens = tokenizer.ReadToEnd();
Assert.AreEqual(tokens [0].Rule, WordRule.Default);
Assert.AreEqual(tokens [1].Rule, WhitespaceRule.Default);
Assert.AreEqual(tokens [2].Rule.GetType(), typeof(WordRule));
Assert.AreEqual(tokens [3].Text, " ");
Assert.AreEqual(tokens [4].Rule.GetType(), typeof(SingleCharRule));
Assert.AreEqual(tokens [5].Text, "test");
Assert.AreEqual(tokens [6].Text, "}");
Assert.AreEqual(tokens [7].Rule.GetType(), typeof(WhitespaceRule));
Assert.AreEqual(tokens [8].Rule, StringRule.Default);
Assert.AreEqual(tokens [8].Text, "te\"xt");
}