public void TestCase() { var text = "This is {test} \"te\\\"xt\"."; var tokenizer = new TextTokenizer(text, WordRule.Default, WhitespaceRule.Default, StringRule.Default, new SingleCharRule('{'), new SingleCharRule('}'), new SingleCharRule('.')); var tokens = tokenizer.ReadToEnd(); Assert.AreEqual(tokens [0].Rule, WordRule.Default); Assert.AreEqual(tokens [1].Rule, WhitespaceRule.Default); Assert.AreEqual(tokens [2].Rule.GetType(), typeof(WordRule)); Assert.AreEqual(tokens [3].Text, " "); Assert.AreEqual(tokens [4].Rule.GetType(), typeof(SingleCharRule)); Assert.AreEqual(tokens [5].Text, "test"); Assert.AreEqual(tokens [6].Text, "}"); Assert.AreEqual(tokens [7].Rule.GetType(), typeof(WhitespaceRule)); Assert.AreEqual(tokens [8].Rule, StringRule.Default); Assert.AreEqual(tokens [8].Text, "te\"xt"); }