public void EscapeCharAtBeggining() { List <Token> tokens = m_Tokenizer.Tokenize("\\hello").Flush(); Assert.AreEqual(1, tokens.Count); TokenizerTestHelper.AssertToken(tokens[0], ETokenType.Value, "hello"); }
public void EscapeAnEscape() { List <Token> tokens = m_Tokenizer.Tokenize("\\\\").Flush(); Assert.AreEqual(1, tokens.Count); TokenizerTestHelper.AssertToken(tokens[0], ETokenType.Value, "\\"); }
public void SimpleEscape() { List <Token> tokens = m_Tokenizer.Tokenize("hello\\, world\\!").Flush(); Assert.AreEqual(3, tokens.Count); TokenizerTestHelper.AssertToken(tokens[0], ETokenType.Value, "hello,", 0, 0); TokenizerTestHelper.AssertToken(tokens[1], ETokenType.Separator, " ", 0, 6); TokenizerTestHelper.AssertToken(tokens[2], ETokenType.Value, "world!", 0, 7); }