public void MixedDistinctSplits_CreateDistinctElements() { var tk = new vcal.Parser.Tokenizer(new[] { "!", "=>" }); var result = tk.Tokenize("This!is=>ATest").ToArray(); Assert.AreEqual(result.Length, 5); }
public void SingleSignSplits_CreateDistinctElements() { var tk = new vcal.Parser.Tokenizer(new[] { " ", "." }); var result = tk.Tokenize("This is.ATest").ToArray(); Assert.AreEqual(result.Length, 5); }