public void TestTokenizer1() { Tokenizer tokenizer = new Tokenizer(new ExpressionDictionary(), "test"); Assert.AreEqual(new Text("test"),tokenizer.NextToken()); }
public void TestPrevious() { Tokenizer tokenizer = new Tokenizer(new ExpressionDictionary(), @"@book{ aaker:1912, author = { tes~est } }"); Assert.Throws<ArgumentOutOfRangeException>(() => tokenizer.GetPreviousCharacters(10)); tokenizer.NextToken(); tokenizer.NextToken(); tokenizer.NextToken(); tokenizer.NextToken(); tokenizer.NextToken(); AbstractToken token = tokenizer.NextToken(); Assert.AreEqual(" author ", tokenizer.GetPreviousCharacters(10)); }