public void TokenizeSentences()
{
string text = "Es eju. Labrīt.";
string[][] sentences = new LuMiiTokenizer().TokenizeSentences(text).Select(s => s.Select(t => t.Text).ToArray()).ToArray();
Assert.AreEqual(2, sentences.Length);
Assert.AreEqual(3, sentences[0].Length);
Assert.AreEqual(2, sentences[1].Length);
}
public void Tokenize()
{
string text = "Es eju.";
string[] tokens = new LuMiiTokenizer().Tokenize(text).Select(t => t.Text).ToArray();
Assert.AreEqual(3, tokens.Length);
}
