public void EqualMatch()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
Token token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is TimeSpanToken);
}
public void kaut_jel_milj_kgs()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Add<LuMiiAbbreviationToken>();
tokenizer.Add<LuMiiCollocationToken>();
tokenizer.Compile();
string text = "kaut JEL 2 milj. mans k-gs";
Token[] tokens = tokenizer.Tokenize(text).ToArray();
Assert.AreEqual(5, tokens.Length);
Assert.AreEqual("kaut JEL", tokens[0].Text);
Assert.AreEqual("milj.", tokens[2].Text);
Assert.AreEqual("k-gs", tokens[4].Text);
}
public void Emotions()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Add<EmotionToken>();
EmotionToken[] tokens = tokenizer.Tokenize("Šodien esmu :) bet vakar biju :(").OfType<EmotionToken>().ToArray();
Assert.AreEqual(":)", tokens[0].Text);
Assert.AreEqual(":(", tokens[1].Text);
}
public void ļcien()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Add<LuMiiAbbreviationToken>();
tokenizer.Compile();
string text = "Ļ.cien. kungs!";
Token[] tokens = tokenizer.Tokenize(text).ToArray();
Assert.AreEqual(3, tokens.Length);
Assert.AreEqual("Ļ.cien.", tokens[0].Text);
}
public void kā_arī()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Add<LuMiiCollocationToken>();
tokenizer.Compile();
string text = "Es skrienu kā arī lecu.";
Token[] tokens = tokenizer.Tokenize(text).ToArray();
Assert.AreEqual(5, tokens.Length);
Assert.AreEqual("kā arī", tokens[2].Text);
}
public void Reorder()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Remove<ClockToken>();
tokenizer.Insert<ClockToken>(0);
Token token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Remove(typeof(ClockToken));
tokenizer.Insert(0, typeof(ClockToken));
token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Move<ClockToken>(0);
token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
}