public void InitialsRemoved()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Remove<InitialsToken>();
tokenizer.Compile(); // optional
Token[] tokens = tokenizer.Tokenize("A.Bērziņš").ToArray();
Assert.AreEqual("A", tokens[0].Text);
Assert.AreEqual(".", tokens[1].Text);
Assert.AreEqual("Bērziņš", tokens[2].Text);
}
public void Reorder()
{
LatvianTokenizer tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Remove<ClockToken>();
tokenizer.Insert<ClockToken>(0);
Token token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Remove(typeof(ClockToken));
tokenizer.Insert(0, typeof(ClockToken));
token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
tokenizer = new LatvianTokenizer(compile: false);
tokenizer.Clear();
tokenizer.Add<TimeSpanToken>(); // matches 00:00:00
tokenizer.Add<ClockToken>(); // matches 00:00:00
tokenizer.Move<ClockToken>(0);
token = tokenizer.Tokenize("00:00:00").First();
Assert.IsTrue(token is ClockToken);
}
