public static IEnumerable <Token> EnumerateTokens(this BreakIterator bi)
{
string text = bi.GetCLRText();
int start = bi.First(), end = bi.Next();
while (end != BreakIterator.DONE)
{
yield return(new Token(start, end, text.Substring(start, end - start), bi.GetRuleStatus()));
start = end;
end = bi.Next();
}
}
public void CanIterateForwards(BreakIterator.UBreakIteratorType type, string text, int[] expected, BreakIterator.UWordBreak[] ruleStatus)
{
var locale = new Locale("zh");
BreakIterator bi = default(BreakIterator);
try
{
switch (type)
{
case BreakIterator.UBreakIteratorType.SENTENCE:
bi = BreakIterator.CreateSentenceInstance(locale);
break;
case BreakIterator.UBreakIteratorType.WORD:
bi = BreakIterator.CreateWordInstance(locale);
break;
default:
throw new NotSupportedException("This iterator type is not supported in this test yet. [" + type + "]");
}
bi.SetText(text);
CollectionAssert.AreEqual(expected, bi.Boundaries);
// Verify each boundary for the sentences
for (int i = 0; i < expected.Length; i++)
{
int current = bi.Current;
int status = bi.GetRuleStatus();
int expectedStatus = (int)ruleStatus[i];
Assert.AreEqual(expected[i], current);
Assert.AreEqual(expectedStatus, status);
CollectionAssert.AreEqual(new[] { expectedStatus }, bi.GetRuleStatusVector());
int moveNext = bi.MoveNext();
int next = i + 1;
if (next < expected.Length)
{
Assert.AreEqual(expected[next], moveNext);
}
else
{
// Verify that the BreakIterator is exhausted because we've
// moved past every item.
Assert.AreEqual(BreakIterator.DONE, moveNext);
}
}
int lastIndex = expected.Length - 1;
Assert.AreEqual(expected[lastIndex], bi.Current);
// We've moved past the last word, it should return the last offset.
Assert.AreEqual(BreakIterator.DONE, bi.MoveNext());
Assert.AreEqual(expected[lastIndex], bi.Current);
// Verify that the first element is correct now that we've moved to the end.
Assert.AreEqual(expected[0], bi.MoveFirst());
Assert.AreEqual(expected[0], bi.Current);
}
finally
{
if (bi != default(BreakIterator))
{
bi.Dispose();
}
}
}
