public static IEnumerable <string> Enumerate(this BreakIterator bi)
{
var sb = new StringBuilder();
string text = bi.GetCLRText();
int start = bi.First(), end = bi.Next();
while (end != BreakIterator.DONE)
{
yield return(text.Substring(start, end - start));
start = end; end = bi.Next();
}
}
public static IEnumerable <Token> EnumerateTokens(this BreakIterator bi)
{
string text = bi.GetCLRText();
int start = bi.First(), end = bi.Next();
while (end != BreakIterator.DONE)
{
yield return(new Token(start, end, text.Substring(start, end - start), bi.GetRuleStatus()));
start = end;
end = bi.Next();
}
}
