void ChopText(ITextChunkView <TDocument> chunk)
{
var doc = chunk.Node.Document.Content;
var it = new BreakIterator <WordBreakType>(doc, rules.IsWordBreak, chunk.TrimmedStartOffset, chunk.TrimmedEndOffset);
var cursor = chunk.TrimmedStartOffset;
var lastBreak = chunk.TrimmedStartOffset;
while (it.MoveNext())
{
var wb = it.Current;
if (wb != WordBreakType.WordBreak)
{
cursor += 1;
continue;
}
if (cursor == lastBreak)
{
continue;
}
// have a new word
ITextChunkView <TDocument> first;
ITextChunkView <TDocument> second;
chunk.BreakAtOffset(cursor, out first, out second);
if (second != null)
{
Add(first);
chunk = second;
}
}
Add(chunk);
}
