public static IEnumerable <string> Split(UBreakIteratorType type, string locale, string text) { if (string.IsNullOrEmpty(text)) { return(new string[] { }); } ErrorCode err; IntPtr bi = NativeMethods.ubrk_open(type, locale, text, text.Length, out err); if (err != ErrorCode.NoErrors) { throw new Exception("BreakIterator.Split() failed with code " + err); } var tokens = new List <string>(); int cur = NativeMethods.ubrk_first(bi); while (cur != DONE) { int next = NativeMethods.ubrk_next(bi); int status = NativeMethods.ubrk_getRuleStatus(bi); if (next != DONE && AddToken(type, status)) { tokens.Add(text.Substring(cur, next - cur)); } cur = next; } NativeMethods.ubrk_close(bi); return(tokens); }
//------------------------------------------------------------------------------------------------- //WinterDev public static IEnumerable <SplitBound> GetSplitBoundIter(UBreakIteratorType type, string locale, char[] charBuffer, int start, int len) { if (charBuffer == null || charBuffer.Length == 0) { return(new SplitBound[] { }); } ErrorCode err; var tokens = new List <SplitBound>(); unsafe { fixed(char *head = &charBuffer[0]) { IntPtr bi = NativeMethods.ubrk_open_unsafe(type, locale, head + start, len, out err); if (err != ErrorCode.NoErrors) { throw new Exception("BreakIterator.Split() failed with code " + err); } int cur = NativeMethods.ubrk_first(bi); while (cur != DONE) { int next = NativeMethods.ubrk_next(bi); int status = NativeMethods.ubrk_getRuleStatus(bi); if (next != DONE && AddToken(type, status)) { tokens.Add(new SplitBound(cur, next - cur)); } cur = next; } NativeMethods.ubrk_close(bi); } } return(tokens); }
/// <summary> /// Sets an existing iterator to point to a new piece of text. /// </summary> /// <param name="text">New text</param> public override void SetText(string text) { if (text == null) { throw new ArgumentNullException("text"); } _text = text; if (string.IsNullOrEmpty(Text)) { _textBoundaries = new TextBoundary[0]; _currentIndex = 0; return; } if (_breakIterator == IntPtr.Zero) { InitializeBreakIterator(); } else { ErrorCode err; NativeMethods.ubrk_setText(_breakIterator, Text, Text.Length, out err); if (err.IsFailure()) { throw new Exception("BreakIterator.ubrk_setText() failed with code " + err); } } List <TextBoundary> textBoundaries = new List <TextBoundary>(); // Start at the the beginning of the text and iterate until all // of the boundaries are consumed. int cur = NativeMethods.ubrk_first(_breakIterator); TextBoundary textBoundary; if (!TryGetTextBoundaryFromOffset(cur, out textBoundary)) { return; } textBoundaries.Add(textBoundary); while (cur != DONE) { int next = NativeMethods.ubrk_next(_breakIterator); if (!TryGetTextBoundaryFromOffset(next, out textBoundary)) { break; } textBoundaries.Add(textBoundary); cur = next; } _textBoundaries = textBoundaries.ToArray(); _currentIndex = 0; }