public List <Token> Tokenize(CultureInfo cultureInfo, string text) { List <Token> list = null; IWordBreaker wordBreaker = this.LoadWordBreaker(cultureInfo); if (wordBreaker != null) { IWordSink wordSink = new WordSink(); TEXT_SOURCE text_SOURCE = default(TEXT_SOURCE); text_SOURCE.FillTextBuffer = new FillTextBuffer(this.FillBuffer); text_SOURCE.Buffer = text; text_SOURCE.Current = 0; text_SOURCE.End = text_SOURCE.Buffer.Length; if (wordBreaker.BreakText(ref text_SOURCE, wordSink, null) == 0) { list = ((WordSink)wordSink).Tokens; } } if (list == null) { list = new List <Token>(); list.Add(new Token(0, text.Length)); } return(list); }
private uint FillBuffer(ref TEXT_SOURCE textSource) { return(2147751808U); }