public List <Token> Tokenize(CultureInfo cultureInfo, string text)
{
List <Token> list = null;
IWordBreaker wordBreaker = this.LoadWordBreaker(cultureInfo);
if (wordBreaker != null)
{
IWordSink wordSink = new WordSink();
TEXT_SOURCE text_SOURCE = default(TEXT_SOURCE);
text_SOURCE.FillTextBuffer = new FillTextBuffer(this.FillBuffer);
text_SOURCE.Buffer = text;
text_SOURCE.Current = 0;
text_SOURCE.End = text_SOURCE.Buffer.Length;
if (wordBreaker.BreakText(ref text_SOURCE, wordSink, null) == 0)
{
list = ((WordSink)wordSink).Tokens;
}
}
if (list == null)
{
list = new List <Token>();
list.Add(new Token(0, text.Length));
}
return(list);
}
private uint FillBuffer(ref TEXT_SOURCE textSource)
{
return(2147751808U);
}