Beispiel #1
0
 public static IEnumerable <Token> TokenizeAll(this JiebaSegmenter segmenter, string text)
 {
     return(new SortedSet <Token>(segmenter.CutToToken(text).Union(segmenter.CutToToken(text, false)), TokenComparer));
 }