public Tokenize ( string text, TokenizerMode mode = TokenizerMode.Default, bool hmm = true ) : IEnumerable |
||
text | string | |
mode | TokenizerMode | |
hmm | bool | |
return | IEnumerable |
public JiebaTokenizer(JiebaSegmenter seg, string input) { segmenter = seg; termAtt = AddAttribute<ITermAttribute>(); offsetAtt = AddAttribute<IOffsetAttribute>(); typeAtt = AddAttribute<ITypeAttribute>(); var text = input; tokens = segmenter.Tokenize(text, TokenizerMode.Search).ToList(); }