Tokenize() public méthode

public Tokenize ( string text, TokenizerMode mode = TokenizerMode.Default, bool hmm = true ) : IEnumerable
text string
mode TokenizerMode
hmm bool
Résultat IEnumerable
JiebaSegmenter Class Documentation
Exemple #1
0
        public JiebaTokenizer(JiebaSegmenter seg, string input)
        {
            segmenter = seg;
            termAtt = AddAttribute<ITermAttribute>();
            offsetAtt = AddAttribute<IOffsetAttribute>();
            typeAtt = AddAttribute<ITypeAttribute>();

            var text = input;
            tokens = segmenter.Tokenize(text, TokenizerMode.Search).ToList();
        }