C# (CSharp) BigramTokenBreaker示例

示例#1

文件： BigramTokenizer.cs 项目： zhengchun/CWSharp

 public IEnumerable<Token> Traverse(string text)
 {
     if (string.IsNullOrEmpty(text))
     {
         yield break;
     }
     using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
     {
         var breaker = new BigramTokenBreaker(reader);
         var token = breaker.Next();
         do
         {
             yield return token;
         } while ((token = breaker.Next()) != null);
     }
 }

示例#2

文件： BigramTokenizer.cs 项目： yao-yi/CWSharp

 public IEnumerable <Token> Traverse(string text)
 {
     if (string.IsNullOrEmpty(text))
     {
         yield break;
     }
     using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
     {
         var breaker = new BigramTokenBreaker(reader);
         var token   = breaker.Next();
         do
         {
             yield return(token);
         } while ((token = breaker.Next()) != null);
     }
 }