示例#1
0
 public IEnumerable<Token> Traverse(string text)
 {
     if (string.IsNullOrEmpty(text))
     {
         yield break;
     }
     using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
     {
         var breaker = new BigramTokenBreaker(reader);
         var token = breaker.Next();
         do
         {
             yield return token;
         } while ((token = breaker.Next()) != null);
     }
 }
示例#2
0
 public IEnumerable <Token> Traverse(string text)
 {
     if (string.IsNullOrEmpty(text))
     {
         yield break;
     }
     using (var reader = new RewindStringReader(text, this.OptionOutputOriginalCase))
     {
         var breaker = new BigramTokenBreaker(reader);
         var token   = breaker.Next();
         do
         {
             yield return(token);
         } while ((token = breaker.Next()) != null);
     }
 }