Example #1
0
 private IEnumerable <string> TokenizeReader(TextReader reader)
 {
     using (var tokenizer = new BasicTokenizer(header.MaxTokenSize))
     {
         foreach (var term in tokenizer.Tokenize(reader).Select(t => t.AsString()))
         {
             yield return(term);
         }
     }
 }
 private IEnumerable <ScopedToken> TokenizeReader(TextReader reader)
 {
     using (var tokenizer = new BasicTokenizer(header.MaxTokenSize))
     {
         foreach (var token in tokenizer.Tokenize(reader))
         {
             yield return(token);
         }
     }
 }