public static IEnumerable<Token> Tokenize(this ITokenizer tokenizer, Stream stream) { if (tokenizer == null) throw new ArgumentNullException("tokenizer"); if (stream == null) throw new ArgumentNullException("stream"); using (TextReader reader = new StreamReader(stream)) foreach (Token token in tokenizer.Tokenize(reader.ReadAll())) yield return token; }
public static IEnumerable<Sentence> TokenizeSentences(this ISentenceTokenizer tokenizer, Stream stream) { if (tokenizer == null) throw new ArgumentNullException("tokenizer"); if (stream == null) throw new ArgumentNullException("stream"); using (TextReader reader = new StreamReader(stream)) foreach (Sentence sentence in tokenizer.TokenizeSentences(reader.ReadAll())) yield return sentence; }