public static IEnumerable<Token> Tokenize(this ITokenizer tokenizer, Stream stream)
{
if (tokenizer == null)
throw new ArgumentNullException("tokenizer");
if (stream == null)
throw new ArgumentNullException("stream");
using (TextReader reader = new StreamReader(stream))
foreach (Token token in tokenizer.Tokenize(reader.ReadAll()))
yield return token;
}
public static IEnumerable<Sentence> TokenizeSentences(this ISentenceTokenizer tokenizer, Stream stream)
{
if (tokenizer == null)
throw new ArgumentNullException("tokenizer");
if (stream == null)
throw new ArgumentNullException("stream");
using (TextReader reader = new StreamReader(stream))
foreach (Sentence sentence in tokenizer.TokenizeSentences(reader.ReadAll()))
yield return sentence;
}