Example #1
0
        public static IEnumerable<Token> Tokenize(this ITokenizer tokenizer, Stream stream)
        {
            if (tokenizer == null)
                throw new ArgumentNullException("tokenizer");
            if (stream == null)
                throw new ArgumentNullException("stream");

            using (TextReader reader = new StreamReader(stream))
                foreach (Token token in tokenizer.Tokenize(reader.ReadAll()))
                    yield return token;
        }
Example #2
0
        public static IEnumerable<Sentence> TokenizeSentences(this ISentenceTokenizer tokenizer, Stream stream)
        {
            if (tokenizer == null)
                throw new ArgumentNullException("tokenizer");
            if (stream == null)
                throw new ArgumentNullException("stream");

            using (TextReader reader = new StreamReader(stream))
                foreach (Sentence sentence in tokenizer.TokenizeSentences(reader.ReadAll()))
                    yield return sentence;
        }