private IEnumerable <string> TokenizeReader(TextReader reader)
{
using (var tokenizer = new BasicTokenizer(header.MaxTokenSize))
{
foreach (var term in tokenizer.Tokenize(reader).Select(t => t.AsString()))
{
yield return(term);
}
}
}
private IEnumerable <ScopedToken> TokenizeReader(TextReader reader)
{
using (var tokenizer = new BasicTokenizer(header.MaxTokenSize))
{
foreach (var token in tokenizer.Tokenize(reader))
{
yield return(token);
}
}
}
