private static IEnumerable <string> Tokenize(string value)
{
using (StringReader stringReader = new StringReader(value))
using (LetterTokenizer letterTokenizer = new LetterTokenizer(LuceneVersion.LUCENE_48, stringReader))
{
letterTokenizer.Reset();
while (letterTokenizer.IncrementToken())
{
yield return(letterTokenizer.GetAttribute <ICharTermAttribute>().ToString());
}
}
}
