public static IEnumerable<string> InnerGetLuceneTokens(string text)
{
TextReader tr = new StringReader(text);
TokenStream tok = new Lucene.Net.Analysis.Standard.StandardTokenizer(tr);
tok = new LowerCaseFilter(tok);
tok = new PorterStemFilter(tok);
Token t = tok.Next();
while (t != null) {
yield return t.TermText();
t = tok.Next();
}
tr.Close();
}
