private IDictionary <string, double> GetWordIfidf(string text, IEnumerable <string> allowPos)
{
IEnumerable <string> words = null;
if (allowPos.IsNotEmpty())
{
words = FilterCutByPos(text, allowPos);
}
else
{
words = Segmenter.Cut(text);
}
// Calculate TF
var freq = new Dictionary <string, double>();
foreach (var word in words)
{
var w = word;
if (string.IsNullOrEmpty(w) || w.Trim().Length < 2 || StopWords.Contains(w.ToLower()))
{
continue;
}
freq[w] = freq.GetDefault(w, 0.0) + 1.0;
}
var total = freq.Values.Sum();
foreach (var k in freq.Keys.ToList())
{
freq[k] *= IdfFreq.GetDefault(k, MedianIdf) / total;
}
return(freq);
}
