static void CreateInvertedIndexAndIDF()
{
MongodbAccess mongo = new MongodbAccess();
Dictionary<ObjectId, Dictionary<string, int>> forward_index = mongo.GetForwardIndex();
Util.log("{0} items in forward index.", forward_index.Count);
HashSet<string> words = mongo.GetWordDict();
Util.log("{0} items in word dict.", words.Count);
foreach (string w in words)
{
InvertedIndexItem inverteditem = new InvertedIndexItem();
inverteditem.word = w;
inverteditem.webpage_ids = new BsonArray();
foreach (ObjectId webpageid in forward_index.Keys)
{
if (forward_index[webpageid].ContainsKey(w))
{
inverteditem.webpage_ids.Add(webpageid);
}
}
mongo.InsertInvertedIndexItem(inverteditem);
double idf = Math.Log10(forward_index.Count * 1.0 / inverteditem.webpage_ids.Count);
mongo.SetWordIDF(w, idf);
}
}
