/// <summary> /// Write dictionaries to disk /// </summary> public void Save() { postingMap.Replace(tempPostingMap); this.WriteDocWeights(); docWeigthsHashMap.Replace(tempDocWeightsHashMap); this.CreateTiers(); tier1.Replace(tempTier1); tier2.Replace(tempTier2); tier3.Replace(tempTier3); tempTermFreq.Clear(); calculatedDocWeights.Clear(); docByteSize.Clear(); tokensPerDocument.Clear(); averageTermFreqPerDoc.Clear(); tempPostingMap.Clear(); tempDocWeightsHashMap.Clear(); tempTier1.Clear(); tempTier2.Clear(); tempTier3.Clear(); }
public void Save() { map.Replace(tempMap); tempMap.Clear(); }
/// <summary> /// Build KGram onto disk /// </summary> /// <param name="vocabularies">List of unique vocabularies</param> public DiskKGram buildKGram(HashSet <string> vocabularies) { Console.WriteLine("Start KGram generating process..."); Console.WriteLine("Vocbularies' size: " + vocabularies.Count); Console.WriteLine("KGram size: " + this.size); Console.WriteLine("Building full size KGrams...."); foreach (string vocab in vocabularies) { //Split the vocabulary List <string> kGrams = this.KGramSplitter("$" + vocab + "$", this.size); //Add k-grams to dictionary foreach (string kGram in kGrams) { if (tempMap.ContainsKey(kGram)) { tempMap[kGram].Add(vocab); } else { tempMap.Add(kGram, new List <string> { vocab }); } } } //Build lesser k-gram to handle wildcard query lesser than size Console.WriteLine("Building lesser size KGrams...."); foreach (string kGram in tempMap.Keys) { for (int k = 0; k < this.size; k++) { List <string> miniKGrams = this.KGramSplitter(kGram, k); foreach (string miniKGram in miniKGrams) { if (!string.IsNullOrWhiteSpace(miniKGram) && miniKGram != "$") { if (tempMiniMap.ContainsKey(miniKGram)) { tempMiniMap[miniKGram].Add(kGram); } else { tempMiniMap.Add(miniKGram, new List <string> { kGram }); } } } } } map.Replace(tempMap); miniMap.Replace(tempMiniMap); //Print Results Console.WriteLine("KGram's size: " + map.GetSize()); Console.WriteLine("Lesser KGram's size: " + miniMap.GetSize()); //WriteKGramToDisk Console.WriteLine("Write K-Gram to disk..."); // Console.WriteLine("Path:" + Path.GetFullPath(this.path)); Console.WriteLine("Complete KGram generating process"); tempMap.Clear(); tempMiniMap.Clear(); return(this); }