Пример #1
0
        /// <summary>
        /// run the engine, control all the classes
        /// </summary>
        /// <param name="path"></param>
        public void Engine(string path, string finalPath, bool stem)
        {
            Stemmer  stemmer     = new Stemmer();
            ReadFile rf          = new ReadFile(path);
            Parser   parser      = new Parser(rf.ReadStopWords(path + "\\stop_words.txt"));
            int      filesAmount = rf.FilesAmount();
            Document currentDoc  = null;
            string   tempPath1   = @"./temp Posting Files1";
            string   tempPath2   = @"./temp Posting Files2";

            Directory.CreateDirectory(tempPath1);
            Directory.CreateDirectory(tempPath2);
            Directory.CreateDirectory(finalPath);
            string[] filesInTmp1 = Directory.GetFiles(tempPath1, "*.*", SearchOption.AllDirectories);
            for (int i = 0; i < filesInTmp1.Length; i++)
            {
                File.Delete(filesInTmp1[i]);
            }
            string[] filesInTmp2 = Directory.GetFiles(tempPath2, "*.*", SearchOption.AllDirectories);
            for (int i = 0; i < filesInTmp2.Length; i++)
            {
                File.Delete(filesInTmp2[i]);
            }
            DirectoryInfo di           = new DirectoryInfo(path);
            long          size         = di.EnumerateFiles("*", SearchOption.AllDirectories).Sum(fi => fi.Length);
            long          avgFilesSize = size / filesAmount;
            long          tenPrecent   = (size * 9) / 100;
            long          numFiles     = tenPrecent / avgFilesSize;
            int           count        = 0;

            //numFiles = 20;
            for (int i = 0; i < filesAmount; i++) //going through the files in the dictionary and send each to the parser
            {
                Match matchTEXT = rf.Seperate(i); // get a sperated files from red file
                while (matchTEXT.Success)
                {
                    Term[] terms = parser.Parse(matchTEXT.Groups[1].Value).Values.ToArray();
                    int    max   = -1;
                    if (stem)
                    {
                        for (int j = 0; j < terms.Length; j++)
                        {
                            terms[j].SetName(stemmer.stemTerm(terms[j].GetName()));
                        }
                    }
                    indexer.PrepareToPosting(terms, currentDoc = parser.GetDoc());
                    for (int j = 0; j < terms.Length; j++)
                    {
                        int currentTF = terms[j].GetTF(currentDoc);
                        if (currentTF > max)
                        {
                            max = currentTF;
                        }
                    }
                    currentDoc.SetMaxTF(max);
                    currentDoc.SetLength(terms.Length);
                    string[] details = new string[4];
                    details[0] = currentDoc.GetMaxTfString();
                    details[1] = currentDoc.GetLengthString();
                    details[2] = currentDoc.GetDateString();
                    details[3] = "";
                    DocDictionary.Add(currentDoc.GetName(), details);
                    matchTEXT = matchTEXT.NextMatch();
                }
                count++;
                if (count == numFiles)
                {
                    Console.WriteLine("create posting");
                    indexer.CreateTempPostingFile(tempPath1);
                    count = 0;
                }
            }//for
            if (count > 0)// if we finished the for and there are still terms in the hash
            {
                indexer.CreateTempPostingFile(tempPath1);
            }
            int temporarlyPostingFolder1 = Directory.GetFiles(tempPath1, "*.*", SearchOption.AllDirectories).Length;
            int temporarlyPostingFolder2 = Directory.GetFiles(tempPath2, "*.*", SearchOption.AllDirectories).Length;

            //continue until there is only two files
            while (!(temporarlyPostingFolder1 == 2 && temporarlyPostingFolder2 == 0) || !(temporarlyPostingFolder1 == 0 && temporarlyPostingFolder2 == 2))
            {
                indexer.SetPostingNumber(0);
                Merge(tempPath1, tempPath2);
                temporarlyPostingFolder1 = Directory.GetFiles(tempPath1, "*.*", SearchOption.AllDirectories).Length;
                temporarlyPostingFolder2 = Directory.GetFiles(tempPath2, "*.*", SearchOption.AllDirectories).Length;
                if (temporarlyPostingFolder1 == 0 && temporarlyPostingFolder2 == 2)
                {
                    string[] temporarlyPostingFolder = Directory.GetFiles(tempPath2, "*.*", SearchOption.AllDirectories);
                    indexer.FinalMerge(temporarlyPostingFolder[0], temporarlyPostingFolder[1], finalPath, stem);
                    File.Delete(temporarlyPostingFolder[0]);
                    File.Delete(temporarlyPostingFolder[1]);
                    break;
                }
                indexer.SetPostingNumber(0);
                Merge(tempPath2, tempPath1);
                temporarlyPostingFolder1 = Directory.GetFiles(tempPath1, "*.*", SearchOption.AllDirectories).Length;
                temporarlyPostingFolder2 = Directory.GetFiles(tempPath2, "*.*", SearchOption.AllDirectories).Length;
                if (temporarlyPostingFolder1 == 2 && temporarlyPostingFolder2 == 0)
                {
                    string[] temporarlyPostingFolder = Directory.GetFiles(tempPath1, "*.*", SearchOption.AllDirectories);
                    indexer.FinalMerge(temporarlyPostingFolder[0], temporarlyPostingFolder[1], finalPath, stem);
                    File.Delete(temporarlyPostingFolder[0]);
                    File.Delete(temporarlyPostingFolder[1]);
                    break;
                }
            }
            theDictionary = indexer.GetFinalDic();
            //cach

            /*     string path1 = finalPath + "\\Poodle_Dictionary";
             *   StreamReader file1 = new StreamReader(path1);
             *   while (!file1.EndOfStream)
             *   {
             *       string line = file1.ReadLine();
             *       StringBuilder sb = new StringBuilder();
             *       for (int i = 0; i < line.IndexOf("total tf:") - 1; i++)
             *       {
             *           sb.Append(line[i]);
             *       }
             *       string name = sb.ToString();
             *       String[] details = new string[4];
             *       int count1 = 0;
             *       for (int i = line.IndexOf("total tf:") + 10; i < line.Length; i++)
             *       {
             *           if (line[i] != '~')
             *               sb.Append(line[i]);
             *           else
             *           {
             *               details[count1] = sb.ToString();
             *               sb.Clear();
             *           }
             *
             *       }
             *       //  string[] data = line.Split('~');
             *       // String[] details = { data[1], data[2], data[3], data[4] };
             *       theDictionary.Add(name, details);
             *   }*/

            List <KeyValuePair <string, string[]> > tempDic = theDictionary.ToList();

            tempDic = tempDic.OrderByDescending(a => Int32.Parse(a.Value[0])).ToList();//sort by max tf
            for (int i = 0; i < 10000; i++)
            {
                string       pathtToPosting = Path.Combine(finalPath, theDictionary[tempDic[i].Key][2]);
                FileStream   fs             = new FileStream(pathtToPosting, FileMode.Open, FileAccess.Read);
                BinaryReader br             = new BinaryReader(fs);
                br.BaseStream.Seek(Int64.Parse(theDictionary[tempDic[i].Key][3]), SeekOrigin.Begin);
                cache.Add(tempDic[i].Key, ReadLine(br));
            }
            tempDic.Clear();
            Save(finalPath, stem);
        }//engine
        public static Dictionary <string, int[]> remove(KeyValuePair <List <string>, List <bool> > pair, string docname, string filename)
        {
            int maxTf = 0;

            int[] temp;
            Dictionary <string, int[]> dic = new Dictionary <string, int[]>();

            if (stemB)
            {
                Stemmer stem = new Stemmer();
                for (int i = pair.Key.Count - 1; i >= 0; i--)
                {
                    pair.Key[i] = stem.stemTerm(pair.Key[i]);

                    if (dic.ContainsKey(pair.Key[i]))
                    {
                        if (pair.Value[i])
                        {
                            dic[pair.Key[i]][0]++;
                        }
                        else
                        {
                            dic[pair.Key[i]][1]++;
                        }

                        //docfile
                        temp = dic[pair.Key[i]];
                        if (temp[0] + temp[1] > maxTf)
                        {
                            maxTf = temp[0] + temp[1];
                        }
                    }
                    else
                    {
                        dic[pair.Key[i]] = new int[2];
                        if (pair.Value[i])
                        {
                            dic[pair.Key[i]][0]++;
                        }
                        else
                        {
                            dic[pair.Key[i]][1]++;
                        }

                        //docfile
                        if (1 > maxTf)
                        {
                            maxTf = 1;
                        }
                    }
                }
            }
            else
            {
                for (int i = pair.Key.Count - 1; i >= 0; i--)
                {
                    if (dic.ContainsKey(pair.Key[i]))
                    {
                        if (pair.Value[i])
                        {
                            dic[pair.Key[i]][0]++;
                        }
                        else
                        {
                            dic[pair.Key[i]][1]++;
                        }

                        //docfile
                        temp = dic[pair.Key[i]];
                        if (temp[0] + temp[1] > maxTf)
                        {
                            maxTf = temp[0] + temp[1];
                        }
                    }
                    else
                    {
                        dic[pair.Key[i]] = new int[2];
                        if (pair.Value[i])
                        {
                            dic[pair.Key[i]][0]++;
                        }
                        else
                        {
                            dic[pair.Key[i]][1]++;
                        }

                        //docfile
                        if (1 > maxTf)
                        {
                            maxTf = 1;
                        }
                    }
                }
            }



            DocDic.Add(docname, filename + "," + maxTf + "," + pair.Key.Count);

            return(dic);
        }