Exemplo n.º 1
0
 public Crawler(string seeds)
 {
     _logger = Log.Instance;
     _visitedServers = new ConcurrentDictionary<int, long>();
     _domainToIpMap = new ConcurrentDictionary<int, int>();
     _urlFrontier = new SimpleUrlFrontier(seeds);
     _store = new Store();
     _parser = new Parser();
     _robotsTxts = new ConcurrentDictionary<int, RobotsTxt>();
     _cts = new CancellationTokenSource();
 }
Exemplo n.º 2
0
        static void Main(string[] args)
        {
            Crawler crawler = new Crawler("Ressources\\Seeds.txt");

            Console.WriteLine("Starting...");
            crawler.Start(1000);
            Console.WriteLine("Finished crawling...");
            Console.WriteLine("Press any key to start indexing...");
            Console.ReadLine();

            Store fileStore = new Store();
            Indexer indexer = new Indexer();
            Dictionary<string, string> filesMap = fileStore.LoadFileMap();
            Console.WriteLine("Indexing started");
            DateTime start = DateTime.Now;
            int filesIndexed = 0;
            foreach (var file in filesMap)
            {
                string fileContent = File.ReadAllText(Path.Combine(fileStore.OutputPath, file.Value));
                indexer.Index(file.Key, fileContent);
                Console.SetCursorPosition(0, 8);
                Console.WriteLine("Pages indexed {0} / {1}", ++filesIndexed, filesMap.Count);
            }
            DateTime end = DateTime.Now;
            TimeSpan timeElapsed = end.Subtract(start);
            double pagesPerSec = filesMap.Count / timeElapsed.TotalSeconds;
            Console.WriteLine("Indexed {0} pages per second", pagesPerSec);
            Console.WriteLine("Indexing finished in {0}", timeElapsed.ToString(@"hh\:mm\:ss"));

            Ranker ranker = new Ranker();

            Console.Write("The search engine ready for your query: ");
            string searchQuery = Console.ReadLine();

            Tokenizer tokenizer = new Tokenizer();
            List<string> queryTerms = tokenizer.TokenizeQuery(searchQuery).ToList();
            for (int i = 0; i < queryTerms.Count(); i++)
            {
                Stemmer stemmer = new Stemmer();
                stemmer.Stem(queryTerms[i]);
                queryTerms[i] = stemmer.ToString();
            }


            IEnumerable<KeyValuePair<int, double>> scores = ranker.Rank(queryTerms, indexer.Terms, indexer.TotalDocuments);

            int resultCount = 1;
            foreach (var item in scores)
            {
                string url = indexer.GetUrlFromHash(item.Key);
                if (url == null)
                    continue;
                Console.WriteLine("{0}: {1}", resultCount++, url);
                if (resultCount > 10)
                    break;
            }

            PageRank pageRank = new PageRank();
            float[] vector = pageRank.DoRank();
            for (int i = 0; i < vector.Length; i++)
            {
                Console.Write(vector[i]);
            }
            Console.WriteLine();

            Console.WriteLine("Press any key to exit...");
            Console.ReadLine();
        }