/// <summary> /// /// </summary> /// <returns></returns> public CrawlerResults Run() { var results = new CrawlerResults { SourceId = _crawlSettings.SourceId }; var startTime = DateTime.Now; LoggerInfo("Starting Web Crawl"); BaseUrl = UrlParser.GetHostName(_crawlSettings.SeedUrl); BaseSchema = UrlParser.GetSchema(_crawlSettings.SeedUrl); var seedPageResp = HttpClient.GetRequest(_crawlSettings.SeedUrl); LoggerInfo(string.Format("Crawling {0}", _crawlSettings.SeedUrl)); LinksProcessed = new List <string>(); LinksToIndex = new List <string>(); var links = GetLinks(seedPageResp); var depth = 1; ProcessLinks(links, depth); var searchableContent = new List <IWebCrawlPage>(); foreach (var link in LinksToIndex) { LoggerDebug(string.Format("Extracting {0}", link)); var page = ProcessPage(link); if (page != null) { searchableContent.Add(page); } if (HandleStatusCallBack()) { return(results); } System.Threading.Thread.Sleep(1000); } LoggerInfo("Running Indexer"); var indexResults = _Indexer.RunUpdate(searchableContent, null, null); results.CrawlPages = searchableContent; results.CrawledCnt = searchableContent.Count; results.IndexedCnt = indexResults.TotalCnt; results.TotalCnt = indexResults.TotalCnt; results.ErrorCnt = indexResults.ErrorCnt; results.Duration = (DateTime.Now - startTime); LoggerInfo("Web Crawler finished."); return(results); }