private void Crawl(Page page) { page.FetchAllUrls(page.url.depth); if (page.UrlList.Count > 0) { foreach (Url url in page.UrlList) HandleURL(url); } else Console.WriteLine("No links found."); Console.WriteLine("Finished crawling page."); }
private void FetchNewPage(Url url) { Log.WriteToLog("Fetching page...", url.uri.AbsoluteUri); NetworkConnection connection = new NetworkConnection(); Page page = new Page(url, connection.Go(url)); if (!String.IsNullOrEmpty(page.source)) { Log.LoadSuccess(url.uri.AbsoluteUri); Crawl(page); } LoadNextURL(); }