private void Crawl(Page page)
{
page.FetchAllUrls(page.url.depth);
if (page.UrlList.Count > 0)
{
foreach (Url url in page.UrlList)
HandleURL(url);
}
else
Console.WriteLine("No links found.");
Console.WriteLine("Finished crawling page.");
}
private void FetchNewPage(Url url)
{
Log.WriteToLog("Fetching page...", url.uri.AbsoluteUri);
NetworkConnection connection = new NetworkConnection();
Page page = new Page(url, connection.Go(url));
if (!String.IsNullOrEmpty(page.source))
{
Log.LoadSuccess(url.uri.AbsoluteUri);
Crawl(page);
}
LoadNextURL();
}
