static void AccessingUrl(string link)
{
try
{
using (WebClient webClient = new WebClient())
{
webClient.Encoding = System.Text.Encoding.UTF8;
string content = webClient.DownloadString(link);
List <string> urls = LinkExtractor.ExtractUrlSameHost(content, link);
foreach (var url in urls.ToList())
{
if (!NewUrls.Contains(url) && (!OldUrls.Contains(url)))
{
NewUrls.Add(url);
}
//Console.WriteLine(url);
}
GetInfoOnPage(content, link);
}
}
catch (System.Net.WebException ex)
{
Console.WriteLine("Error: " + ex.Response);
Console.WriteLine("Try again...");
AccessingUrl(link);
}
}
private void AccessingPageWithoutCheck(Uri address)
{
new Task(() =>
{
using (WebClient webClient = new WebClient())
{
webClient.Encoding = System.Text.Encoding.UTF8;
webClient.DownloadStringCompleted +=
(sender, e) =>
{
if (e.Error == null)
{
OldUrls.Add(address.ToString());
List <string> urls = LinkExtractor.ExtractUrlSameHost(e.Result, address.ToString());
foreach (var url in urls.ToList())
{
string strUrl = url;
if (!strUrl.EndsWith("/"))
{
if (!Path.HasExtension(new Uri(strUrl).AbsolutePath))
{
strUrl = url + "/";
}
}
if (!NewUrls.Contains(strUrl) && (!OldUrls.Contains(strUrl)))
{
NewUrls.Add(strUrl);
}
Console.WriteLine(strUrl);
}
}
};
webClient.DownloadStringAsync(address);
}
}).Start();
}