static void AccessingUrl(string link) { try { using (WebClient webClient = new WebClient()) { webClient.Encoding = System.Text.Encoding.UTF8; string content = webClient.DownloadString(link); List <string> urls = LinkExtractor.ExtractUrlSameHost(content, link); foreach (var url in urls.ToList()) { if (!NewUrls.Contains(url) && (!OldUrls.Contains(url))) { NewUrls.Add(url); } //Console.WriteLine(url); } GetInfoOnPage(content, link); } } catch (System.Net.WebException ex) { Console.WriteLine("Error: " + ex.Response); Console.WriteLine("Try again..."); AccessingUrl(link); } }
private void AccessingPageWithoutCheck(Uri address) { new Task(() => { using (WebClient webClient = new WebClient()) { webClient.Encoding = System.Text.Encoding.UTF8; webClient.DownloadStringCompleted += (sender, e) => { if (e.Error == null) { OldUrls.Add(address.ToString()); List <string> urls = LinkExtractor.ExtractUrlSameHost(e.Result, address.ToString()); foreach (var url in urls.ToList()) { string strUrl = url; if (!strUrl.EndsWith("/")) { if (!Path.HasExtension(new Uri(strUrl).AbsolutePath)) { strUrl = url + "/"; } } if (!NewUrls.Contains(strUrl) && (!OldUrls.Contains(strUrl))) { NewUrls.Add(strUrl); } Console.WriteLine(strUrl); } } }; webClient.DownloadStringAsync(address); } }).Start(); }