private void RetrievePages(string url, int deep) { try { HtmlWeb hw = new HtmlWeb(); HtmlAgilityPack.HtmlDocument doc = hw.Load(url); //foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]")) HtmlNodeCollection htmlNodes = doc.DocumentNode.SelectNodes("//a[@href]"); htmlNodes.ProperQuickShuffle(random); for (int i = 0; i < htmlNodes.Count - 1; i++) { if (bw.CancellationPending) { break; } Mre.WaitOne(); HtmlNode link = htmlNodes[i]; if (link != null) { HtmlAttribute att = link.Attributes["href"]; if ((att.Value.Length > 12) && (att.Value.Substring(0, 4).ToUpper() == "HTTP") && !EndsWithForbiddenExtension(att.Value) && !ContainsForbiden(att.Value) && !UrlList.EndWithPart(att.Value.Substring(att.Value.Length - Math.Min(att.Value.Length, 20))) ) { //if ((!UrlList.Contains(att.Value)) && // (!UrlList.EndWithPart(att.Value.Substring(att.Value.Length - Math.Min(att.Value.Length, 20))))) UrlList.Add(att.Value); UrlListToProcess.Add(att.Value); NoOfLinks++; bw.ReportProgress(NoOfLinks); Log.Information(NoOfLinks + ": " + att.Value); //Console.WriteLine(NoOfLinks.ToString() + ": " + att.Value); if (random.Next(i, htmlNodes.Count) % 2 == 0) { //Log.Information("Soda"); LoadPage(att.Value); RetrieveNextPages(att.Value, deep); } else { //Log.Information("Liha"); RetrieveNextPages(att.Value, deep); LoadPage(att.Value); } if (MaxDepth != deep) { RetrievePages(att.Value, deep + 1); } } } } } catch (Exception ex) { Log.Error($"Error: {ex}", ex); } }