public virtual async Task <CrawlSummary> CrawlAsync(Uri RootUri) { var summary = new CrawlSummary(); var queue = new Queue <CrawledPage>(); var rootPage = new CrawledPage(RootUri, 1); await this.CrawlPageAsync(rootPage); queue.Enqueue(rootPage); var Domain = RootUri.Host; while (queue.Count > 0) { var current = queue.Dequeue(); try { if (current.Depth > this.Config.MaxDepth) { continue; } if (this.Config.MaxPagesToCrawl > 0 && summary.PagesCount >= this.Config.MaxPagesToCrawl) { continue; } if (summary.PageExists(current)) { continue; } if (string.IsNullOrEmpty(current.URI.AbsoluteUri)) { continue; } summary.AddPages(current); OnPageCrawlCompleted(new CrawlEventArgs { CrawledPage = current }); if (current.URI.Host != Domain) { continue; } await this.CrawlPageAsync(current); foreach (var link in current.Links) { try { var uri = new Uri(link); queue.Enqueue(new CrawledPage(uri, current.Depth + 1)); } catch (Exception Ex) { Console.WriteLine($"[ERROR] Link: '{link}', EX: {Ex.Message}"); } } } catch (Exception Ex) { Console.WriteLine($"[ERROR] Page: '{current.URI}', EX: {Ex.Message}"); } } return(summary); }
public bool PageExists(CrawledPage Page) { return(this.Pages.Contains(Page, new CrawledPageComparer())); }