/// <summary> /// Returns true to continue crawl of this url, else false /// </summary> /// <returns>True if this step should be cancelled, else false</returns> private bool OnAfterDownload(CrawlStep crawlStep, PropertyBag response) { EventHandler <AfterDownloadEventArgs> afterDownloadTmp = AfterDownload; if (afterDownloadTmp.IsNull()) { return(crawlStep.IsAllowed); } var e = new AfterDownloadEventArgs(!crawlStep.IsAllowed, response); afterDownloadTmp(this, e); return(!e.Cancel); }
private void CrawlerAfterDownload(object sender, AfterDownloadEventArgs e) { Uri uriToCrawl = null; String responseTime = "unknown"; if (e.Response != null) { uriToCrawl = e.Response.ResponseUri; responseTime = e.Response.DownloadTime.TotalSeconds.ToString(); } if (uriToCrawl == null) { uriToCrawl = e.CrawlStep.Uri; } if (_logger != null) { _logger.InfoFormat("{0} in {1}", uriToCrawl, responseTime); } }