public override int GetHashCode() { unchecked { int result = (CrawlStep != null ? CrawlStep.GetHashCode() : 0); result = (result * 397) ^ (Properties != null ? Properties.GetHashCode() : 0); result = (result * 397) ^ (Referrer != null ? Referrer.GetHashCode() : 0); return(result); } }
/// <summary> /// Executes OnDownloadException event /// </summary> private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer) { long downloadErrors = Interlocked.Increment(ref m_DownloadErrors); if (MaximumHttpDownloadErrors.HasValue && MaximumHttpDownloadErrors.Value > downloadErrors) { m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", MaximumHttpDownloadErrors.Value); StopCrawl(); } m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception); DownloadException.ExecuteEvent(this, () => new DownloadExceptionEventArgs(crawlStep, referrer, exception)); }
/// <summary> /// Returns true to continue crawl of this url, else false /// </summary> /// <returns>True if this step should be cancelled, else false</returns> private bool OnBeforeDownload(CrawlStep crawlStep) { EventHandler <BeforeDownloadEventArgs> beforeDownloadTmp = BeforeDownload; if (beforeDownloadTmp.IsNull()) { return(crawlStep.IsAllowed); } BeforeDownloadEventArgs e = new BeforeDownloadEventArgs(!crawlStep.IsAllowed, crawlStep); beforeDownloadTmp(this, e); return(!e.Cancel); }
/// <summary> /// Returns true to continue crawl of this url, else false /// </summary> /// <returns>True if this step should be cancelled, else false</returns> private bool OnAfterDownload(CrawlStep crawlStep, PropertyBag response) { EventHandler <AfterDownloadEventArgs> afterDownloadTmp = AfterDownload; if (afterDownloadTmp.IsNull()) { return(crawlStep.IsAllowed); } AfterDownloadEventArgs e = new AfterDownloadEventArgs(!crawlStep.IsAllowed, response); afterDownloadTmp(this, e); return(!e.Cancel); }
/// <summary> /// Queue a new step on the crawler queue /// </summary> /// <param name = "uri">url to crawl</param> /// <param name = "depth">depth of the url</param> /// <param name = "referrer">Step which the url was located</param> /// <param name = "properties">Custom properties</param> public void AddStep(Uri uri, int depth, CrawlStep referrer, Dictionary <string, object> properties) { if (!m_Crawling) { throw new InvalidOperationException("Crawler must be running before adding steps"); } if (m_CrawlStopped) { return; } if ((uri.Scheme != "https" && uri.Scheme != "http") || // Only accept http(s) schema (MaximumCrawlDepth.HasValue && MaximumCrawlDepth.Value > 0 && depth >= MaximumCrawlDepth.Value) || !m_CrawlerRules.IsAllowedUrl(uri, referrer) || !m_CrawlerHistory.Register(uri.GetUrlKeyString(UriSensitivity))) { if (depth == 0) { StopCrawl(); } return; } // Make new crawl step CrawlStep crawlStep = new CrawlStep(uri, depth) { IsExternalUrl = m_CrawlerRules.IsExternalUrl(uri), IsAllowed = true, }; m_CrawlerQueue.Push(new CrawlerQueueEntry { CrawlStep = crawlStep, Referrer = referrer, Properties = properties }); m_Logger.Verbose("Added {0} to queue referred from {1}", crawlStep.Uri, referrer.IsNull() ? string.Empty : referrer.Uri.ToString()); ProcessQueue(); }
public int CompareTo(CrawlerQueueEntry other) { return(CrawlStep.CompareTo(other.CrawlStep)); }