/// <summary> /// Executes OnDownloadException event /// </summary> private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer) { long downloadErrors = Interlocked.Increment(ref m_DownloadErrors); if (MaximumHttpDownloadErrors.HasValue && MaximumHttpDownloadErrors.Value > downloadErrors) { m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", MaximumHttpDownloadErrors.Value); StopCrawl(); } m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception); DownloadException.ExecuteEvent(this, () => new DownloadExceptionEventArgs(crawlStep, referrer, exception)); }
/// <summary> /// Executes OnDownloadException event /// </summary> private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer) { var downloadErrors = Interlocked.Increment(ref this.m_DownloadErrors); if (this.MaximumHttpDownloadErrors.HasValue && this.MaximumHttpDownloadErrors.Value > downloadErrors) { this.m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", this.MaximumHttpDownloadErrors.Value); this.StopCrawl(); } this.m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception); DownloadException?.Invoke(this, new DownloadExceptionEventArgs(crawlStep, referrer, exception)); }
private bool GetResponse(HttpWebRequest request, out string source) { StreamReader stream; int retries = 5; TryAgain: try { using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) { stream = new StreamReader(response.GetResponseStream(), UTF8Encoding.UTF8); source = stream.ReadToEnd(); return(true); } } catch (WebException web) { // C'est possible que ça corrige certaine erreur de connexion if ((web.Status == WebExceptionStatus.ConnectFailure) || (web.Status == WebExceptionStatus.NameResolutionFailure)) { if (0 < --retries) { Thread.Sleep(1000); goto TryAgain; } } throw; } catch (Exception dwl) { DownloadException downloadException = new DownloadException(null, dwl); downloadException.OriginException = dwl; DoError(downloadException); source = string.Empty; return(false); } }