/// <summary>
/// Executes OnDownloadException event
/// </summary>
private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer)
{
long downloadErrors = Interlocked.Increment(ref m_DownloadErrors);
if (MaximumHttpDownloadErrors.HasValue && MaximumHttpDownloadErrors.Value > downloadErrors)
{
m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", MaximumHttpDownloadErrors.Value);
StopCrawl();
}
m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception);
DownloadException.ExecuteEvent(this, () => new DownloadExceptionEventArgs(crawlStep, referrer, exception));
}
/// <summary>
/// Executes OnDownloadException event
/// </summary>
private void OnDownloadException(Exception exception, CrawlStep crawlStep, CrawlStep referrer)
{
var downloadErrors = Interlocked.Increment(ref this.m_DownloadErrors);
if (this.MaximumHttpDownloadErrors.HasValue && this.MaximumHttpDownloadErrors.Value > downloadErrors)
{
this.m_Logger.Error("Number of maximum failed downloads exceeded({0}), cancelling crawl", this.MaximumHttpDownloadErrors.Value);
this.StopCrawl();
}
this.m_Logger.Error("Download exception while downloading {0}, error was {1}", crawlStep.Uri, exception);
DownloadException?.Invoke(this, new DownloadExceptionEventArgs(crawlStep, referrer, exception));
}
private bool GetResponse(HttpWebRequest request, out string source)
{
StreamReader stream;
int retries = 5;
TryAgain:
try
{
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
stream = new StreamReader(response.GetResponseStream(), UTF8Encoding.UTF8);
source = stream.ReadToEnd();
return(true);
}
}
catch (WebException web)
{
// C'est possible que ça corrige certaine erreur de connexion
if ((web.Status == WebExceptionStatus.ConnectFailure) || (web.Status == WebExceptionStatus.NameResolutionFailure))
{
if (0 < --retries)
{
Thread.Sleep(1000);
goto TryAgain;
}
}
throw;
}
catch (Exception dwl)
{
DownloadException downloadException = new DownloadException(null, dwl);
downloadException.OriginException = dwl;
DoError(downloadException);
source = string.Empty;
return(false);
}
}
