private Tuple <HtmlDocument, WebResponse> getPage(string url)
{
Tuple <HtmlDocument, WebResponse> result = null;
ExtendedWebClient client = new ExtendedWebClient();
int retry = 1;
while (retry <= Properties.Settings.Default.RetryCount)
{
try
{
var imagePage = client.DownloadData(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(Encoding.UTF8.GetString(imagePage));
result = new Tuple <HtmlDocument, WebResponse>(doc, client.Response);
break;
}
catch (Exception ex)
{
checkHttpStatusCode(url, ex);
++retry;
if (retry > Properties.Settings.Default.RetryCount)
{
throw;
}
}
}
return(result);
}
/// <summary>
/// Returns the header response for an HTTP request.
/// </summary>
public WebHeaderCollection DownloadHttpHeader(Uri endpoint, bool retryWithGet = false)
{
if (endpoint == null)
{
throw new ArgumentNullException("endpoint");
}
try
{
using (ExtendedWebClient client = new ExtendedWebClient {
Method = Constants.HttpHeadRequest
})
{
client.DownloadData(endpoint);
return(client.ResponseHeaders);
}
}
catch (Exception exception)
{
if (retryWithGet)
{
try
{
using (ExtendedWebClient client = new ExtendedWebClient())
{
client.DownloadData(endpoint);
return(client.ResponseHeaders);
}
}
catch (Exception retryException)
{
log.Info(Debug.DownloadHttpGetFailed.FormatWith(endpoint), retryException);
return(null);
}
}
log.Info(Debug.DownloadHttpHeadFailed.FormatWith(endpoint), exception);
return(null);
}
}
/// <summary>
/// Download data to memory byte array.
/// </summary>
/// <param name="url"></param>
/// <param name="referer"></param>
/// <returns>downloaded data.</returns>
public byte[] DownloadData(string url, string referer)
{
int retry = 1;
byte[] result = null;
while (retry <= Properties.Settings.Default.RetryCount)
{
try
{
ExtendedWebClient client = new ExtendedWebClient();
client.Referer = referer;
result = client.DownloadData(url);
break;
}
catch (Exception ex)
{
checkHttpStatusCode(url, ex);
Log.Warn(String.Format("Error when downloading data: {0} ==> {1}, Retrying {2} of {3}...", url, ex.Message, retry, Properties.Settings.Default.RetryCount));
//for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
//{
// Thread.Sleep(1000);
//}
++retry;
if (retry > Properties.Settings.Default.RetryCount)
{
throw new NijieException(String.Format("Error when downloading data: {0} ==> {1}", url, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
}
}
}
return(result);
}
/// <summary>
/// Returns the header response for an HTTP request.
/// </summary>
public WebHeaderCollection DownloadHttpHeader(Uri endpoint, bool retryWithGet = false)
{
if (endpoint == null)
{
throw new ArgumentNullException("endpoint");
}
try
{
using (ExtendedWebClient client = new ExtendedWebClient {Method = Constants.HttpHeadRequest})
{
client.DownloadData(endpoint);
return client.ResponseHeaders;
}
}
catch (Exception exception)
{
if (retryWithGet)
{
try
{
using (ExtendedWebClient client = new ExtendedWebClient())
{
client.DownloadData(endpoint);
return client.ResponseHeaders;
}
}
catch (Exception retryException)
{
log.Info(Debug.DownloadHttpGetFailed.FormatWith(endpoint), retryException);
return null;
}
}
log.Info(Debug.DownloadHttpHeadFailed.FormatWith(endpoint), exception);
return null;
}
}