Beispiel #1
0
        private Tuple <HtmlDocument, WebResponse> getPage(string url)
        {
            Tuple <HtmlDocument, WebResponse> result = null;
            ExtendedWebClient client = new ExtendedWebClient();
            int retry = 1;

            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    var          imagePage = client.DownloadData(url);
                    HtmlDocument doc       = new HtmlDocument();
                    doc.LoadHtml(Encoding.UTF8.GetString(imagePage));
                    result = new Tuple <HtmlDocument, WebResponse>(doc, client.Response);
                    break;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    ++retry;
                    if (retry > Properties.Settings.Default.RetryCount)
                    {
                        throw;
                    }
                }
            }
            return(result);
        }
Beispiel #2
0
 /// <summary>
 /// Returns the header response for an HTTP request.
 /// </summary>
 public WebHeaderCollection DownloadHttpHeader(Uri endpoint, bool retryWithGet = false)
 {
     if (endpoint == null)
     {
         throw new ArgumentNullException("endpoint");
     }
     try
     {
         using (ExtendedWebClient client = new ExtendedWebClient {
             Method = Constants.HttpHeadRequest
         })
         {
             client.DownloadData(endpoint);
             return(client.ResponseHeaders);
         }
     }
     catch (Exception exception)
     {
         if (retryWithGet)
         {
             try
             {
                 using (ExtendedWebClient client = new ExtendedWebClient())
                 {
                     client.DownloadData(endpoint);
                     return(client.ResponseHeaders);
                 }
             }
             catch (Exception retryException)
             {
                 log.Info(Debug.DownloadHttpGetFailed.FormatWith(endpoint), retryException);
                 return(null);
             }
         }
         log.Info(Debug.DownloadHttpHeadFailed.FormatWith(endpoint), exception);
         return(null);
     }
 }
Beispiel #3
0
        /// <summary>
        /// Download data to memory byte array.
        /// </summary>
        /// <param name="url"></param>
        /// <param name="referer"></param>
        /// <returns>downloaded data.</returns>
        public byte[] DownloadData(string url, string referer)
        {
            int retry = 1;

            byte[] result = null;

            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    ExtendedWebClient client = new ExtendedWebClient();
                    client.Referer = referer;

                    result = client.DownloadData(url);
                    break;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    Log.Warn(String.Format("Error when downloading data: {0} ==> {1}, Retrying {2} of {3}...", url, ex.Message, retry, Properties.Settings.Default.RetryCount));

                    //for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
                    //{
                    //    Thread.Sleep(1000);
                    //}

                    ++retry;
                    if (retry > Properties.Settings.Default.RetryCount)
                    {
                        throw new NijieException(String.Format("Error when downloading data: {0} ==> {1}", url, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
                    }
                }
            }
            return(result);
        }
Beispiel #4
0
 /// <summary>
 /// Returns the header response for an HTTP request.
 /// </summary>
 public WebHeaderCollection DownloadHttpHeader(Uri endpoint, bool retryWithGet = false)
 {
     if (endpoint == null)
     {
         throw new ArgumentNullException("endpoint");
     }
     try
     {
         using (ExtendedWebClient client = new ExtendedWebClient {Method = Constants.HttpHeadRequest})
         {
             client.DownloadData(endpoint);
             return client.ResponseHeaders;
         }
     }
     catch (Exception exception)
     {
         if (retryWithGet)
         {
             try
             {
                 using (ExtendedWebClient client = new ExtendedWebClient())
                 {
                     client.DownloadData(endpoint);
                     return client.ResponseHeaders;
                 }
             }
             catch (Exception retryException)
             {
                 log.Info(Debug.DownloadHttpGetFailed.FormatWith(endpoint), retryException);
                 return null;
             }
         }
         log.Info(Debug.DownloadHttpHeadFailed.FormatWith(endpoint), exception);
         return null;
     }
 }