Beispiel #1
0
 /// <summary>
 /// Attempts to download the response string and parse it into an HTML document.
 /// </summary>
 public HtmlDocument DownloadAsHtml(Uri endpoint)
 {
     if (endpoint == null)
     {
         throw new ArgumentNullException("endpoint");
     }
     log.DebugFormat(Debug.DownloadingAsHtml, endpoint);
     try
     {
         using (ExtendedWebClient client = new ExtendedWebClient())
         {
             using (Stream stream = client.OpenRead(endpoint))
             {
                 if (stream != null)
                 {
                     Encoding encoding = GetHttpResponseEncoding(client.ResponseHeaders);
                     HtmlDocument document = new HtmlDocument();
                     document.Load(stream, encoding);
                     if (document.DocumentNode.SelectSingleNode("/html") != null) // we just parsed some mumbo-jumbo as HTML, let's discard it.
                     {
                         return document;
                     }
                 }
             }
         }
     }
     catch (Exception exception)
     {
         log.Info(Debug.DownloadHtmlFailed.FormatWith(endpoint), exception);
     }
     return null;
 }
Beispiel #2
0
        /// <summary>
        /// Attempts to create an image downloading the response stream.
        /// </summary>
        public Image DownloadAsImage(Uri endpoint, bool inferContentTypeFromHttpHeaders = false)
        {
            Ensure.That(() => endpoint).IsNotNull();

            log.DebugFormat(Debug.DownloadingAsImage, endpoint);
            try
            {
                Image image = null;
                using (ExtendedWebClient client = new ExtendedWebClient())
                {
                    using (Stream stream = client.OpenRead(endpoint))
                    {
                        if (!inferContentTypeFromHttpHeaders || HasImageContentType(client.ResponseHeaders))
                        {
                            try
                            {
                                image = Image.FromStream(stream);
                            }
                            catch (ArgumentException)
                            {
                                log.DebugFormat(Debug.HttpResourceNotAnImage, endpoint);
                                return null;
                            }
                        }
                    }
                }
                return image;
            }
            catch (Exception exception)
            {
                log.Info(Debug.DownloadImageFailed.FormatWith(endpoint), exception);
                return null;
            }
        }
Beispiel #3
0
 /// <summary>
 /// Attempts to download the response string and parse it into an HTML document.
 /// </summary>
 public HtmlDocument DownloadAsHtml(Uri endpoint)
 {
     if (endpoint == null)
     {
         throw new ArgumentNullException("endpoint");
     }
     log.DebugFormat(Debug.DownloadingAsHtml, endpoint);
     try
     {
         using (ExtendedWebClient client = new ExtendedWebClient())
         {
             using (Stream stream = client.OpenRead(endpoint))
             {
                 if (stream != null)
                 {
                     Encoding     encoding = GetHttpResponseEncoding(client.ResponseHeaders);
                     HtmlDocument document = new HtmlDocument();
                     document.Load(stream, encoding);
                     if (document.DocumentNode.SelectSingleNode("/html") != null) // we just parsed some mumbo-jumbo as HTML, let's discard it.
                     {
                         return(document);
                     }
                 }
             }
         }
     }
     catch (Exception exception)
     {
         log.Info(Debug.DownloadHtmlFailed.FormatWith(endpoint), exception);
     }
     return(null);
 }
Beispiel #4
0
        public MinerResponse GetData(Miner miner)
        {
            var result = new MinerResponse();

            using (var client = new ExtendedWebClient())
            {
                client.Timeout = TimeOut;

                if (Proxy != null)
                {
                    WebRequest.DefaultWebProxy = Proxy;
                    client.Proxy = Proxy;
                }

                if (miner.Credentials != null)
                {
                    client.Credentials = miner.Credentials;

                    try
                    {
                        client.OpenRead(miner.Uri);
                    }
                    catch (Exception e)
                    {
                        result.Error = new WebError(DateTime.Now, e.Message);
                    }
                }

                try
                {
                    var response = client.DownloadString(miner.Uri);

                    if (response != null)
                    {
                        result.Data   = JsonConvert.DeserializeObject <MinerData>(response);
                        result.Status = true;
                    }
                }
                catch (Exception e)
                {
                    result.Error = new WebError(DateTime.Now, e.Message);
                }
            }

            return(result);
        }
        private string HttpGet(string uri)
        {
            String            _ret;
            ExtendedWebClient _webClient = new ExtendedWebClient(5000);

            using (_webClient)
            {
                try
                {
                    // open and read from the supplied URI
                    Stream       _stream = _webClient.OpenRead(uri);
                    StreamReader _reader = new StreamReader(_stream);
                    _ret = _reader.ReadToEnd();
                }
                catch (WebException _ex)
                {
                    if (_ex.Response is HttpWebResponse)
                    {
                        // Add you own error handling as required
                        switch (((HttpWebResponse)_ex.Response).StatusCode)
                        {
                        case HttpStatusCode.NotFound:
                        case HttpStatusCode.Unauthorized:
                            _ret = null;
                            break;

                        default:
                            throw _ex;
                        }
                    }

                    _ret = "-1";
                }
            }

            return(_ret);
        }
Beispiel #6
0
 /// <summary>
 /// Attempts to create an image downloading the response stream.
 /// </summary>
 public Image DownloadAsImage(Uri endpoint, bool inferContentTypeFromHttpHeaders = false)
 {
     if (endpoint == null)
     {
         throw new ArgumentNullException("endpoint");
     }
     log.DebugFormat(Debug.DownloadingAsImage, endpoint);
     try
     {
         Image image = null;
         using (ExtendedWebClient client = new ExtendedWebClient())
         {
             using (Stream stream = client.OpenRead(endpoint))
             {
                 if (!inferContentTypeFromHttpHeaders || HasImageContentType(client.ResponseHeaders))
                 {
                     try
                     {
                         image = Image.FromStream(stream);
                     }
                     catch (ArgumentException)
                     {
                         log.DebugFormat(Debug.HttpResourceNotAnImage, endpoint);
                         return(null);
                     }
                 }
             }
         }
         return(image);
     }
     catch (Exception exception)
     {
         log.Info(Debug.DownloadImageFailed.FormatWith(endpoint), exception);
         return(null);
     }
 }
Beispiel #7
0
        /// <summary>
        /// Download image
        /// </summary>
        /// <param name="url"></param>
        /// <param name="referer"></param>
        /// <param name="filename"></param>
        /// <param name="progressChanged"></param>
        /// <returns></returns>
        public string Download(string url, string referer, string filename, Action <string> progressChanged, CancellationToken cancelToken)
        {
            String message = "";

            checkOverwrite(filename, progressChanged, ref message);

            ExtendedWebClient client = new ExtendedWebClient();

            client.Referer = referer;

            var tempFilename = DeleteTempFile(filename, progressChanged);

            Util.CreateSubDir(tempFilename);
            int retry = 1;

            while (retry <= Properties.Settings.Default.RetryCount)
            {
                try
                {
                    using (var stream = client.OpenRead(url))
                    {
                        Int64 bytes_total = downloadPreCheck(filename, client, progressChanged, ref message);

                        using (var f = File.Create(tempFilename))
                        {
                            stream.CopyTo(f);
                        }

                        downloadPostCheck(filename, tempFilename, bytes_total, progressChanged, ref message);
                    }
                    break;
                }
                catch (NijieException)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    checkHttpStatusCode(url, ex);

                    Log.Warn(string.Format("Error when downloading: {0} to {1} ==> {2}, Retrying {2} of {3}...", url, tempFilename, ex.Message, retry, Properties.Settings.Default.RetryCount));
                    DeleteTempFile(filename, progressChanged);

                    var prefixMsg = message.Clone();
                    for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
                    {
                        message = String.Format("{0} waiting: {1}", prefixMsg, i);
                        Thread.Sleep(1000);
                        if (cancelToken != null && cancelToken.IsCancellationRequested)
                        {
                            throw new NijieException(string.Format("Cancel requested, error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_CANCELLED);
                        }
                    }
                    ++retry;

                    if (retry > Properties.Settings.Default.RetryCount)
                    {
                        throw new NijieException(string.Format("Error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
                    }
                }
            }

            Thread.Sleep(100); // delay before renaming
            File.Move(tempFilename, filename);
            message = "Saved to: " + filename;

            // set server time, based utc time
            if (client.ResponseHeaders.HasKeys())
            {
                var dateTime = client.ResponseHeaders["Last-Modified"];
                if (!String.IsNullOrWhiteSpace(dateTime))
                {
                    try
                    {
                        // var date = DateTime.Parse(dateTime);
                        var date = DateTime.ParseExact(dateTime, "ddd, dd MMM yyyy HH:mm:ss GMT", System.Globalization.CultureInfo.InvariantCulture);
                        File.SetCreationTimeUtc(filename, date.ToUniversalTime());
                        File.SetLastWriteTimeUtc(filename, date.ToUniversalTime());
                    }
                    catch (FormatException ex)
                    {
                        Log.Error(String.Format("Failed to parse datetime: {0}", dateTime), ex);
                    }
                }
            }

            if (progressChanged != null)
            {
                progressChanged(message);
            }
            return(message);
        }