/// <summary>
/// Attempts to download the response string and parse it into an HTML document.
/// </summary>
public HtmlDocument DownloadAsHtml(Uri endpoint)
{
if (endpoint == null)
{
throw new ArgumentNullException("endpoint");
}
log.DebugFormat(Debug.DownloadingAsHtml, endpoint);
try
{
using (ExtendedWebClient client = new ExtendedWebClient())
{
using (Stream stream = client.OpenRead(endpoint))
{
if (stream != null)
{
Encoding encoding = GetHttpResponseEncoding(client.ResponseHeaders);
HtmlDocument document = new HtmlDocument();
document.Load(stream, encoding);
if (document.DocumentNode.SelectSingleNode("/html") != null) // we just parsed some mumbo-jumbo as HTML, let's discard it.
{
return document;
}
}
}
}
}
catch (Exception exception)
{
log.Info(Debug.DownloadHtmlFailed.FormatWith(endpoint), exception);
}
return null;
}
/// <summary>
/// Attempts to create an image downloading the response stream.
/// </summary>
public Image DownloadAsImage(Uri endpoint, bool inferContentTypeFromHttpHeaders = false)
{
Ensure.That(() => endpoint).IsNotNull();
log.DebugFormat(Debug.DownloadingAsImage, endpoint);
try
{
Image image = null;
using (ExtendedWebClient client = new ExtendedWebClient())
{
using (Stream stream = client.OpenRead(endpoint))
{
if (!inferContentTypeFromHttpHeaders || HasImageContentType(client.ResponseHeaders))
{
try
{
image = Image.FromStream(stream);
}
catch (ArgumentException)
{
log.DebugFormat(Debug.HttpResourceNotAnImage, endpoint);
return null;
}
}
}
}
return image;
}
catch (Exception exception)
{
log.Info(Debug.DownloadImageFailed.FormatWith(endpoint), exception);
return null;
}
}
/// <summary>
/// Attempts to download the response string and parse it into an HTML document.
/// </summary>
public HtmlDocument DownloadAsHtml(Uri endpoint)
{
if (endpoint == null)
{
throw new ArgumentNullException("endpoint");
}
log.DebugFormat(Debug.DownloadingAsHtml, endpoint);
try
{
using (ExtendedWebClient client = new ExtendedWebClient())
{
using (Stream stream = client.OpenRead(endpoint))
{
if (stream != null)
{
Encoding encoding = GetHttpResponseEncoding(client.ResponseHeaders);
HtmlDocument document = new HtmlDocument();
document.Load(stream, encoding);
if (document.DocumentNode.SelectSingleNode("/html") != null) // we just parsed some mumbo-jumbo as HTML, let's discard it.
{
return(document);
}
}
}
}
}
catch (Exception exception)
{
log.Info(Debug.DownloadHtmlFailed.FormatWith(endpoint), exception);
}
return(null);
}
public MinerResponse GetData(Miner miner)
{
var result = new MinerResponse();
using (var client = new ExtendedWebClient())
{
client.Timeout = TimeOut;
if (Proxy != null)
{
WebRequest.DefaultWebProxy = Proxy;
client.Proxy = Proxy;
}
if (miner.Credentials != null)
{
client.Credentials = miner.Credentials;
try
{
client.OpenRead(miner.Uri);
}
catch (Exception e)
{
result.Error = new WebError(DateTime.Now, e.Message);
}
}
try
{
var response = client.DownloadString(miner.Uri);
if (response != null)
{
result.Data = JsonConvert.DeserializeObject <MinerData>(response);
result.Status = true;
}
}
catch (Exception e)
{
result.Error = new WebError(DateTime.Now, e.Message);
}
}
return(result);
}
private string HttpGet(string uri)
{
String _ret;
ExtendedWebClient _webClient = new ExtendedWebClient(5000);
using (_webClient)
{
try
{
// open and read from the supplied URI
Stream _stream = _webClient.OpenRead(uri);
StreamReader _reader = new StreamReader(_stream);
_ret = _reader.ReadToEnd();
}
catch (WebException _ex)
{
if (_ex.Response is HttpWebResponse)
{
// Add you own error handling as required
switch (((HttpWebResponse)_ex.Response).StatusCode)
{
case HttpStatusCode.NotFound:
case HttpStatusCode.Unauthorized:
_ret = null;
break;
default:
throw _ex;
}
}
_ret = "-1";
}
}
return(_ret);
}
/// <summary>
/// Attempts to create an image downloading the response stream.
/// </summary>
public Image DownloadAsImage(Uri endpoint, bool inferContentTypeFromHttpHeaders = false)
{
if (endpoint == null)
{
throw new ArgumentNullException("endpoint");
}
log.DebugFormat(Debug.DownloadingAsImage, endpoint);
try
{
Image image = null;
using (ExtendedWebClient client = new ExtendedWebClient())
{
using (Stream stream = client.OpenRead(endpoint))
{
if (!inferContentTypeFromHttpHeaders || HasImageContentType(client.ResponseHeaders))
{
try
{
image = Image.FromStream(stream);
}
catch (ArgumentException)
{
log.DebugFormat(Debug.HttpResourceNotAnImage, endpoint);
return(null);
}
}
}
}
return(image);
}
catch (Exception exception)
{
log.Info(Debug.DownloadImageFailed.FormatWith(endpoint), exception);
return(null);
}
}
/// <summary>
/// Download image
/// </summary>
/// <param name="url"></param>
/// <param name="referer"></param>
/// <param name="filename"></param>
/// <param name="progressChanged"></param>
/// <returns></returns>
public string Download(string url, string referer, string filename, Action <string> progressChanged, CancellationToken cancelToken)
{
String message = "";
checkOverwrite(filename, progressChanged, ref message);
ExtendedWebClient client = new ExtendedWebClient();
client.Referer = referer;
var tempFilename = DeleteTempFile(filename, progressChanged);
Util.CreateSubDir(tempFilename);
int retry = 1;
while (retry <= Properties.Settings.Default.RetryCount)
{
try
{
using (var stream = client.OpenRead(url))
{
Int64 bytes_total = downloadPreCheck(filename, client, progressChanged, ref message);
using (var f = File.Create(tempFilename))
{
stream.CopyTo(f);
}
downloadPostCheck(filename, tempFilename, bytes_total, progressChanged, ref message);
}
break;
}
catch (NijieException)
{
throw;
}
catch (Exception ex)
{
checkHttpStatusCode(url, ex);
Log.Warn(string.Format("Error when downloading: {0} to {1} ==> {2}, Retrying {2} of {3}...", url, tempFilename, ex.Message, retry, Properties.Settings.Default.RetryCount));
DeleteTempFile(filename, progressChanged);
var prefixMsg = message.Clone();
for (int i = 0; i < Properties.Settings.Default.RetryDelay; ++i)
{
message = String.Format("{0} waiting: {1}", prefixMsg, i);
Thread.Sleep(1000);
if (cancelToken != null && cancelToken.IsCancellationRequested)
{
throw new NijieException(string.Format("Cancel requested, error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_CANCELLED);
}
}
++retry;
if (retry > Properties.Settings.Default.RetryCount)
{
throw new NijieException(string.Format("Error when downloading: {0} to {1} ==> {2}", url, tempFilename, ex.Message), ex, NijieException.DOWNLOAD_UNKNOWN_ERROR);
}
}
}
Thread.Sleep(100); // delay before renaming
File.Move(tempFilename, filename);
message = "Saved to: " + filename;
// set server time, based utc time
if (client.ResponseHeaders.HasKeys())
{
var dateTime = client.ResponseHeaders["Last-Modified"];
if (!String.IsNullOrWhiteSpace(dateTime))
{
try
{
// var date = DateTime.Parse(dateTime);
var date = DateTime.ParseExact(dateTime, "ddd, dd MMM yyyy HH:mm:ss GMT", System.Globalization.CultureInfo.InvariantCulture);
File.SetCreationTimeUtc(filename, date.ToUniversalTime());
File.SetLastWriteTimeUtc(filename, date.ToUniversalTime());
}
catch (FormatException ex)
{
Log.Error(String.Format("Failed to parse datetime: {0}", dateTime), ex);
}
}
}
if (progressChanged != null)
{
progressChanged(message);
}
return(message);
}