public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
{
if (!this.CanDownload())
{
return;
}
ScraperHelper.SetOrigenToClient(this.Url, client);
Console.WriteLine("Downloading: " + this.Url);
var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false);
if (result.IsSuccessStatusCode)
{
var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);
if (!this.ValidateHtml(html))
{
throw new HttpRequestException("HTML returned was not verified.");
}
this.Html = html;
}
else
{
throw new HttpRequestException("HTML returned was not verified.");
}
}
public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
{
if (!this.CanDownload())
{
logger.Debug("Can download returned false, stopping the download job.");
return;
}
ScraperHelper.SetOrigenToClient(this.Url, client);
if (this.headers != null)
{
foreach (KeyValuePair <string, string> keyValuePair in headers)
{
client.DefaultRequestHeaders.Add(keyValuePair.Key, keyValuePair.Value);
}
}
logger.Trace("Downloading: " + this.Url);
var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false);
if (result.IsSuccessStatusCode)
{
var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);
if (!this.ValidateHtml(html))
{
logger.Warn("Failed HTML verification");
throw new HttpRequestException("HTML returned was not verified.");
}
this.Html = html;
}
else
{
throw new HttpRequestException(string.Format("HTML request was not successfull. Status Code: {0} ", result.StatusCode));
}
}
public override async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
{
var uri = new Uri(this.GetUrl());
client.BaseAddress = this.GetUrl().Contains("https:") ? new Uri("https://" + uri.Host) : new Uri("http://" + uri.Host);
ScraperHelper.SetOrigenToClient(this.GetUrl(), client);
var message = new HttpRequestMessage(HttpMethod.Post, uri.LocalPath);
if (this.headers != null)
{
foreach (var header in this.headers)
{
message.Headers.Add(header.Key, header.Value);
}
}
var keyValue = new List <KeyValuePair <string, string> >();
if (this.formData != null)
{
keyValue.AddRange(this.formData);
}
if (this.formString != null)
{
message.Content = new StringContent(this.formString, Encoding.UTF8, "application/json");
}
else
{
StringBuilder stringBuilder = new StringBuilder();
foreach (KeyValuePair <string, string> current in keyValue)
{
if (stringBuilder.Length > 0)
{
stringBuilder.Append('&');
}
stringBuilder.Append(Encode(current.Key));
stringBuilder.Append('=');
stringBuilder.Append(Encode(current.Value));
}
message.Content = new StringContent(stringBuilder.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded");
//message.Content = new MyFormUrlEncodedContent(keyValue);
}
var result = await client.SendAsync(message, cancelToken).ConfigureAwait(false);
this.responseHeaders = result.Headers;
if (result.IsSuccessStatusCode)
{
var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);
if (!this.ValidateHtml(html))
{
throw new HttpRequestException("HTML returned was not verified.");
}
this.SetHtml(html);
}
else
{
throw new HttpRequestException("HTML returned was not verified.");
}
}