public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken) { if (!this.CanDownload()) { return; } ScraperHelper.SetOrigenToClient(this.Url, client); Console.WriteLine("Downloading: " + this.Url); var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false); if (result.IsSuccessStatusCode) { var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false); if (!this.ValidateHtml(html)) { throw new HttpRequestException("HTML returned was not verified."); } this.Html = html; } else { throw new HttpRequestException("HTML returned was not verified."); } }
public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken) { if (!this.CanDownload()) { logger.Debug("Can download returned false, stopping the download job."); return; } ScraperHelper.SetOrigenToClient(this.Url, client); if (this.headers != null) { foreach (KeyValuePair <string, string> keyValuePair in headers) { client.DefaultRequestHeaders.Add(keyValuePair.Key, keyValuePair.Value); } } logger.Trace("Downloading: " + this.Url); var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false); if (result.IsSuccessStatusCode) { var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false); if (!this.ValidateHtml(html)) { logger.Warn("Failed HTML verification"); throw new HttpRequestException("HTML returned was not verified."); } this.Html = html; } else { throw new HttpRequestException(string.Format("HTML request was not successfull. Status Code: {0} ", result.StatusCode)); } }
public override async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken) { var uri = new Uri(this.GetUrl()); client.BaseAddress = this.GetUrl().Contains("https:") ? new Uri("https://" + uri.Host) : new Uri("http://" + uri.Host); ScraperHelper.SetOrigenToClient(this.GetUrl(), client); var message = new HttpRequestMessage(HttpMethod.Post, uri.LocalPath); if (this.headers != null) { foreach (var header in this.headers) { message.Headers.Add(header.Key, header.Value); } } var keyValue = new List <KeyValuePair <string, string> >(); if (this.formData != null) { keyValue.AddRange(this.formData); } if (this.formString != null) { message.Content = new StringContent(this.formString, Encoding.UTF8, "application/json"); } else { StringBuilder stringBuilder = new StringBuilder(); foreach (KeyValuePair <string, string> current in keyValue) { if (stringBuilder.Length > 0) { stringBuilder.Append('&'); } stringBuilder.Append(Encode(current.Key)); stringBuilder.Append('='); stringBuilder.Append(Encode(current.Value)); } message.Content = new StringContent(stringBuilder.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded"); //message.Content = new MyFormUrlEncodedContent(keyValue); } var result = await client.SendAsync(message, cancelToken).ConfigureAwait(false); this.responseHeaders = result.Headers; if (result.IsSuccessStatusCode) { var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false); if (!this.ValidateHtml(html)) { throw new HttpRequestException("HTML returned was not verified."); } this.SetHtml(html); } else { throw new HttpRequestException("HTML returned was not verified."); } }