Esempio n. 1
0
        public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
        {
            if (!this.CanDownload())
            {
                return;
            }

            ScraperHelper.SetOrigenToClient(this.Url, client);
            Console.WriteLine("Downloading: " + this.Url);
            var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false);

            if (result.IsSuccessStatusCode)
            {
                var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);

                if (!this.ValidateHtml(html))
                {
                    throw new HttpRequestException("HTML returned was not verified.");
                }
                this.Html = html;
            }
            else
            {
                throw new HttpRequestException("HTML returned was not verified.");
            }
        }
Esempio n. 2
0
        public virtual async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
        {
            if (!this.CanDownload())
            {
                logger.Debug("Can download returned false, stopping the download job.");
                return;
            }

            ScraperHelper.SetOrigenToClient(this.Url, client);
            if (this.headers != null)
            {
                foreach (KeyValuePair <string, string> keyValuePair in headers)
                {
                    client.DefaultRequestHeaders.Add(keyValuePair.Key, keyValuePair.Value);
                }
            }

            logger.Trace("Downloading: " + this.Url);
            var result = await client.GetAsync(Url, cancelToken).ConfigureAwait(false);

            if (result.IsSuccessStatusCode)
            {
                var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);

                if (!this.ValidateHtml(html))
                {
                    logger.Warn("Failed HTML verification");
                    throw new HttpRequestException("HTML returned was not verified.");
                }
                this.Html = html;
            }
            else
            {
                throw new HttpRequestException(string.Format("HTML request was not successfull. Status Code: {0} ", result.StatusCode));
            }
        }
Esempio n. 3
0
        public override async Task ExecuteDownload(HttpClient client, CancellationToken cancelToken)
        {
            var uri = new Uri(this.GetUrl());

            client.BaseAddress = this.GetUrl().Contains("https:") ? new Uri("https://" + uri.Host) : new Uri("http://" + uri.Host);
            ScraperHelper.SetOrigenToClient(this.GetUrl(), client);

            var message = new HttpRequestMessage(HttpMethod.Post, uri.LocalPath);

            if (this.headers != null)
            {
                foreach (var header in this.headers)
                {
                    message.Headers.Add(header.Key, header.Value);
                }
            }

            var keyValue = new List <KeyValuePair <string, string> >();

            if (this.formData != null)
            {
                keyValue.AddRange(this.formData);
            }
            if (this.formString != null)
            {
                message.Content = new StringContent(this.formString, Encoding.UTF8, "application/json");
            }
            else
            {
                StringBuilder stringBuilder = new StringBuilder();
                foreach (KeyValuePair <string, string> current in keyValue)
                {
                    if (stringBuilder.Length > 0)
                    {
                        stringBuilder.Append('&');
                    }

                    stringBuilder.Append(Encode(current.Key));
                    stringBuilder.Append('=');
                    stringBuilder.Append(Encode(current.Value));
                }

                message.Content = new StringContent(stringBuilder.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded");
                //message.Content = new MyFormUrlEncodedContent(keyValue);
            }

            var result = await client.SendAsync(message, cancelToken).ConfigureAwait(false);

            this.responseHeaders = result.Headers;
            if (result.IsSuccessStatusCode)
            {
                var html = await result.Content.ReadAsStringAsync().WithCancellation(cancelToken).ConfigureAwait(false);

                if (!this.ValidateHtml(html))
                {
                    throw new HttpRequestException("HTML returned was not verified.");
                }
                this.SetHtml(html);
            }
            else
            {
                throw new HttpRequestException("HTML returned was not verified.");
            }
        }