Example #1
0
        /// <summary>
        /// Http请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="requestParams"></param>
        /// <param name="requestType"></param>
        /// <param name="cookieContainer"></param>
        /// <param name="referer"></param>
        /// <param name="contentType"></param>
        /// <param name="accept"></param>
        /// <param name="isNeedSleep"></param>
        /// <param name="host"></param>
        /// <param name="isEnableFreePrxy"></param>
        /// <returns></returns>
        public static DataResult <string> QueryRequest(string url, string requestParams = "", RequestEnum requestType = RequestEnum.GET, CookieContainer cookieContainer = null, string referer = "", string contentType = "", string accept = "", bool isNeedSleep = false, string host = "", bool isEnableFreePrxy = false)
        {
            var dataResult = new DataResult <string>();

            var retryCount = 2;

            if (isNeedSleep)
            {
                SpinWait.SpinUntil(() => false, TimeSpan.FromSeconds(new Random().Next(1, 2)));
            }

Retry:

            try
            {
                HttpWebRequest httpRequest;

                if (requestType == RequestEnum.POST)
                {
                    httpRequest = (HttpWebRequest)WebRequest.Create(url);
                }
                else
                {
                    if (string.IsNullOrWhiteSpace(requestParams))
                    {
                        httpRequest = (HttpWebRequest)WebRequest.Create(url);
                    }
                    else
                    {
                        httpRequest = (HttpWebRequest)WebRequest.Create(url + "?" + requestParams.Trim());
                    }
                }

                httpRequest.Method = requestType.ToString();

                httpRequest.Timeout = 30 * 1000;

                httpRequest.ContentType = string.IsNullOrWhiteSpace(contentType) ? defaultContentType : contentType;

                httpRequest.Accept = string.IsNullOrWhiteSpace(accept) ? defaultAccept : accept;

                httpRequest.Headers.Add("Accept-Encoding", "gzip, deflate");

                httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;

                httpRequest.UserAgent = userAgent;

                httpRequest.CookieContainer = cookieContainer;

                if (string.IsNullOrEmpty(requestParams))
                {
                    httpRequest.ContentLength = 0;
                }

                httpRequest.Headers["X-Forwarded-For"] = BaseFanctory.GetRandomIp();

                if (!url.IsInnerIP() && (!string.IsNullOrEmpty(host) || isEnableFreePrxy))
                {
                    if (!string.IsNullOrEmpty(host))
                    {
                        var index = host.IndexOf(":", StringComparison.Ordinal);

                        httpRequest.Proxy = new WebProxy(host.Substring(0, index), Convert.ToInt32(host.Substring(index + 1)));
                    }
                    else
                    {
                        var proxy = ProxyFanctory.GetFreeProxy();

                        httpRequest.Proxy = new WebProxy(proxy.Ip, proxy.Port);
                    }
                }

                if (!string.IsNullOrWhiteSpace(referer))
                {
                    httpRequest.Referer = referer;
                }

                if (requestType == RequestEnum.POST && !string.IsNullOrWhiteSpace(requestParams))
                {
                    var encoding = Encoding.GetEncoding("utf-8");

                    var bytesToPost = encoding.GetBytes(requestParams);

                    httpRequest.ContentLength = bytesToPost.Length;

                    var requestStream = httpRequest.GetRequestStream();

                    requestStream.Write(bytesToPost, 0, bytesToPost.Length);

                    requestStream.Close();
                }

                var response = (HttpWebResponse)httpRequest.GetResponse();

                var stream = response.GetResponseStream();

                var reStr = string.Empty;

                if (stream != null)
                {
                    var sr = new StreamReader(stream, Encoding.GetEncoding("utf-8"));

                    reStr = sr.ReadToEnd();

                    sr.Close();
                }
                else
                {
                    dataResult.IsSuccess = false;
                }

                response.Close();

                dataResult.Data = reStr;

                return(dataResult);
            }
            catch (WebException ex)
            {
                if (isEnableFreePrxy)
                {
                    ProxyFanctory.NextProxy();
                }

                if (string.IsNullOrWhiteSpace(host) && --retryCount >= 0)
                {
                    if (ex.Status == WebExceptionStatus.ProtocolError)
                    {
                        SpinWait.SpinUntil(() => false, TimeSpan.FromSeconds(2));
                    }

                    goto Retry;
                }

                dataResult.IsSuccess = false;

                var hostStr = !string.IsNullOrEmpty(host) ? "Host:" + host : "";

                dataResult.ErrorMsg = $"Web响应异常,请求Url:{url},{hostStr} 异常信息:{ex.Message}";

                return(dataResult);
            }
            catch (Exception ex)
            {
                var hostStr = !string.IsNullOrEmpty(host) ? "Host:" + host : "";

                LogFactory.Error($"请求异常!请求Url:{url} 请求参数:{requestParams} {hostStr} {Environment.NewLine}异常信息:{ex.Message}{Environment.NewLine}{ex.StackTrace}");

                dataResult.IsSuccess = false;

                dataResult.ErrorMsg = $"请求异常!请求Url:{url} {hostStr} 异常信息:{ex.Message} 详情请错误看日志!";

                return(dataResult);
            }
        }
        /// <summary>
        /// HttpClient 请求
        /// </summary>
        /// <param name="url"></param>
        /// <param name="method"></param>
        /// <param name="dictionary"></param>
        /// <param name="cookieContainer"></param>
        /// <param name="referer"></param>
        /// <param name="contentType"></param>
        /// <param name="accept"></param>
        /// <param name="userAgent"></param>
        /// <param name="isRandomIP"></param>
        /// <param name="webProxy"></param>
        /// <returns></returns>
        private static DataResult <HttpContent> Request(string url, HttpMethod method, IEnumerable <KeyValuePair <string, string> > dictionary = null, CookieContainer cookieContainer = null, string referer = "", string contentType = "", string accept = "", string userAgent = "", bool isRandomIP = false, IWebProxy webProxy = null)
        {
            var dataResult = new DataResult <HttpContent>();

            if (url.Contains("https"))
            {
                ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, errors) => true;
            }

            try
            {
                using (var client = new HttpClient(new HttpClientHandler
                {
                    CookieContainer = cookieContainer ?? new CookieContainer(),
                    Proxy = webProxy,
                    AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.None
                }))
                {
                    client.DefaultRequestHeaders.Add("Accept", string.IsNullOrWhiteSpace(accept) ? defaultAccept : accept);

                    client.DefaultRequestHeaders.Add("User-Agent", string.IsNullOrWhiteSpace(userAgent) ? defaultUserAgent : userAgent);

                    //if (isRandomIP) client.DefaultRequestHeaders.Add("X-Forwarded-For", BaseFanctory.GetRandomHost());

                    if (!string.IsNullOrEmpty(referer))
                    {
                        client.DefaultRequestHeaders.Add("Referer", referer);
                    }

                    var httpRequestMessage = new HttpRequestMessage(method, new Uri(url));

                    if (method == HttpMethod.Post && dictionary != null)
                    {
                        var httpContent = new FormUrlEncodedContent(dictionary);

                        httpContent.Headers.ContentType = new MediaTypeHeaderValue(string.IsNullOrEmpty(contentType) ? defaultContentType : contentType);

                        httpRequestMessage.Content = httpContent;
                    }

                    dataResult.Data = client.SendAsync(httpRequestMessage).Result.Content;

                    return(dataResult);
                }
            }
            catch (WebException ex)
            {
                dataResult.IsSuccess = false;

                dataResult.ErrorMsg = $"Web响应异常,请求Url:{url},{JsonConvert.SerializeObject(dictionary)} 异常信息:{ex.Message}";

                return(dataResult);
            }
            catch (Exception ex)
            {
                while (true)
                {
                    if (ex.InnerException == null)
                    {
                        break;
                    }

                    ex = ex.InnerException;
                }

                LogFactory.Error($"请求异常!请求Url:{url} 请求参数:{JsonConvert.SerializeObject(dictionary)} {Environment.NewLine}异常信息:{ex.Message}{Environment.NewLine}{ex.StackTrace}");

                dataResult.IsSuccess = false;

                dataResult.ErrorMsg = $"请求异常!请求Url:{url} 异常信息:{ex.Message} 详情请错误看日志!";

                return(dataResult);
            }
            finally
            {
                if (cookieContainer != null)
                {
                    cookieContainer.GetAllCookies().Select(s => s.Domain).Distinct().ToList().ForEach(f =>
                    {
                        foreach (var cookie in cookieContainer.GetCookies(new Uri("http://www" + f)).OfType <Cookie>())
                        {
                            cookie.Expired = false;
                        }
                    });

                    _cookieContainer.GetAllCookies().ForEach(cookieContainer.Add);
                }
            }
        }
Example #3
0
        /// <summary>
        /// 下载文件
        /// </summary>
        /// <param name="url"></param>
        /// <param name="path"></param>
        /// <param name="requestParams"></param>
        /// <param name="cookieContainer"></param>
        /// <returns></returns>
        public static DataResult HttpDownloadFile(string url, string path, string requestParams = "", CookieContainer cookieContainer = null)
        {
            var dataResult = new DataResult();

            try
            {
                var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);

                var httpRequest = (HttpWebRequest)WebRequest.Create(url);

                httpRequest.ContentType = "application/x-www-form-urlencoded";

                httpRequest.Headers["Upgrade-Insecure-Requests"] = "1";

                httpRequest.Method = "POST";

                httpRequest.ContentLength = 0;

                httpRequest.CookieContainer = cookieContainer;

                httpRequest.Headers.Add("Accept-Encoding", "gzip, deflate");

                httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;

                httpRequest.UserAgent = userAgent;

                httpRequest.Accept = defaultAccept;

                if (!string.IsNullOrWhiteSpace(requestParams))
                {
                    var encoding = Encoding.GetEncoding("utf-8");

                    var bytesToPost = encoding.GetBytes(requestParams);

                    httpRequest.ContentLength = bytesToPost.Length;

                    var requestStream = httpRequest.GetRequestStream();

                    requestStream.Write(bytesToPost, 0, bytesToPost.Length);

                    requestStream.Close();
                }

                using (var response = httpRequest.GetResponse())
                {
                    using (var responseStream = response.GetResponseStream())
                    {
                        var bArr = new byte[1024];

                        if (responseStream != null)
                        {
                            var size = responseStream.Read(bArr, 0, bArr.Length);

                            while (size > 0)
                            {
                                fs.Write(bArr, 0, size);

                                size = responseStream.Read(bArr, 0, bArr.Length);
                            }

                            responseStream.Close();
                        }

                        fs.Close();
                    }
                }
            }
            catch (Exception ex)
            {
                LogFactory.Error($"下载异常,请求Url:{url},{Environment.NewLine}异常信息:{ex.Message}{Environment.NewLine}{ex.StackTrace}");

                dataResult.IsSuccess = false;

                dataResult.ErrorMsg = $"请求异常,请求Url:{url},异常信息:{ex.Message},详情请错误看日志!";
            }

            return(dataResult);
        }