Пример #1
0
        public Image GetImage(string uri, string postData, string referer)
        {
            WaitAMoment();
            lock (_lockImageSerialize)
            {
                Image img = null;

                HttpWebResponse response       = null;
                Stream          responseStream = null;
                StreamReader    sr             = null;

                try
                {
                    if (referer == null)
                    {
                        referer = _referer;
                    }

                    if (BeginRequest != null)
                    {
                        HttpClientEventArgs e = new HttpClientEventArgs();
                        e.Uri      = uri;
                        e.PostData = postData;
                        e.Referer  = referer;
                        BeginRequest(e);
                    }

                    var _request = (HttpWebRequest)WebRequest.Create(uri);
                    _request.Referer         = referer;
                    _request.Accept          = _accept;
                    _request.UserAgent       = _userAgent;
                    _request.CookieContainer = _cookieContainer;
                    _request.Timeout         = Timeout;
                    if (_proxy != null)
                    {
                        _request.Proxy = _proxy;
                    }
                    _request.AllowWriteStreamBuffering = true;
                    _request.AllowAutoRedirect         = true;
                    _request.KeepAlive = true;

                    foreach (Cookie cookie in _cookieCollection)
                    {
                        _cookieContainer.SetCookies(_request.RequestUri, cookie.Name + "=" + cookie.Value);
                    }

                    if (string.IsNullOrEmpty(postData))
                    {
                        _request.Method = "GET";
                    }
                    else
                    {
                        byte[] bytes = Encoding.GetEncoding(_charset).GetBytes(postData);

                        _request.Method        = "POST";
                        _request.ContentType   = "application/x-www-form-urlencoded";
                        _request.ContentLength = bytes.Length;
                        using (Stream writer = _request.GetRequestStream())
                        {
                            writer.Write(bytes, 0, bytes.Length);
                        }
                    }

                    response         = (HttpWebResponse)_request.GetResponse();
                    _cookieContainer = _request.CookieContainer;
                    responseStream   = response.GetResponseStream();

                    img = Image.FromStream(responseStream);

                    if (EndRequest != null)
                    {
                        HttpClientEventArgs e = new HttpClientEventArgs();
                        e.Uri           = uri;
                        e.PostData      = postData;
                        e.Referer       = referer;
                        e.ResponseImage = img;
                        EndRequest(e);
                    }
                }
                catch (Exception ex)
                {
                    log.Error("获取图片出现异常!", ex);
                    //LogCache.Instance.AddInfo(
                    //    $"HttpClient.GetImage() 发生异常。(url:{uri},postdata:{postData},referer{referer}) \n异常详情:{ex}\n{ex.StackTrace}");
                    if (Error != null)
                    {
                        Error(ex);
                    }
                }
                finally
                {
                    if (response != null)
                    {
                        response.Close();
                        response = null;
                    }

                    if (responseStream != null)
                    {
                        responseStream.Close();
                        responseStream = null;
                    }

                    if (sr != null)
                    {
                        sr.Close();
                        sr = null;
                    }
                    if (img == null)
                    {
                        //LogCache.Instance.AddInfo(
                        //    $"HttpClient.GetImage() 请求image为null。(invokmethod{new StackFrame(1).GetMethod()}, url:{uri},postdata:{postData},referer{referer}) ");
                        log.Error("图片为空!");
                    }
                }

                return(img);
            }
        }
Пример #2
0
        public string Open(string uri, string postData, string referer)
        {
            HttpWebRequest _request;
            int            i  = 0;
            var            sw = new Stopwatch();

            sw.Start();
            //WaitAMoment();
            //lock (_lockOpenSerialize)
            {
                string html = null;

                HttpWebResponse response       = null;
                Stream          responseStream = null;
                StreamReader    sr             = null;

                try
                {
                    if (RequestInterval > 0)
                    {
                        TimeSpan ts    = new TimeSpan(DateTime.Now.Ticks - _ticks);
                        double   sleep = ts.TotalMilliseconds;
                        if (sleep < RequestInterval)
                        {
                            Thread.Sleep((int)(RequestInterval - sleep));
                        }

                        _ticks = DateTime.Now.Ticks;
                    }

                    if (referer == null)
                    {
                        referer = _referer;
                    }

                    if (BeginRequest != null)
                    {
                        HttpClientEventArgs e = new HttpClientEventArgs();
                        e.Uri      = uri;
                        e.PostData = postData;
                        e.Referer  = referer;
                        BeginRequest(e);
                    }

                    if (uri.StartsWith("https", StringComparison.OrdinalIgnoreCase))
                    {
                        ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
                        _request = (HttpWebRequest)WebRequest.Create(uri);
                        _request.ProtocolVersion = HttpVersion.Version10;
                    }
                    else
                    {
                        _request = (HttpWebRequest)WebRequest.Create(uri);
                    }
                    _request.Referer = referer;
                    _request.Headers.Add("x-requested-with", "XMLHttpRequest");
                    _request.Accept          = _accept;
                    _request.UserAgent       = _userAgent;
                    _request.CookieContainer = _cookieContainer;
                    _request.Timeout         = Timeout;

                    if (_proxy != null)
                    {
                        _request.Proxy = _proxy;
                    }
                    _request.AllowWriteStreamBuffering = true;
                    _request.AllowAutoRedirect         = AllowAutoRedirect;
                    _request.KeepAlive = true;

                    if (AddRequestHeader != null)
                    {
                        foreach (var addheader in AddRequestHeader)
                        {
                            _request.Headers.Add(addheader.Key, addheader.Value);
                        }
                    }

                    foreach (Cookie cookie in _cookieCollection)
                    {
                        _cookieContainer.SetCookies(_request.RequestUri, cookie.Name + "=" + cookie.Value);
                    }

                    if (string.IsNullOrEmpty(postData))
                    {
                        _request.Method = "GET";
                    }
                    else
                    {
                        byte[] bytes = Encoding.GetEncoding(_charset).GetBytes(postData);

                        _request.Method        = "POST";
                        _request.ContentType   = _contentType;
                        _request.ContentLength = bytes.Length;

                        using (Stream writer = _request.GetRequestStream())
                        {
                            writer.Write(bytes, 0, bytes.Length);
                        }
                    }
                    response = (HttpWebResponse)_request.GetResponse();

                    ResponeHeader = response.Headers;

                    #region Cookie管理

                    // 1.当前地址的cookie (基础类自动处理,不会处理非当前路径的cookie)
                    _cookieContainer = _request.CookieContainer;

                    // 2. 处理response返回的所有cookie(非当前url路径的)
                    string allCookies = response.Headers["Set-Cookie"];
                    if (!string.IsNullOrWhiteSpace(allCookies))
                    {
                        string pathSplit = "path=";
                        while (allCookies.Contains(pathSplit))
                        {
                            #region 分解["Set-Cookie"]字符串

                            string cookieText =
                                allCookies.Substring(0, allCookies.IndexOf(pathSplit,
                                                                           StringComparison.CurrentCultureIgnoreCase));
                            allCookies = allCookies.Substring($"{cookieText}{pathSplit}".Length);

                            int nextCookieSplitIndex = allCookies.IndexOf(",",
                                                                          StringComparison.CurrentCultureIgnoreCase);

                            string cookiePath = nextCookieSplitIndex == -1
                                ? allCookies
                                : allCookies.Substring(0, nextCookieSplitIndex);

                            allCookies = nextCookieSplitIndex == -1
                                ? string.Empty
                                : allCookies.Substring($"{cookiePath},".Length);

                            #endregion

                            #region 注入 _cookieContainer

                            string cookiePathFull;
                            cookieText = cookieText.Trim();
                            cookiePath = cookiePath.Trim();
                            if (cookiePath.Contains("://"))
                            {
                                cookiePathFull = cookiePath;
                            }
                            else
                            {
                                cookiePathFull = $"{_request.RequestUri.Scheme}://{_request.RequestUri.Authority}{cookiePath}";
                            }
                            _cookieContainer.SetCookies(new Uri(cookiePathFull), cookieText);

                            #endregion
                        }
                    }

                    #endregion


                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                    {
                        responseStream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                    }
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                    {
                        responseStream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                    }
                    else
                    {
                        responseStream = response.GetResponseStream();
                    }

                    sr   = new StreamReader(responseStream, Encoding.GetEncoding(_charset));
                    html = sr.ReadToEnd();

                    if (EndRequest != null)
                    {
                        HttpClientEventArgs e = new HttpClientEventArgs();
                        e.Uri          = uri;
                        e.PostData     = postData;
                        e.Referer      = referer;
                        e.ResponseText = html;
                        EndRequest(e);
                    }
                }
                catch (Exception ex)
                {
                    //LogCache.Instance.AddInfo(
                    //    $"HttpClient.Open() 发生异常。(url:{uri},postdata:{postData},referer{referer}) \n异常详情:{ex}\n{ex.StackTrace}");
                    //if (Error != null)
                    //{
                    //    Error(ex);
                    //}
                    log.Error("请求发生异常", ex);
                }
                finally
                {
                    _referer = uri;

                    if (response != null)
                    {
                        response.Close();
                        response = null;
                    }

                    if (responseStream != null)
                    {
                        responseStream.Close();
                        responseStream = null;
                    }

                    if (sr != null)
                    {
                        sr.Close();
                        sr = null;
                    }
                    if (html == null)
                    {
                        log.Error($"请求html为null,url:{uri},postdata:{postData}");
                        //LogCache.Instance.AddInfo(
                        //    $"HttpClient.Open() 请求html为null。(invokmethod{new StackFrame(1).GetMethod()}, url:{uri},postdata:{postData},referer{referer}) ");
                    }
                }

                sw.Stop();
                //log.Debug($"{i}请求服务花费了{sw.ElapsedMilliseconds}ms。");
                i++;
                return(html);
            }
        }