Beispiel #1
0
 private void threadRunRefreshSource()
 {
     while (true)
     {
         List <ProxyServer> ps = retrieveProxySource(true, true, true);
         //推送代理更新到代理池
         RunningCathe.refreshPoolProxyFromSource(this);
         //如果proxyServerList元素个数超过缓存容量,则调用缓存垃圾收集
         if (proxyServerList.Count > catheSize)
         {
             RunningCathe.catheTrashCollectBySource(this);
         }
         Thread.Sleep(refreshTimeSpan);
     }
 }
Beispiel #2
0
        /// <summary>
        /// 根据配置的url及参数,从代理源查询最新的代理清单。
        /// 根据入参将结果保存到ProxyServerList,并去重
        /// </summary>
        /// <param name="updateProxyServerList">是否更新ProxyServerList</param>
        /// <param name="saveToDB">是否保存到DB</param>
        /// <param name="distinct">是否去重</param>
        /// <returns></returns>
        public List <ProxyServer> retrieveProxySource(bool updateProxyServerList = true, bool saveToDB = true, bool rtnDistinct = true)
        {
            //校验缓存池是否超容
            if (catheSize < proxyServerList.Count)
            {
                RunningCathe.addLog(new Log(DateTime.Now, "从代理源[" + this.name + "]获取代理服务器时发生异常:缓存池容量已满。当前缓存代理数量:" + proxyServerList.Count + ",缓存池容量:" + catheSize.ToString(), 0, 0, this.id));
                return(null);
            }

            if (allUrl.Count > 0)
            {
                List <ProxyServer> psList = new List <ProxyServer>();
                for (int i = 0; i < allUrl.Count; i++)
                {
                    try
                    {
                        System.Net.WebProxy wb = RunningCathe.getAnyActiveProxy();
                        var task               = HttpAdapter.CreateGetHttpResponse(allUrl[i], "", null, wb, 0);
                        var response           = task.Result;
                        List <ProxyServer> psl = new List <ProxyServer>();
                        using (Stream stream = response.GetResponseStream())
                        {
                            //StreamReader sr = new StreamReader(stream, System.Text.Encoding.GetEncoding(charset));
                            StreamReader sr = new StreamReader(stream, System.Text.Encoding.GetEncoding(charset), true);
                            //StreamReader sr = new StreamReader(stream, true);

                            string content = sr.ReadToEnd();


                            if (searchType.Equals("XPATH", StringComparison.OrdinalIgnoreCase))
                            {
                                psl = getProxyServerByXpath(content);
                            }
                            else
                            {
                                psl = getProxyServerByRegex(content);
                            }
                        }
                        psList.AddRange(psl);
                        RunningCathe.addLog(new Log(DateTime.Now, "从" + allUrl[i] + "获取代理服务器" + psl.Count.ToString() + "个", 0, 0, this.id));
                        Thread.Sleep(requestTimeSpan);
                    }
                    catch (Exception ex)
                    {
                        RunningCathe.addLog(new Log(DateTime.Now, "从" + allUrl[i] + "获取代理服务器时发生异常:" + ex.Message, 0, 0, this.id));
                    }
                }
                int originalSize;
                lock (thisLock)
                {
                    originalSize = proxyServerList.Count;
                    if (updateProxyServerList)
                    {
                        proxyServerList.AddRange(psList);
                        proxyServerList = proxyServerList.Distinct(new ProxyServerCompare()).ToList();
                    }
                }
                RunningCathe.addLog(new Log(DateTime.Now, "从代理源\"" + this.id + "-" + this.name + "\"获取代理服务器" + (proxyServerList.Count - originalSize).ToString() + "个", 0, 0, this.id));
                if (saveToDB)
                {
                    for (int i = originalSize; i < proxyServerList.Count; i++)
                    {
                        try
                        {
                            proxyServerList[i].saveToDB();
                        }
                        catch (Exception ex)
                        {
                            RunningCathe.addLog(new Log(DateTime.Now, "保存代理服务器信息时出错:" + proxyServerList[i].ProxyIpAddress + "," + proxyServerList[i].ProxyPort + "," + proxyServerList[i].ProxyProtocal
                                                        + "," + proxyServerList[i].ProxyLocation + "," + proxyServerList[i].ProxyType + "。" + ex.Message, 0, 0, this.id));
                        }
                    }
                }
                if (rtnDistinct)
                {
                    return(proxyServerList.GetRange(originalSize, proxyServerList.Count - originalSize));
                }
                else
                {
                    return(psList);
                }
            }
            else
            {
                return(null);
            }
        }