private void threadRunRefreshSource()
{
while (true)
{
List <ProxyServer> ps = retrieveProxySource(true, true, true);
//推送代理更新到代理池
RunningCathe.refreshPoolProxyFromSource(this);
//如果proxyServerList元素个数超过缓存容量，则调用缓存垃圾收集
if (proxyServerList.Count > catheSize)
{
RunningCathe.catheTrashCollectBySource(this);
}
Thread.Sleep(refreshTimeSpan);
}
}
/// <summary>
/// 根据配置的url及参数，从代理源查询最新的代理清单。
/// 根据入参将结果保存到ProxyServerList，并去重
/// </summary>
/// <param name="updateProxyServerList">是否更新ProxyServerList</param>
/// <param name="saveToDB">是否保存到DB</param>
/// <param name="distinct">是否去重</param>
/// <returns></returns>
public List <ProxyServer> retrieveProxySource(bool updateProxyServerList = true, bool saveToDB = true, bool rtnDistinct = true)
{
//校验缓存池是否超容
if (catheSize < proxyServerList.Count)
{
RunningCathe.addLog(new Log(DateTime.Now, "从代理源[" + this.name + "]获取代理服务器时发生异常：缓存池容量已满。当前缓存代理数量：" + proxyServerList.Count + ",缓存池容量：" + catheSize.ToString(), 0, 0, this.id));
return(null);
}
if (allUrl.Count > 0)
{
List <ProxyServer> psList = new List <ProxyServer>();
for (int i = 0; i < allUrl.Count; i++)
{
try
{
System.Net.WebProxy wb = RunningCathe.getAnyActiveProxy();
var task = HttpAdapter.CreateGetHttpResponse(allUrl[i], "", null, wb, 0);
var response = task.Result;
List <ProxyServer> psl = new List <ProxyServer>();
using (Stream stream = response.GetResponseStream())
{
//StreamReader sr = new StreamReader(stream, System.Text.Encoding.GetEncoding(charset));
StreamReader sr = new StreamReader(stream, System.Text.Encoding.GetEncoding(charset), true);
//StreamReader sr = new StreamReader(stream, true);
string content = sr.ReadToEnd();
if (searchType.Equals("XPATH", StringComparison.OrdinalIgnoreCase))
{
psl = getProxyServerByXpath(content);
}
else
{
psl = getProxyServerByRegex(content);
}
}
psList.AddRange(psl);
RunningCathe.addLog(new Log(DateTime.Now, "从" + allUrl[i] + "获取代理服务器" + psl.Count.ToString() + "个", 0, 0, this.id));
Thread.Sleep(requestTimeSpan);
}
catch (Exception ex)
{
RunningCathe.addLog(new Log(DateTime.Now, "从" + allUrl[i] + "获取代理服务器时发生异常：" + ex.Message, 0, 0, this.id));
}
}
int originalSize;
lock (thisLock)
{
originalSize = proxyServerList.Count;
if (updateProxyServerList)
{
proxyServerList.AddRange(psList);
proxyServerList = proxyServerList.Distinct(new ProxyServerCompare()).ToList();
}
}
RunningCathe.addLog(new Log(DateTime.Now, "从代理源\"" + this.id + "-" + this.name + "\"获取代理服务器" + (proxyServerList.Count - originalSize).ToString() + "个", 0, 0, this.id));
if (saveToDB)
{
for (int i = originalSize; i < proxyServerList.Count; i++)
{
try
{
proxyServerList[i].saveToDB();
}
catch (Exception ex)
{
RunningCathe.addLog(new Log(DateTime.Now, "保存代理服务器信息时出错：" + proxyServerList[i].ProxyIpAddress + "," + proxyServerList[i].ProxyPort + "," + proxyServerList[i].ProxyProtocal
+ "," + proxyServerList[i].ProxyLocation + "," + proxyServerList[i].ProxyType + "。" + ex.Message, 0, 0, this.id));
}
}
}
if (rtnDistinct)
{
return(proxyServerList.GetRange(originalSize, proxyServerList.Count - originalSize));
}
else
{
return(psList);
}
}
else
{
return(null);
}
}