public ProxyEntity Get(ProxyEntity model)
{
using (LiGatherContext _db = new LiGatherContext())
{
return _db.ProxyEntities.SingleOrDefault(t => t.Id == model.Id);
}
}
public bool IsExist(ProxyEntity model)
{
using (LiGatherContext _db = new LiGatherContext())
{
return _db.ProxyEntities.Any(t => t.IpAddress == model.IpAddress);
}
}
public void Update(ProxyEntity model)
{
using (LiGatherContext _db = new LiGatherContext())
{
_db.ProxyEntities.AddOrUpdate(model);
_db.SaveChanges();
}
}
public void LockUpdate(ProxyEntity model)
{
using (LiGatherContext _db = new LiGatherContext())
{
var proxyEntity = _db.ProxyEntities.Where(t => t.Id == model.Id && t.CanUse == true).ToList();
if (!proxyEntity.Any()) return;
_db.ProxyEntities.AddOrUpdate(model);
var i = _db.SaveChanges();
if (i == 0)
{
Console.WriteLine("EF 内存泄露，启动清扫策略。");
//日志记录
new LogDomain().Add(new LogEntity { ErrorDetails = "EF 内存泄露，启动清扫策略。", TriggerTime = DateTime.Now });
_db.Database.ExecuteSqlCommand("update ProxyEntity set CanUse=0 where CanUse!=0");
}
Console.WriteLine("移除失效代理:{0}:{1}", model.IpAddress, model.Port);
}
}
/// <summary>
/// 移除失效代理
/// </summary>
private void RemoveOldIp(ProxyEntity model)
{
model.CanUse = false;
model.LastUseTime = DateTime.Now;
new ProxyDomain().LockUpdate(model);
}
/// <summary>
/// 直接采集IP并更新到数据库
/// </summary>
/// <param name="countNum">采集IP总数</param>
/// <param name="getNum">每次提取数量</param>
/// <param name="isValidate">是否对代理验证</param>
public void ProxySave(int countNum = 100, int getNum = 5, bool isValidate = false)
{
new Thread(() =>
{
while (true)
{
if (countNum == 0)
break;
var ipLists = GetProxyByHttp(getNum).Split(Environment.NewLine.ToCharArray()).ExceptNull().ToList();
foreach (var ipList in ipLists)
{
var ipAndPort = ipList.Split(':');
if (string.IsNullOrWhiteSpace(ipList) || ipAndPort.Length < 2)
continue;
Console.WriteLine("获取到代理：" + ipList);
var model = new ProxyEntity();
model.IpAddress = ipAndPort[0];
model.Port = Conv.ToInt(ipAndPort[1]);
model.Usage = 0;
model.CreateTime = DateTime.Now;
ProxyDomain proxyDomain = new ProxyDomain();
if (isValidate)
{
if (ThreadValidate.VerificationIp(model.IpAddress, model.Port))
{
if (!proxyDomain.IsExist(model))
proxyDomain.Add(model);
countNum--;
}
}
if (!proxyDomain.IsExist(model))
proxyDomain.Add(model);
countNum--;
}
}
Console.WriteLine("IP采集完毕");
})
{ IsBackground = false }.Start();
}
/// <summary>
/// 通过在线代理直接获取未验证的IP代理
/// 主要用于本地IP代理库存不足，临时获取
/// </summary>
/// <returns></returns>
public ProxyEntity GetHttProxyEntity()
{
var ipList = GetProxyByHttp();
var ipAndPort = ipList.Split(':');
var model = new ProxyEntity();
model.IpAddress = ipAndPort[0];
model.Port = Conv.ToInt(ipAndPort[1]);
model.Usage = 1;
model.CanUse = true;
model.CreateTime = DateTime.Now;
model.LastUseTime = DateTime.Now;
ProxyDomain proxyDomain = new ProxyDomain();
if (!proxyDomain.IsExist(model))
proxyDomain.Add(model);
return model;
}