/// <summary>
/// 使用代理池
/// </summary>
/// <param name="spider"></param>
/// <param name="proxyFinder">代理获取</param>
/// <param name="proxyValidator">代理有效验证</param>
/// <param name="reuseInterval">复用间隔时间(ms)</param>
/// <returns></returns>
public static Spider UseHttpProxyPools(this Spider spider, IHttpProxyFinder proxyFinder, IProxyValidator proxyValidator, int reuseInterval, int refreshInterval = 2 * 1000)
{
DownloaderProxyPools downloaderProxy = new DownloaderProxyPools(proxyFinder, proxyValidator, reuseInterval, refreshInterval);
spider.SetDownloaderProxy(downloaderProxy);
return spider;
}
public HttpProxyPool(IHttpProxyFinder finder, IProxyValidator proxyValidator, int reuseInterval, int refreshInterval = 2 * 1000)
{
_finder = finder;
_reuseInterval = reuseInterval;
_refreshInterval = refreshInterval;
if (refreshInterval < 100)
{
throw new ArgumentException("'refreshInterval' must greater than 100 ");
}
_proxyValidator = proxyValidator;
Task.Run(() => RefreshProxies());
}
public DownloaderProxyPools(IHttpProxyFinder proxyFinder, IProxyValidator proxyValidator, int reuseInterval, int refreshInterval)
{
proxyPool = new HttpProxyPool(proxyFinder, proxyValidator, reuseInterval, refreshInterval);
}
/// <summary>
/// 使用代理池
/// </summary>
public static Spider UseHttpProxyPools(this Spider spider, IHttpProxyFinder proxyFinder)
{
return UseHttpProxyPools(spider, proxyFinder, new DefaultProxyValidator(), 5000);
}
/// <summary>
/// 使用代理池
/// </summary>
public static Spider UseHttpProxyPools(this Spider spider, IHttpProxyFinder proxyFinder, int reuseInterval, int refreshInterval = 2 * 1000)
{
return UseHttpProxyPools(spider, proxyFinder, new DefaultProxyValidator(), reuseInterval, refreshInterval);
}
