public override void Handle(ref Page page, ISpider spider)
{
if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(Content) && CookieInjector != null && page.Content.Contains(Content))
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
spider.Exit();
}
Spider.AddToCycleRetry(page.Request, spider.Site);
CookieInjector?.Inject(spider);
page.Exception = new DownloadException($"Content downloaded contains string: {Content}.");
}
}
public override void Handle(ref Page page, ISpider spider)
{
if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(Content) && page.Content.Contains(Content))
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
Logger.MyLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error);
spider.Exit();
}
page = Spider.AddToCycleRetry(page.Request, spider.Site);
page.Exception = new DownloadException($"Content downloaded contains string: {Content}.");
}
}
/// <summary>
/// 当页面数据中的异常信息包含指定内容时触发ADSL拨号
/// </summary>
/// <param name="page">页面数据</param>
/// <param name="spider">爬虫</param>
public override void Handle(ref Page page, ISpider spider)
{
if (page == null || string.IsNullOrEmpty(page.Content) || string.IsNullOrWhiteSpace(page.Content) || page.Exception == null)
{
return;
}
if (page.Exception.Message.Contains(_exceptionMessage))
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
Logger.AllLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error);
spider.Exit();
}
Spider.AddToCycleRetry(page.Request, spider.Site);
page.Exception = new DownloadException("Download failed and redial finished already.");
}
}
/// <summary>
/// 当页面数据包含指定内容时触发ADSL拨号, 并且重新获取Cookie
/// </summary>
/// <param name="page">页面数据</param>
/// <param name="spider">爬虫</param>
public override void Handle(ref Page page, ISpider spider)
{
if (!string.IsNullOrEmpty(page?.Content))
{
var content = page.Content;
var containContent = _contents.FirstOrDefault(c => content.Contains(c));
if (containContent != null)
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
spider.Exit();
}
Spider.AddToCycleRetry(page.Request, spider.Site);
_cookieInjector.Inject(spider);
page.Exception = new DownloadException($"Downloaded content contains: {containContent}.");
}
}
}
/// <summary>
/// 当包含指定内容时触发ADSL拨号
/// </summary>
/// <param name="page">页面数据</param>
/// <param name="spider">爬虫</param>
public override void Handle(ref Page page, ISpider spider)
{
if (!string.IsNullOrEmpty(page?.Content))
{
var content = page.Content;
var containContent = _contents.FirstOrDefault(c => content.Contains(c));
if (containContent != null)
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
Logger.AllLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error);
spider.Exit();
}
page = Spider.AddToCycleRetry(page.Request, spider.Site);
page.Exception = new DownloadException($"Downloaded content contains: {containContent}.");
}
}
}
/// <summary>
/// 订阅 Redis的消息队列, 实现消息队列对爬虫的控制
/// </summary>
/// <param name="spider">爬虫</param>
protected void RegisterControl(ISpider spider)
{
if (RedisConnection.Default == null)
{
return;
}
try
{
RedisConnection.Default.Subscriber.Subscribe($"{spider.Identity}", (c, m) =>
{
switch (m)
{
case "PAUSE":
{
spider.Pause();
break;
}
case "CONTINUE":
{
spider.Continue();
break;
}
case "RUNASYNC":
{
spider.RunAsync();
break;
}
case "EXIT":
{
spider.Exit();
break;
}
}
});
}
catch (Exception e)
{
Logger.LogError($"Register control failed：{e}");
}
}
protected void RegisterControl(ISpider spider)
{
if (RedisConnection.Default != null)
{
try
{
RedisConnection.Default.Subscriber.Subscribe($"{spider.Identity}", (c, m) =>
{
switch (m)
{
case "PAUSE":
{
spider.Pause();
break;
}
case "CONTINUE":
{
spider.Contiune();
break;
}
case "RUNASYNC":
{
spider.RunAsync();
break;
}
case "EXIT":
{
spider.Exit();
break;
}
}
});
}
catch (Exception e)
{
Logger.MyLog(Identity, "Register contol failed.", LogLevel.Error, e);
}
}
}
public override void Handle(ref Page page, ISpider spider)
{
if (RedialLimit != 0)
{
lock (_locker)
{
++RequestedCount;
if (RedialLimit > 0 && RequestedCount == RedialLimit)
{
RequestedCount = 0;
Spider.AddToCycleRetry(page.Request, spider.Site);
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
spider.Exit();
}
}
}
}
}
public override void Handle(ref Page page, ISpider spider)
{
if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(ExceptionMessage) && page.Exception != null)
{
if (string.IsNullOrEmpty(ExceptionMessage))
{
page.Exception = new SpiderException("ExceptionMessage should not be empty/null.");
}
if (page.Exception.Message.Contains(ExceptionMessage))
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
Logger.MyLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error);
spider.Exit();
}
Spider.AddToCycleRetry(page.Request, spider.Site);
page.Exception = new DownloadException("Download failed and redial finished already.");
}
}
}
/// <summary>
/// Redial ADSL and re-obtain cookie when <see cref="Page.Content"/> contains specified contents.
/// </summary>
/// <summary xml:lang="zh-CN">
/// 当页面数据包含指定内容时触发ADSL拨号, 并且重新获取Cookie
/// </summary>
/// <param name="page">页面数据</param>
/// <param name="downloader">下载器</param>
/// <param name="spider">爬虫</param>
public override void Handle(ref Page page, IDownloader downloader, ISpider spider)
{
if (page == null || string.IsNullOrWhiteSpace(page.Content))
{
return;
}
var content = page.Content;
var containContent = _contents.FirstOrDefault(c => content.Contains(c));
if (containContent != null)
{
if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed)
{
spider.Exit();
}
page = spider.Site.AddToCycleRetry(page.Request);
_cookieInjector.Inject(downloader, spider);
page.Exception = new DownloadException($"Downloaded content contains: {containContent}.");
}
}