public override void Handle(ref Page page, ISpider spider) { if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(Content) && CookieInjector != null && page.Content.Contains(Content)) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { spider.Exit(); } Spider.AddToCycleRetry(page.Request, spider.Site); CookieInjector?.Inject(spider); page.Exception = new DownloadException($"Content downloaded contains string: {Content}."); } }
public override void Handle(ref Page page, ISpider spider) { if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(Content) && page.Content.Contains(Content)) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.MyLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error); spider.Exit(); } page = Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException($"Content downloaded contains string: {Content}."); } }
/// <summary> /// 当页面数据中的异常信息包含指定内容时触发ADSL拨号 /// </summary> /// <param name="page">页面数据</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, ISpider spider) { if (page == null || string.IsNullOrEmpty(page.Content) || string.IsNullOrWhiteSpace(page.Content) || page.Exception == null) { return; } if (page.Exception.Message.Contains(_exceptionMessage)) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.AllLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error); spider.Exit(); } Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException("Download failed and redial finished already."); } }
/// <summary> /// 当页面数据包含指定内容时触发ADSL拨号, 并且重新获取Cookie /// </summary> /// <param name="page">页面数据</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, ISpider spider) { if (!string.IsNullOrEmpty(page?.Content)) { var content = page.Content; var containContent = _contents.FirstOrDefault(c => content.Contains(c)); if (containContent != null) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { spider.Exit(); } Spider.AddToCycleRetry(page.Request, spider.Site); _cookieInjector.Inject(spider); page.Exception = new DownloadException($"Downloaded content contains: {containContent}."); } } }
/// <summary> /// 当包含指定内容时触发ADSL拨号 /// </summary> /// <param name="page">页面数据</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, ISpider spider) { if (!string.IsNullOrEmpty(page?.Content)) { var content = page.Content; var containContent = _contents.FirstOrDefault(c => content.Contains(c)); if (containContent != null) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.AllLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error); spider.Exit(); } page = Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException($"Downloaded content contains: {containContent}."); } } }
/// <summary> /// 订阅 Redis的消息队列, 实现消息队列对爬虫的控制 /// </summary> /// <param name="spider">爬虫</param> protected void RegisterControl(ISpider spider) { if (RedisConnection.Default == null) { return; } try { RedisConnection.Default.Subscriber.Subscribe($"{spider.Identity}", (c, m) => { switch (m) { case "PAUSE": { spider.Pause(); break; } case "CONTINUE": { spider.Continue(); break; } case "RUNASYNC": { spider.RunAsync(); break; } case "EXIT": { spider.Exit(); break; } } }); } catch (Exception e) { Logger.LogError($"Register control failed:{e}"); } }
protected void RegisterControl(ISpider spider) { if (RedisConnection.Default != null) { try { RedisConnection.Default.Subscriber.Subscribe($"{spider.Identity}", (c, m) => { switch (m) { case "PAUSE": { spider.Pause(); break; } case "CONTINUE": { spider.Contiune(); break; } case "RUNASYNC": { spider.RunAsync(); break; } case "EXIT": { spider.Exit(); break; } } }); } catch (Exception e) { Logger.MyLog(Identity, "Register contol failed.", LogLevel.Error, e); } } }
public override void Handle(ref Page page, ISpider spider) { if (RedialLimit != 0) { lock (_locker) { ++RequestedCount; if (RedialLimit > 0 && RequestedCount == RedialLimit) { RequestedCount = 0; Spider.AddToCycleRetry(page.Request, spider.Site); if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { spider.Exit(); } } } } }
public override void Handle(ref Page page, ISpider spider) { if (page != null && !string.IsNullOrEmpty(page.Content) && !string.IsNullOrEmpty(ExceptionMessage) && page.Exception != null) { if (string.IsNullOrEmpty(ExceptionMessage)) { page.Exception = new SpiderException("ExceptionMessage should not be empty/null."); } if (page.Exception.Message.Contains(ExceptionMessage)) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { Logger.MyLog(spider.Identity, "Exit program because redial failed.", LogLevel.Error); spider.Exit(); } Spider.AddToCycleRetry(page.Request, spider.Site); page.Exception = new DownloadException("Download failed and redial finished already."); } } }
/// <summary> /// Redial ADSL and re-obtain cookie when <see cref="Page.Content"/> contains specified contents. /// </summary> /// <summary xml:lang="zh-CN"> /// 当页面数据包含指定内容时触发ADSL拨号, 并且重新获取Cookie /// </summary> /// <param name="page">页面数据</param> /// <param name="downloader">下载器</param> /// <param name="spider">爬虫</param> public override void Handle(ref Page page, IDownloader downloader, ISpider spider) { if (page == null || string.IsNullOrWhiteSpace(page.Content)) { return; } var content = page.Content; var containContent = _contents.FirstOrDefault(c => content.Contains(c)); if (containContent != null) { if (NetworkCenter.Current.Executor.Redial() == RedialResult.Failed) { spider.Exit(); } page = spider.Site.AddToCycleRetry(page.Request); _cookieInjector.Inject(downloader, spider); page.Exception = new DownloadException($"Downloaded content contains: {containContent}."); } }