static void Main(string[] args)
        {
            string url      = @"http://css.cps.qqt-dev.com/account/logon";
            var    procesor = new LoginProcessor();
            Spider spider   = Spider.Create(
                new QueueDuplicateRemovedScheduler(), procesor);

            ICookieInjector cookieInjector = null;

            spider.Downloader = new HttpClientDownloader()
            {
                CookieInjector = cookieInjector, AllowAutoRedirect = false
            };
            spider.Downloader.AddAfterDownloadCompleteHandler(new SetCookieHandler());
            string  loginJson = JsonConvert.SerializeObject(new { Username = "******", Password = "******" });
            Request request   = new Request(url);

            request.ContentType = "application/json";
            request.Content     = loginJson;
            request.Method      = HttpMethod.Post;

            //request.AddHeader("Cookie", cookie);
            spider.AddRequest(request);
            spider.Run();
        }
Exemple #2
0
 /// <summary>
 /// 构造方法
 /// </summary>
 /// <param name="cookieInjector">Cookie注入器</param>
 /// <param name="contents">包含的内容</param>
 public RedialAndUpdateCookiesWhenContainsHandler(ICookieInjector cookieInjector, params string[] contents)
 {
     _cookieInjector = cookieInjector ?? throw new SpiderException("cookieInjector should not be null.");
     if (contents == null || contents.Length == 0)
     {
         throw new SpiderException("contents should not be null or empty.");
     }
     _contents = contents;
 }
Exemple #3
0
 /// <summary>
 /// 构造方法
 /// </summary>
 /// <param name="interval">间隔时间</param>
 /// <param name="injector">Cookie注入器</param>
 public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
 {
     if (interval <= 0)
     {
         throw new SpiderException("dueTime should be large than 0.");
     }
     _cookieInjector = injector ?? throw new SpiderException("CookieInjector should not be null.");
     _next           = DateTime.Now.AddSeconds(_interval);
     _interval       = interval;
 }
 public TimingUpdateCookieHandler(int dueTime, ICookieInjector injector)
 {
     if (dueTime <= 0)
     {
         throw new SpiderException("dueTime should be large than 0.");
     }
     _cookieInjector = injector ?? throw new SpiderException("CookieInjector should not be null.");
     _nextTime       = DateTime.Now.AddSeconds(_dueTime);
     _dueTime        = dueTime;
 }
        /// <summary>
        /// Construct a <see cref="TimingUpdateCookieHandler"/> instance.
        /// </summary>
        /// <summary xml:lang="zh-CN">
        /// 构造方法
        /// </summary>
        /// <param name="interval">间隔时间 interval time in second</param>
        /// <param name="injector">Cookie注入器 <see cref="ICookieInjector"/></param>
        /// <exception cref="ArgumentException">dueTime should be large than 0.</exception>
        public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
        {
            if (interval <= 0)
            {
                throw new ArgumentException("interval should be large than 0.");
            }

            _cookieInjector = injector ?? throw new DownloaderException($"{nameof(injector)} should not be null.");
            _next           = DateTime.Now.AddSeconds(_interval);
            _interval       = interval;
        }
Exemple #6
0
        /// <summary>
        /// Construct a <see cref="TimingUpdateCookieHandler"/> instance.
        /// </summary>
        /// <summary xml:lang="zh-CN">
        /// 构造方法
        /// </summary>
        /// <param name="interval">间隔时间 interval time in second</param>
        /// <param name="injector">Cookie注入器 <see cref="ICookieInjector"/></param>
        /// <exception cref="ArgumentException">dueTime should be large than 0.</exception>
        public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
        {
            if (interval <= 0)
            {
                throw new ArgumentException("interval should be large than 0.");
            }

            _cookieInjector = injector;
            if (injector == null)
            {
                throw new ArgumentNullException("CookieInjector should not be null.");
            }
            _next     = DateTime.Now.AddSeconds(_interval);
            _interval = interval;
        }
Exemple #7
0
 public UpdateCookieTimerHandler(int dueTime, ICookieInjector injector)
 {
     _dueTime        = dueTime;
     _cookieInjector = injector;
     _nextTime       = DateTime.Now.AddSeconds(_dueTime);
 }
Exemple #8
0
 public RedialAndUpdateCookieWhenContainsContentHandler(ICookieInjector cookieInjector, string content)
 {
     _cookieInjector = cookieInjector;
     _content        = content;
 }
        private CookieContainer GenerateNewCookieContainer(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, ICookieInjector cookieInjector = null)
        {
            var key = cookieInjector == null ? $"{cookieContainer.GetHashCode()}" : $"{cookieContainer.GetHashCode()}_{cookieInjector?.GetHashCode()}";

            if (!_initedCookieContainers.ContainsKey(key))
            {
                cookieInjector?.Inject(downloader, spider);
                // 此处完成COPY一个新的Container的原因是, 某此网站会在COOKIE中设置值, 上下访问有承向启下的关系, 所以必须独立的CookieContainer来管理
                var newCookieContainer = CopyCookieContainer(cookieContainer);
                _initedCookieContainers.Add(key, newCookieContainer);
            }
            return(_initedCookieContainers[key]);
        }
        public HttpClientElement GetHttpClient(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, int?hashCode = null, ICookieInjector cookieInjector = null)
        {
            if (cookieContainer == null)
            {
                throw new SpiderException($"{nameof(cookieContainer)} should not be null.");
            }
            if (downloader == null)
            {
                throw new SpiderException($"{nameof(downloader)} should not be null.");
            }
            var newCookieContainer = GenerateNewCookieContainer(spider, downloader, cookieContainer, cookieInjector);

            if (hashCode == null)
            {
                return(_defaultHttpClientItem ?? (_defaultHttpClientItem = CreateDefaultHttpClient(newCookieContainer)));
            }

            _getHttpClientCount++;

            if (_getHttpClientCount % 100 == 0)
            {
                ClearHttpClient();
            }

            if (_pool.ContainsKey(hashCode.Value))
            {
                _pool[hashCode.Value].LastUsedTime = DateTime.Now;
                return(_pool[hashCode.Value]);
            }
            else
            {
                var item = CreateDefaultHttpClient(newCookieContainer);
                _pool.TryAdd(hashCode.Value, item);
                return(item);
            }
        }
Exemple #11
0
 public UpdateCookieTimerHandler(int dueTime, ICookieInjector injector)
 {
     DueTime        = dueTime;
     CookieInjector = injector;
     NextTime       = DateTime.Now.AddSeconds(DueTime);
 }
Exemple #12
0
 public TimerUpdateCookieHandler(int interval, ICookieInjector injector)
 {
     Interval       = interval;
     CookieInjector = injector;
     Next           = DateTime.Now.AddSeconds(Interval);
 }
        public HttpClientElement GetHttpClient(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, IWebProxy proxy, ICookieInjector cookieInjector = null)
        {
            var httpClientItem = GetHttpClient(spider, downloader, cookieContainer, proxy.ToString(), cookieInjector);

            if (httpClientItem.Handler.Proxy == null)
            {
                httpClientItem.Handler.Proxy = proxy;
            }
            return(httpClientItem);
        }