static void Main(string[] args)
{
string url = @"http://css.cps.qqt-dev.com/account/logon";
var procesor = new LoginProcessor();
Spider spider = Spider.Create(
new QueueDuplicateRemovedScheduler(), procesor);
ICookieInjector cookieInjector = null;
spider.Downloader = new HttpClientDownloader()
{
CookieInjector = cookieInjector, AllowAutoRedirect = false
};
spider.Downloader.AddAfterDownloadCompleteHandler(new SetCookieHandler());
string loginJson = JsonConvert.SerializeObject(new { Username = "******", Password = "******" });
Request request = new Request(url);
request.ContentType = "application/json";
request.Content = loginJson;
request.Method = HttpMethod.Post;
//request.AddHeader("Cookie", cookie);
spider.AddRequest(request);
spider.Run();
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="cookieInjector">Cookie注入器</param>
/// <param name="contents">包含的内容</param>
public RedialAndUpdateCookiesWhenContainsHandler(ICookieInjector cookieInjector, params string[] contents)
{
_cookieInjector = cookieInjector ?? throw new SpiderException("cookieInjector should not be null.");
if (contents == null || contents.Length == 0)
{
throw new SpiderException("contents should not be null or empty.");
}
_contents = contents;
}
/// <summary>
/// 构造方法
/// </summary>
/// <param name="interval">间隔时间</param>
/// <param name="injector">Cookie注入器</param>
public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
{
if (interval <= 0)
{
throw new SpiderException("dueTime should be large than 0.");
}
_cookieInjector = injector ?? throw new SpiderException("CookieInjector should not be null.");
_next = DateTime.Now.AddSeconds(_interval);
_interval = interval;
}
public TimingUpdateCookieHandler(int dueTime, ICookieInjector injector)
{
if (dueTime <= 0)
{
throw new SpiderException("dueTime should be large than 0.");
}
_cookieInjector = injector ?? throw new SpiderException("CookieInjector should not be null.");
_nextTime = DateTime.Now.AddSeconds(_dueTime);
_dueTime = dueTime;
}
/// <summary>
/// Construct a <see cref="TimingUpdateCookieHandler"/> instance.
/// </summary>
/// <summary xml:lang="zh-CN">
/// 构造方法
/// </summary>
/// <param name="interval">间隔时间 interval time in second</param>
/// <param name="injector">Cookie注入器 <see cref="ICookieInjector"/></param>
/// <exception cref="ArgumentException">dueTime should be large than 0.</exception>
public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
{
if (interval <= 0)
{
throw new ArgumentException("interval should be large than 0.");
}
_cookieInjector = injector ?? throw new DownloaderException($"{nameof(injector)} should not be null.");
_next = DateTime.Now.AddSeconds(_interval);
_interval = interval;
}
/// <summary>
/// Construct a <see cref="TimingUpdateCookieHandler"/> instance.
/// </summary>
/// <summary xml:lang="zh-CN">
/// 构造方法
/// </summary>
/// <param name="interval">间隔时间 interval time in second</param>
/// <param name="injector">Cookie注入器 <see cref="ICookieInjector"/></param>
/// <exception cref="ArgumentException">dueTime should be large than 0.</exception>
public TimingUpdateCookieHandler(int interval, ICookieInjector injector)
{
if (interval <= 0)
{
throw new ArgumentException("interval should be large than 0.");
}
_cookieInjector = injector;
if (injector == null)
{
throw new ArgumentNullException("CookieInjector should not be null.");
}
_next = DateTime.Now.AddSeconds(_interval);
_interval = interval;
}
public UpdateCookieTimerHandler(int dueTime, ICookieInjector injector)
{
_dueTime = dueTime;
_cookieInjector = injector;
_nextTime = DateTime.Now.AddSeconds(_dueTime);
}
public RedialAndUpdateCookieWhenContainsContentHandler(ICookieInjector cookieInjector, string content)
{
_cookieInjector = cookieInjector;
_content = content;
}
private CookieContainer GenerateNewCookieContainer(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, ICookieInjector cookieInjector = null)
{
var key = cookieInjector == null ? $"{cookieContainer.GetHashCode()}" : $"{cookieContainer.GetHashCode()}_{cookieInjector?.GetHashCode()}";
if (!_initedCookieContainers.ContainsKey(key))
{
cookieInjector?.Inject(downloader, spider);
// 此处完成COPY一个新的Container的原因是, 某此网站会在COOKIE中设置值, 上下访问有承向启下的关系, 所以必须独立的CookieContainer来管理
var newCookieContainer = CopyCookieContainer(cookieContainer);
_initedCookieContainers.Add(key, newCookieContainer);
}
return(_initedCookieContainers[key]);
}
public HttpClientElement GetHttpClient(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, int?hashCode = null, ICookieInjector cookieInjector = null)
{
if (cookieContainer == null)
{
throw new SpiderException($"{nameof(cookieContainer)} should not be null.");
}
if (downloader == null)
{
throw new SpiderException($"{nameof(downloader)} should not be null.");
}
var newCookieContainer = GenerateNewCookieContainer(spider, downloader, cookieContainer, cookieInjector);
if (hashCode == null)
{
return(_defaultHttpClientItem ?? (_defaultHttpClientItem = CreateDefaultHttpClient(newCookieContainer)));
}
_getHttpClientCount++;
if (_getHttpClientCount % 100 == 0)
{
ClearHttpClient();
}
if (_pool.ContainsKey(hashCode.Value))
{
_pool[hashCode.Value].LastUsedTime = DateTime.Now;
return(_pool[hashCode.Value]);
}
else
{
var item = CreateDefaultHttpClient(newCookieContainer);
_pool.TryAdd(hashCode.Value, item);
return(item);
}
}
public UpdateCookieTimerHandler(int dueTime, ICookieInjector injector)
{
DueTime = dueTime;
CookieInjector = injector;
NextTime = DateTime.Now.AddSeconds(DueTime);
}
public TimerUpdateCookieHandler(int interval, ICookieInjector injector)
{
Interval = interval;
CookieInjector = injector;
Next = DateTime.Now.AddSeconds(Interval);
}
public HttpClientElement GetHttpClient(ISpider spider, IDownloader downloader, CookieContainer cookieContainer, IWebProxy proxy, ICookieInjector cookieInjector = null)
{
var httpClientItem = GetHttpClient(spider, downloader, cookieContainer, proxy.ToString(), cookieInjector);
if (httpClientItem.Handler.Proxy == null)
{
httpClientItem.Handler.Proxy = proxy;
}
return(httpClientItem);
}
