static void Main(string[] args)
{
string url = @"http://css.cps.qqt-dev.com/account/logon";
var procesor = new LoginProcessor();
Spider spider = Spider.Create(
new QueueDuplicateRemovedScheduler(), procesor);
ICookieInjector cookieInjector = null;
spider.Downloader = new HttpClientDownloader()
{
CookieInjector = cookieInjector, AllowAutoRedirect = false
};
spider.Downloader.AddAfterDownloadCompleteHandler(new SetCookieHandler());
string loginJson = JsonConvert.SerializeObject(new { Username = "******", Password = "******" });
Request request = new Request(url);
request.ContentType = "application/json";
request.Content = loginJson;
request.Method = HttpMethod.Post;
//request.AddHeader("Cookie", cookie);
spider.AddRequest(request);
spider.Run();
}
public void Run()
{
Spider spider = SpiderBuilder.CreateBuilder()
.AddRequest("https://weixin.sogou.com/").Buid();
spider.AddPageProcessor(new Processor2());
for (int i = 1; i <= 10; i++)
{
spider.AddRequest($"https://weixin.sogou.com/weixin?type=2&ie=utf8&page={i}&query=马云");
}
spider.NewRequestSleepInterval = 2000; // 2s
// spider.EmptySleepTime = 60; // 60s
spider.ThreadNumber = 5;
//spider.UseNLog();
spider.UseRedisScheduler("localhost");
spider.Run();
spider.OnStatusChanged += Spider_OnStatusChanged;
// Console.WriteLine("end main ");
}
public void RunAsyncAndStop()
{
if (Environment.GetEnvironmentVariable("TRAVIS") == "1")
{
return;
}
Spider spider = Spider.Create(new TestPageProcessor()).AddPipeline(new TestPipeline());
spider.ThreadNum = 1;
spider.SleepTime = 1000;
for (int i = 0; i < 10000; i++)
{
spider.AddRequest(new Request("http://www.baidu.com/" + i)
{
EncodingName = "UTF-8"
});
}
spider.RunAsync();
Thread.Sleep(5000);
spider.Pause(() =>
{
spider.RunAsync();
});
Thread.Sleep(3000);
}
public override SpiderResponse Process(SpiderResponse response)
{
if (response.httpResponse.status["StatusCode"] == "302")
{
var request = response.request;
request.Url = response.httpResponse.header["Location"];
Console.WriteLine("302 to:" + request.Url);
Spider.AddRequest(request);
return(null);
}
return(response);
}