static void Main(string[] args) { string url = @"http://css.cps.qqt-dev.com/account/logon"; var procesor = new LoginProcessor(); Spider spider = Spider.Create( new QueueDuplicateRemovedScheduler(), procesor); ICookieInjector cookieInjector = null; spider.Downloader = new HttpClientDownloader() { CookieInjector = cookieInjector, AllowAutoRedirect = false }; spider.Downloader.AddAfterDownloadCompleteHandler(new SetCookieHandler()); string loginJson = JsonConvert.SerializeObject(new { Username = "******", Password = "******" }); Request request = new Request(url); request.ContentType = "application/json"; request.Content = loginJson; request.Method = HttpMethod.Post; //request.AddHeader("Cookie", cookie); spider.AddRequest(request); spider.Run(); }
public void Run() { Spider spider = SpiderBuilder.CreateBuilder() .AddRequest("https://weixin.sogou.com/").Buid(); spider.AddPageProcessor(new Processor2()); for (int i = 1; i <= 10; i++) { spider.AddRequest($"https://weixin.sogou.com/weixin?type=2&ie=utf8&page={i}&query=马云"); } spider.NewRequestSleepInterval = 2000; // 2s // spider.EmptySleepTime = 60; // 60s spider.ThreadNumber = 5; //spider.UseNLog(); spider.UseRedisScheduler("localhost"); spider.Run(); spider.OnStatusChanged += Spider_OnStatusChanged; // Console.WriteLine("end main "); }
public void RunAsyncAndStop() { if (Environment.GetEnvironmentVariable("TRAVIS") == "1") { return; } Spider spider = Spider.Create(new TestPageProcessor()).AddPipeline(new TestPipeline()); spider.ThreadNum = 1; spider.SleepTime = 1000; for (int i = 0; i < 10000; i++) { spider.AddRequest(new Request("http://www.baidu.com/" + i) { EncodingName = "UTF-8" }); } spider.RunAsync(); Thread.Sleep(5000); spider.Pause(() => { spider.RunAsync(); }); Thread.Sleep(3000); }
public override SpiderResponse Process(SpiderResponse response) { if (response.httpResponse.status["StatusCode"] == "302") { var request = response.request; request.Url = response.httpResponse.header["Location"]; Console.WriteLine("302 to:" + request.Url); Spider.AddRequest(request); return(null); } return(response); }