Example #1
0
        static void Main(string[] args)
        {
            string url      = @"http://css.cps.qqt-dev.com/account/logon";
            var    procesor = new LoginProcessor();
            Spider spider   = Spider.Create(
                new QueueDuplicateRemovedScheduler(), procesor);

            ICookieInjector cookieInjector = null;

            spider.Downloader = new HttpClientDownloader()
            {
                CookieInjector = cookieInjector, AllowAutoRedirect = false
            };
            spider.Downloader.AddAfterDownloadCompleteHandler(new SetCookieHandler());
            string  loginJson = JsonConvert.SerializeObject(new { Username = "******", Password = "******" });
            Request request   = new Request(url);

            request.ContentType = "application/json";
            request.Content     = loginJson;
            request.Method      = HttpMethod.Post;

            //request.AddHeader("Cookie", cookie);
            spider.AddRequest(request);
            spider.Run();
        }
Example #2
0
        public void Run()
        {
            Spider spider = SpiderBuilder.CreateBuilder()
                            .AddRequest("https://weixin.sogou.com/").Buid();

            spider.AddPageProcessor(new Processor2());

            for (int i = 1; i <= 10; i++)
            {
                spider.AddRequest($"https://weixin.sogou.com/weixin?type=2&ie=utf8&page={i}&query=马云");
            }

            spider.NewRequestSleepInterval = 2000; // 2s
            // spider.EmptySleepTime = 60; // 60s

            spider.ThreadNumber = 5;

            //spider.UseNLog();
            spider.UseRedisScheduler("localhost");

            spider.Run();

            spider.OnStatusChanged += Spider_OnStatusChanged;

            // Console.WriteLine("end main ");
        }
        public void RunAsyncAndStop()
        {
            if (Environment.GetEnvironmentVariable("TRAVIS") == "1")
            {
                return;
            }
            Spider spider = Spider.Create(new TestPageProcessor()).AddPipeline(new TestPipeline());

            spider.ThreadNum = 1;
            spider.SleepTime = 1000;
            for (int i = 0; i < 10000; i++)
            {
                spider.AddRequest(new Request("http://www.baidu.com/" + i)
                {
                    EncodingName = "UTF-8"
                });
            }
            spider.RunAsync();
            Thread.Sleep(5000);
            spider.Pause(() =>
            {
                spider.RunAsync();
            });
            Thread.Sleep(3000);
        }
Example #4
0
 public override SpiderResponse Process(SpiderResponse response)
 {
     if (response.httpResponse.status["StatusCode"] == "302")
     {
         var request = response.request;
         request.Url = response.httpResponse.header["Location"];
         Console.WriteLine("302 to:" + request.Url);
         Spider.AddRequest(request);
         return(null);
     }
     return(response);
 }