public void QueueSchedulerPushPollSynchronized()
        {
            QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
            ISpider spider = new DefaultSpider("test", new Site());

            Parallel.For(0, 1000, new ParallelOptions() { MaxDegreeOfParallelism = 30 }, i =>
            {
                scheduler.Push(new Request("http://www.a.com", 1, null));
                scheduler.Push(new Request("http://www.a.com", 1, null));
                scheduler.Push(new Request("http://www.a.com", 1, null));

                scheduler.Push(new Request("http://www.b.com", 1, null));

                scheduler.Push(new Request($"http://www.{i.ToString()}.com", 1, null));
            });

            Parallel.For(0, 1000, new ParallelOptions() { MaxDegreeOfParallelism = 30 }, i =>
            {
                scheduler.Poll();
            });

            long left = scheduler.GetLeftRequestsCount();
            long total = scheduler.GetTotalRequestsCount();

            Assert.AreEqual(left, 2);
            Assert.AreEqual(total, 1002);
        }
        public void QueueSchedulerPush()
        {
            QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
            ISpider spider = new DefaultSpider("test", new Site());
            scheduler.Push(new Request("http://www.a.com", 1, null));
            scheduler.Push(new Request("http://www.a.com", 1, null));
            scheduler.Push(new Request("http://www.a.com", 1, null));

            scheduler.Push(new Request("http://www.b.com", 1, null));
            long left = scheduler.GetLeftRequestsCount();
            long total = scheduler.GetTotalRequestsCount();

            Assert.AreEqual(left, 2);
            Assert.AreEqual(total, 2);
        }
Exemplo n.º 3
0
        public void SchedulerLoadTest()
        {
            QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
            ISpider spider = new DefaultSpider("test", new Site());
            scheduler.Push(new Request("http://www.a.com/", 1, null));
            scheduler.Push(new Request("http://www.b.com/", 1, null));
            scheduler.Push(new Request("http://www.c.com/", 1, null));
            scheduler.Push(new Request("http://www.d.com/", 1, null));

            RedisScheduler redisScheduler = new RedisScheduler("localhost", "");
            redisScheduler.Load(scheduler.ToList());

            Assert.AreEqual("http://www.d.com/", redisScheduler.Poll().Url.ToString());
            Assert.AreEqual("http://www.c.com/", redisScheduler.Poll().Url.ToString());
            Assert.AreEqual("http://www.b.com/", redisScheduler.Poll().Url.ToString());
            Assert.AreEqual("http://www.a.com/", redisScheduler.Poll().Url.ToString());
        }