public void QueueSchedulerPushPollSynchronized()
{
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
ISpider spider = new DefaultSpider("test", new Site());
Parallel.For(0, 1000, new ParallelOptions() { MaxDegreeOfParallelism = 30 }, i =>
{
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.b.com", 1, null));
scheduler.Push(new Request($"http://www.{i.ToString()}.com", 1, null));
});
Parallel.For(0, 1000, new ParallelOptions() { MaxDegreeOfParallelism = 30 }, i =>
{
scheduler.Poll();
});
long left = scheduler.GetLeftRequestsCount();
long total = scheduler.GetTotalRequestsCount();
Assert.AreEqual(left, 2);
Assert.AreEqual(total, 1002);
}
public void QueueSchedulerPush()
{
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
ISpider spider = new DefaultSpider("test", new Site());
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.a.com", 1, null));
scheduler.Push(new Request("http://www.b.com", 1, null));
long left = scheduler.GetLeftRequestsCount();
long total = scheduler.GetTotalRequestsCount();
Assert.AreEqual(left, 2);
Assert.AreEqual(total, 2);
}
public void SchedulerLoadTest()
{
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
ISpider spider = new DefaultSpider("test", new Site());
scheduler.Push(new Request("http://www.a.com/", 1, null));
scheduler.Push(new Request("http://www.b.com/", 1, null));
scheduler.Push(new Request("http://www.c.com/", 1, null));
scheduler.Push(new Request("http://www.d.com/", 1, null));
RedisScheduler redisScheduler = new RedisScheduler("localhost", "");
redisScheduler.Load(scheduler.ToList());
Assert.AreEqual("http://www.d.com/", redisScheduler.Poll().Url.ToString());
Assert.AreEqual("http://www.c.com/", redisScheduler.Poll().Url.ToString());
Assert.AreEqual("http://www.b.com/", redisScheduler.Poll().Url.ToString());
Assert.AreEqual("http://www.a.com/", redisScheduler.Poll().Url.ToString());
}