public void PushAndPoll1()
{
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler();
ISpider spider = new DefaultSpider();
scheduler.Init(spider);
scheduler.Dispose();
Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", null)
{
Site = spider.Site
};
request.PutExtra("1", "2");
scheduler.Push(request);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", result.Url.ToString());
Assert.Equal("2", request.GetExtra("1"));
Request result1 = scheduler.Poll();
Assert.Null(result1);
scheduler.Dispose();
}
public void PushAndPollDepthFirst()
{
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler();
scheduler.DepthFirst = true;
ISpider spider = new DefaultSpider();
scheduler.Init(spider);
scheduler.Dispose();
Request request1 = new Request("http://www.ibm.com/1", null);
Request request2 = new Request("http://www.ibm.com/2", null);
Request request3 = new Request("http://www.ibm.com/3", null);
Request request4 = new Request("http://www.ibm.com/4", null);
scheduler.Push(request1);
scheduler.Push(request2);
scheduler.Push(request3);
scheduler.Push(request4);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/4", result.Url.ToString());
Request result1 = scheduler.Poll();
Assert.Equal("http://www.ibm.com/3", result1.Url.ToString());
scheduler.Dispose();
scheduler.Dispose();
}
public void Load()
{
QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler();
ISpider spider = new DefaultSpider("test", new Site());
scheduler.Init(spider);
scheduler.Push(new Request("http://www.a.com/", null));
scheduler.Push(new Request("http://www.b.com/", null));
scheduler.Push(new Request("http://www.c.com/", null));
scheduler.Push(new Request("http://www.d.com/", null));
Extension.Scheduler.RedisScheduler redisScheduler = GetRedisScheduler();
redisScheduler.Init(spider);
redisScheduler.Dispose();
redisScheduler.Import(scheduler.ToList());
Assert.Equal("http://www.d.com/", redisScheduler.Poll().Url.ToString());
Assert.Equal("http://www.c.com/", redisScheduler.Poll().Url.ToString());
Assert.Equal("http://www.b.com/", redisScheduler.Poll().Url.ToString());
Assert.Equal("http://www.a.com/", redisScheduler.Poll().Url.ToString());
redisScheduler.Dispose();
}
public void PushAndPollBreadthFirst()
{
ISpider spider = new DefaultSpider();
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity);
scheduler.TraverseStrategy = TraverseStrategy.Bfs;
scheduler.Dispose();
Request request1 = new Request("http://www.ibm.com/1", null);
Request request2 = new Request("http://www.ibm.com/2", null);
Request request3 = new Request("http://www.ibm.com/3", null);
Request request4 = new Request("http://www.ibm.com/4", null);
scheduler.Push(request1);
scheduler.Push(request2);
scheduler.Push(request3);
scheduler.Push(request4);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/1", result.Url.ToString());
Request result1 = scheduler.Poll();
Assert.Equal("http://www.ibm.com/2", result1.Url.ToString());
scheduler.Dispose();
scheduler.Dispose();
}
public void Clear()
{
Extension.Scheduler.RedisScheduler scheduler = new Extension.Scheduler.RedisScheduler("localhost", "");
ISpider spider = new DefaultSpider();
scheduler.Init(spider);
scheduler.Clear();
Request request1 = new Request("http://www.ibm.com/1", 1, null);
Request request2 = new Request("http://www.ibm.com/2", 1, null);
Request request3 = new Request("http://www.ibm.com/3", 1, null);
Request request4 = new Request("http://www.ibm.com/4", 1, null);
scheduler.Push(request1);
scheduler.Push(request2);
scheduler.Push(request3);
scheduler.Push(request4);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/4", result.Url.ToString());
scheduler.Clear();
scheduler.Dispose();
}
public void PushAndPoll1()
{
ISpider spider = new DefaultSpider();
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity);
scheduler.Dispose();
Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", null);
request.Properties.Add("1", "2");
scheduler.Push(request);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", result.Url.ToString());
Assert.Equal("2", request.Properties["1"]);
Request result1 = scheduler.Poll();
Assert.Null(result1);
scheduler.Dispose();
}
public void Status()
{
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler();
ISpider spider = new DefaultSpider("test", new Site());
scheduler.Init(spider);
scheduler.Dispose();
scheduler.Push(new Request("http://www.a.com/", null));
scheduler.Push(new Request("http://www.b.com/", null));
scheduler.Push(new Request("http://www.c.com/", null));
scheduler.Push(new Request("http://www.d.com/", null));
Assert.Equal(0, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetLeftRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.IncreaseErrorCounter();
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(0, scheduler.GetSuccessRequestsCount());
scheduler.IncreaseSuccessCounter();
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
scheduler.Poll();
Assert.Equal(3, scheduler.GetLeftRequestsCount());
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.Poll();
Assert.Equal(2, scheduler.GetLeftRequestsCount());
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.Poll();
Assert.Equal(1, scheduler.GetLeftRequestsCount());
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.Poll();
Assert.Equal(0, scheduler.GetLeftRequestsCount());
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.Poll();
scheduler.Poll();
Assert.Equal(0, scheduler.GetLeftRequestsCount());
Assert.Equal(1, scheduler.GetSuccessRequestsCount());
Assert.Equal(1, scheduler.GetErrorRequestsCount());
Assert.Equal(4, scheduler.GetTotalRequestsCount());
scheduler.Dispose();
}
public void Status()
{
ISpider spider = new DefaultSpider("test");
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity);
scheduler.Dispose();
scheduler.Push(new Request("http://www.a.com/", null));
scheduler.Push(new Request("http://www.b.com/", null));
scheduler.Push(new Request("http://www.c.com/", null));
scheduler.Push(new Request("http://www.d.com/", null));
Assert.Equal(0, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.LeftRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.IncreaseErrorCount();
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(0, scheduler.SuccessRequestsCount);
scheduler.IncreaseSuccessCount();
Assert.Equal(1, scheduler.SuccessRequestsCount);
scheduler.Poll();
Assert.Equal(3, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.Poll();
Assert.Equal(2, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.Poll();
Assert.Equal(1, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.Poll();
Assert.Equal(0, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.Poll();
scheduler.Poll();
Assert.Equal(0, scheduler.LeftRequestsCount);
Assert.Equal(1, scheduler.SuccessRequestsCount);
Assert.Equal(1, scheduler.ErrorRequestsCount);
Assert.Equal(4, scheduler.TotalRequestsCount);
scheduler.Dispose();
}
public void Clear()
{
ISpider spider = new DefaultSpider();
Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity);
scheduler.Dispose();
Request request1 = new Request("http://www.ibm.com/1", null)
{
Site = spider.Site
};
Request request2 = new Request("http://www.ibm.com/2", null)
{
Site = spider.Site
};
Request request3 = new Request("http://www.ibm.com/3", null)
{
Site = spider.Site
};
Request request4 = new Request("http://www.ibm.com/4", null)
{
Site = spider.Site
};
scheduler.Push(request1);
scheduler.Push(request2);
scheduler.Push(request3);
scheduler.Push(request4);
Request result = scheduler.Poll();
Assert.Equal("http://www.ibm.com/4", result.Url.ToString());
scheduler.Dispose();
}