public void PushAndPoll1() { Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(); ISpider spider = new DefaultSpider(); scheduler.Init(spider); scheduler.Dispose(); Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", null) { Site = spider.Site }; request.PutExtra("1", "2"); scheduler.Push(request); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", result.Url.ToString()); Assert.Equal("2", request.GetExtra("1")); Request result1 = scheduler.Poll(); Assert.Null(result1); scheduler.Dispose(); }
public void PushAndPollDepthFirst() { Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(); scheduler.DepthFirst = true; ISpider spider = new DefaultSpider(); scheduler.Init(spider); scheduler.Dispose(); Request request1 = new Request("http://www.ibm.com/1", null); Request request2 = new Request("http://www.ibm.com/2", null); Request request3 = new Request("http://www.ibm.com/3", null); Request request4 = new Request("http://www.ibm.com/4", null); scheduler.Push(request1); scheduler.Push(request2); scheduler.Push(request3); scheduler.Push(request4); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/4", result.Url.ToString()); Request result1 = scheduler.Poll(); Assert.Equal("http://www.ibm.com/3", result1.Url.ToString()); scheduler.Dispose(); scheduler.Dispose(); }
public void Load() { QueueDuplicateRemovedScheduler scheduler = new QueueDuplicateRemovedScheduler(); ISpider spider = new DefaultSpider("test", new Site()); scheduler.Init(spider); scheduler.Push(new Request("http://www.a.com/", null)); scheduler.Push(new Request("http://www.b.com/", null)); scheduler.Push(new Request("http://www.c.com/", null)); scheduler.Push(new Request("http://www.d.com/", null)); Extension.Scheduler.RedisScheduler redisScheduler = GetRedisScheduler(); redisScheduler.Init(spider); redisScheduler.Dispose(); redisScheduler.Import(scheduler.ToList()); Assert.Equal("http://www.d.com/", redisScheduler.Poll().Url.ToString()); Assert.Equal("http://www.c.com/", redisScheduler.Poll().Url.ToString()); Assert.Equal("http://www.b.com/", redisScheduler.Poll().Url.ToString()); Assert.Equal("http://www.a.com/", redisScheduler.Poll().Url.ToString()); redisScheduler.Dispose(); }
public void PushAndPollBreadthFirst() { ISpider spider = new DefaultSpider(); Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity); scheduler.TraverseStrategy = TraverseStrategy.Bfs; scheduler.Dispose(); Request request1 = new Request("http://www.ibm.com/1", null); Request request2 = new Request("http://www.ibm.com/2", null); Request request3 = new Request("http://www.ibm.com/3", null); Request request4 = new Request("http://www.ibm.com/4", null); scheduler.Push(request1); scheduler.Push(request2); scheduler.Push(request3); scheduler.Push(request4); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/1", result.Url.ToString()); Request result1 = scheduler.Poll(); Assert.Equal("http://www.ibm.com/2", result1.Url.ToString()); scheduler.Dispose(); scheduler.Dispose(); }
public void Clear() { Extension.Scheduler.RedisScheduler scheduler = new Extension.Scheduler.RedisScheduler("localhost", ""); ISpider spider = new DefaultSpider(); scheduler.Init(spider); scheduler.Clear(); Request request1 = new Request("http://www.ibm.com/1", 1, null); Request request2 = new Request("http://www.ibm.com/2", 1, null); Request request3 = new Request("http://www.ibm.com/3", 1, null); Request request4 = new Request("http://www.ibm.com/4", 1, null); scheduler.Push(request1); scheduler.Push(request2); scheduler.Push(request3); scheduler.Push(request4); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/4", result.Url.ToString()); scheduler.Clear(); scheduler.Dispose(); }
public void PushAndPoll1() { ISpider spider = new DefaultSpider(); Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity); scheduler.Dispose(); Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", null); request.Properties.Add("1", "2"); scheduler.Push(request); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/", result.Url.ToString()); Assert.Equal("2", request.Properties["1"]); Request result1 = scheduler.Poll(); Assert.Null(result1); scheduler.Dispose(); }
public void Status() { Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(); ISpider spider = new DefaultSpider("test", new Site()); scheduler.Init(spider); scheduler.Dispose(); scheduler.Push(new Request("http://www.a.com/", null)); scheduler.Push(new Request("http://www.b.com/", null)); scheduler.Push(new Request("http://www.c.com/", null)); scheduler.Push(new Request("http://www.d.com/", null)); Assert.Equal(0, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetLeftRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.IncreaseErrorCounter(); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(0, scheduler.GetSuccessRequestsCount()); scheduler.IncreaseSuccessCounter(); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); scheduler.Poll(); Assert.Equal(3, scheduler.GetLeftRequestsCount()); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.Poll(); Assert.Equal(2, scheduler.GetLeftRequestsCount()); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.Poll(); Assert.Equal(1, scheduler.GetLeftRequestsCount()); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.Poll(); Assert.Equal(0, scheduler.GetLeftRequestsCount()); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.Poll(); scheduler.Poll(); Assert.Equal(0, scheduler.GetLeftRequestsCount()); Assert.Equal(1, scheduler.GetSuccessRequestsCount()); Assert.Equal(1, scheduler.GetErrorRequestsCount()); Assert.Equal(4, scheduler.GetTotalRequestsCount()); scheduler.Dispose(); }
public void Status() { ISpider spider = new DefaultSpider("test"); Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity); scheduler.Dispose(); scheduler.Push(new Request("http://www.a.com/", null)); scheduler.Push(new Request("http://www.b.com/", null)); scheduler.Push(new Request("http://www.c.com/", null)); scheduler.Push(new Request("http://www.d.com/", null)); Assert.Equal(0, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.LeftRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.IncreaseErrorCount(); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(0, scheduler.SuccessRequestsCount); scheduler.IncreaseSuccessCount(); Assert.Equal(1, scheduler.SuccessRequestsCount); scheduler.Poll(); Assert.Equal(3, scheduler.LeftRequestsCount); Assert.Equal(1, scheduler.SuccessRequestsCount); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.Poll(); Assert.Equal(2, scheduler.LeftRequestsCount); Assert.Equal(1, scheduler.SuccessRequestsCount); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.Poll(); Assert.Equal(1, scheduler.LeftRequestsCount); Assert.Equal(1, scheduler.SuccessRequestsCount); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.Poll(); Assert.Equal(0, scheduler.LeftRequestsCount); Assert.Equal(1, scheduler.SuccessRequestsCount); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.Poll(); scheduler.Poll(); Assert.Equal(0, scheduler.LeftRequestsCount); Assert.Equal(1, scheduler.SuccessRequestsCount); Assert.Equal(1, scheduler.ErrorRequestsCount); Assert.Equal(4, scheduler.TotalRequestsCount); scheduler.Dispose(); }
public void Clear() { ISpider spider = new DefaultSpider(); Extension.Scheduler.RedisScheduler scheduler = GetRedisScheduler(spider.Identity); scheduler.Dispose(); Request request1 = new Request("http://www.ibm.com/1", null) { Site = spider.Site }; Request request2 = new Request("http://www.ibm.com/2", null) { Site = spider.Site }; Request request3 = new Request("http://www.ibm.com/3", null) { Site = spider.Site }; Request request4 = new Request("http://www.ibm.com/4", null) { Site = spider.Site }; scheduler.Push(request1); scheduler.Push(request2); scheduler.Push(request3); scheduler.Push(request4); Request result = scheduler.Poll(); Assert.Equal("http://www.ibm.com/4", result.Url.ToString()); scheduler.Dispose(); }