public void RetryWhenResultIsEmpty() { //TODO: var spider = SpiderFactory.Create <Spider>(); spider.Id = Guid.NewGuid().ToString("N"); spider.Name = "RetryWhenResultIsEmpty"; spider.Speed = 1; spider.Depth = 3; spider.EmptySleepTime = 2; spider.RetryDownloadTimes = 5; spider.RetryWhenResultIsEmpty = false; spider.DownloaderOptions.Type = DownloaderType.HttpClient; spider.Scheduler = new QueueDistinctBfsScheduler(); spider.AddRequests("http://www.devfans.com/home/testempty"); spider.RunAsync().Wait(); // 启动 var statisticsStore = SpiderFactory.GetStatisticsStore(); var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result; Assert.Equal(6, s.Total); Assert.Equal(6, s.Failed); Assert.Equal(0, s.Success); var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0]; Assert.Equal(6, ds.Failed); Assert.Equal(0, ds.Success); }
public void RetryDownloadTimes() { // 配置使用 TestDownloader, 一直抛错,检测到达指定尝试次数是否不再重试。 var spider = SpiderFactory.Create <Spider>(); spider.Id = Guid.NewGuid().ToString("N"); spider.Name = "RetryDownloadTimes"; spider.Speed = 1; spider.Depth = 3; spider.EmptySleepTime = 30; spider.RetryDownloadTimes = 5; spider.DownloaderOptions.Type = DownloaderType.Exception; spider.Scheduler = new QueueDistinctBfsScheduler(); spider.AddRequests("http://www.baidu.com"); spider.RunAsync().Wait(); // 启动 var statisticsStore = SpiderFactory.GetStatisticsStore(); var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result; Assert.Equal(6, s.Total); Assert.Equal(6, s.Failed); Assert.Equal(0, s.Success); var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0]; Assert.Equal(6, ds.Failed); Assert.Equal(0, ds.Success); }