public void RetryWhenResultIsEmpty()
{
//TODO:
var spider = SpiderFactory.Create <Spider>();
spider.Id = Guid.NewGuid().ToString("N");
spider.Name = "RetryWhenResultIsEmpty";
spider.Speed = 1;
spider.Depth = 3;
spider.EmptySleepTime = 2;
spider.RetryDownloadTimes = 5;
spider.RetryWhenResultIsEmpty = false;
spider.DownloaderOptions.Type = DownloaderType.HttpClient;
spider.Scheduler = new QueueDistinctBfsScheduler();
spider.AddRequests("http://www.devfans.com/home/testempty");
spider.RunAsync().Wait(); // 启动
var statisticsStore = SpiderFactory.GetStatisticsStore();
var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result;
Assert.Equal(6, s.Total);
Assert.Equal(6, s.Failed);
Assert.Equal(0, s.Success);
var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0];
Assert.Equal(6, ds.Failed);
Assert.Equal(0, ds.Success);
}
public void RetryDownloadTimes()
{
// 配置使用 TestDownloader, 一直抛错，检测到达指定尝试次数是否不再重试。
var spider = SpiderFactory.Create <Spider>();
spider.Id = Guid.NewGuid().ToString("N");
spider.Name = "RetryDownloadTimes";
spider.Speed = 1;
spider.Depth = 3;
spider.EmptySleepTime = 30;
spider.RetryDownloadTimes = 5;
spider.DownloaderOptions.Type = DownloaderType.Exception;
spider.Scheduler = new QueueDistinctBfsScheduler();
spider.AddRequests("http://www.baidu.com");
spider.RunAsync().Wait(); // 启动
var statisticsStore = SpiderFactory.GetStatisticsStore();
var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result;
Assert.Equal(6, s.Total);
Assert.Equal(6, s.Failed);
Assert.Equal(0, s.Success);
var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0];
Assert.Equal(6, ds.Failed);
Assert.Equal(0, ds.Success);
}
