public void RetryDownloadTimes()
{
var spider = SpiderFactory.Create <Spider>();
spider.NewGuidId();
spider.Name = "RetryDownloadTimes";
spider.RetryDownloadTimes = 5;
spider.EmptySleepTime = 15;
spider.DownloaderSettings.Type = DownloaderType.Exception;
var scheduler = new QueueDistinctBfsScheduler();
spider.Scheduler = scheduler;
spider.AddRequests("http://www.RetryDownloadTimes.com");
spider.Run();
var statisticsStore = SpiderFactory.GetRequiredService <IStatisticsStore>();
var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result;
Assert.Equal(1, s.Total);
Assert.Equal(1, s.Failed);
Assert.Equal(0, s.Success);
var dss = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result;
var ds = dss[0];
Assert.Equal(6, ds.Failed);
Assert.Equal(0, ds.Success);
}
public void RunThenPauseThenContinueThenExit()
{
var url = "http://www.RunThenPauseThenContinueThenExit.com/";
var spider = SpiderFactory.Create <Spider>();
spider.NewGuidId();
spider.Name = "RunAsyncAndStop";
spider.EmptySleepTime = 15;
spider.DownloaderSettings.Type = DownloaderType.Empty;
for (int i = 0; i < 10000; i++)
{
spider.AddRequests(new Request(url + i));
}
spider.RunAsync();
Thread.Sleep(2000);
spider.Pause();
Thread.Sleep(2000);
spider.Continue();
Thread.Sleep(2000);
spider.Exit().WaitForExit();
Assert.Equal(Status.Exited, spider.Status);
}
public void RetryWhenResultIsEmpty()
{
//TODO:
var spider = SpiderFactory.Create <Spider>();
spider.Id = Guid.NewGuid().ToString("N");
spider.Name = "RetryWhenResultIsEmpty";
spider.Speed = 1;
spider.Depth = 3;
spider.EmptySleepTime = 2;
spider.RetryDownloadTimes = 5;
spider.RetryWhenResultIsEmpty = false;
spider.DownloaderOptions.Type = DownloaderType.HttpClient;
spider.Scheduler = new QueueDistinctBfsScheduler();
spider.AddRequests("http://www.devfans.com/home/testempty");
spider.RunAsync().Wait(); // 启动
var statisticsStore = SpiderFactory.GetStatisticsStore();
var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result;
Assert.Equal(6, s.Total);
Assert.Equal(6, s.Failed);
Assert.Equal(0, s.Success);
var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0];
Assert.Equal(6, ds.Failed);
Assert.Equal(0, ds.Success);
}
public void RetryDownloadTimes()
{
// 配置使用 TestDownloader, 一直抛错，检测到达指定尝试次数是否不再重试。
var spider = SpiderFactory.Create <Spider>();
spider.Id = Guid.NewGuid().ToString("N");
spider.Name = "RetryDownloadTimes";
spider.Speed = 1;
spider.Depth = 3;
spider.EmptySleepTime = 30;
spider.RetryDownloadTimes = 5;
spider.DownloaderOptions.Type = DownloaderType.Exception;
spider.Scheduler = new QueueDistinctBfsScheduler();
spider.AddRequests("http://www.baidu.com");
spider.RunAsync().Wait(); // 启动
var statisticsStore = SpiderFactory.GetStatisticsStore();
var s = statisticsStore.GetSpiderStatisticsAsync(spider.Id).Result;
Assert.Equal(6, s.Total);
Assert.Equal(6, s.Failed);
Assert.Equal(0, s.Success);
var ds = statisticsStore.GetDownloadStatisticsListAsync(1, 10).Result[0];
Assert.Equal(6, ds.Failed);
Assert.Equal(0, ds.Success);
}
public void RunThenPauseThenContinueThenExit()
{
var url = Environment.GetEnvironmentVariable("TRAVIS") == "1"
? "https://www.google.com/"
: "http://www.baidu.com/";
var spider = SpiderFactory.Create <Spider>();
spider.Id = Guid.NewGuid().ToString("N"); // 设置任务标识
spider.Name = "RunAsyncAndStop"; // 设置任务名称
spider.Speed = 1; // 设置采集速度, 表示每秒下载多少个请求, 大于 1 时越大速度越快, 小于 1 时越小越慢, 不能为0.
spider.Depth = 3; // 设置采集深度
spider.DownloaderOptions.Type = DownloaderType.HttpClient; // 使用普通下载器, 无关 Cookie, 干净的 HttpClient
for (int i = 0; i < 10000; i++)
{
spider.AddRequests(new Request(url + i)
{
Encoding = "UTF-8"
});
}
spider.RunAsync();
Thread.Sleep(4000);
spider.Pause();
Thread.Sleep(4000);
spider.Continue();
Thread.Sleep(4000);
spider.Exit();
for (int i = 0; i < 50; ++i)
{
if (spider.Status == Status.Exited)
{
break;
}
else
{
Thread.Sleep(1000);
}
}
Assert.Equal(Status.Exited, spider.Status);
}
public void MmfCloseSignal()
{
var url = "http://www.MmfCloseSignal.com/";
var spider = SpiderFactory.Create <Spider>();
spider.MmfSignal = true;
spider.NewGuidId();
spider.Name = "MmfCloseSignal";
spider.DownloaderSettings.Type = DownloaderType.Empty;
for (int i = 0; i < 10000; i++)
{
spider.AddRequests(new Request(url + i));
}
spider.RunAsync();
Thread.Sleep(2000);
spider.ExitBySignal().WaitForExit(15000);
Assert.Equal(Status.Exited, spider.Status);
}