public void RetryWhenContainsIllegalStringHandler()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsContentHandler("aabbcccdefg下载人数100"));
var request1 = new Request("http://www.163.com/", null);
Page page = downloader1.Download(request1, spider).Result;
Assert.Single(page.TargetRequests);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsContentHandler("网易倒闭啦"));
page = downloader1.Download(request1, spider).Result;
Assert.Empty(page.TargetRequests);
}
public void RetryWhenContainsIllegalStringHandler()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsHandler
{
Content = "aabbcccdefg下载人数100"
});
var request1 = new Request("http://www.163.com/", null);
Page page = downloader1.Download(request1, spider);
Assert.AreEqual(1, page.TargetRequests.Count);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsHandler
{
Content = "网易倒闭啦"
});
page = downloader1.Download(request1, spider);
Assert.AreEqual(0, page.TargetRequests.Count);
}
public void SubContentHandler()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c"
});
var request1 = new Request("http://a.com/", null);
Page page = downloader1.Download(request1, spider);
Assert.AreEqual("aabbc", page.Content);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c",
EndOffset = 1
});
page = downloader1.Download(request1, spider);
Assert.AreEqual("aabb", page.Content);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c",
StartOffset = 1
});
page = downloader1.Download(request1, spider);
Assert.AreEqual("abbc", page.Content);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c",
StartOffset = 1,
EndOffset = 1
});
page = downloader1.Download(request1, spider);
Assert.AreEqual("abb", page.Content);
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c",
StartOffset = 10
});
var downloader2 = downloader1;
try
{
page = downloader2.Download(request1, spider);
throw new System.Exception("test failed.");
}
catch (SpiderException exception)
{
Assert.AreEqual("Sub content failed. Please check your settings.", exception.Message);
}
downloader1 = new TestDownloader();
downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler
{
Start = "a",
End = "c",
EndOffset = 20
});
try
{
page = downloader1.Download(request1, spider);
throw new System.Exception("test failed.");
}
catch (SpiderException exception)
{
Assert.AreEqual("Sub content failed. Please check your settings.", exception.Message);
}
}
