public void RetryWhenContainsIllegalStringHandler() { var spider = new DefaultSpider("test", new Site()); TestDownloader downloader1 = new TestDownloader() { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new RetryWhenContainsHandler { Content = "aabbcccdefg下载人数100" } } }; var request1 = new Request("http://www.163.com/", null); Page page = downloader1.Download(request1, spider); Assert.AreEqual(1, page.TargetRequests.Count); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new RetryWhenContainsHandler { Content = "网易倒闭啦" } } }; page = downloader1.Download(request1, spider); Assert.AreEqual(0, page.TargetRequests.Count); }
public void GeneratePostBody() { var spider = new DefaultSpider("test", new Site { Arguments = new Dictionary <string, string> { { "TOKEN1", "TEST1" } } }); TestDownloader downloader = new TestDownloader { BeforeDownloadHandlers = new IBeforeDownloadHandler[] { new GeneratePostBodyHandler { ArgumnetNames = new[] { "TOKEN1", "TOKEN2" } } } }; var request1 = new Request("http://a.com/", new Dictionary <string, dynamic> { { "TOKEN2", "TEST2" } }) { PostBody = "{0}{1}" }; Page page = downloader.Download(request1, spider); Assert.AreEqual("TEST1TEST2", page.Request.PostBody); var request2 = new Request("http://a.com/", new Dictionary <string, dynamic>()) { PostBody = "{0}{1}" }; page = downloader.Download(request2, spider); Assert.AreEqual("TEST1", page.Request.PostBody); request1 = new Request("http://a.com/", new Dictionary <string, dynamic> { { "TOKEN2", "TEST2" } }) { PostBody = "{0}{1}" }; var spider2 = new DefaultSpider("test", new Site()); page = downloader.Download(request1, spider2); Assert.AreEqual("TEST2", page.Request.PostBody); }
public void RetryWhenContainsIllegalStringHandler() { var spider = new DefaultSpider("test", new Site()); TestDownloader downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsContentHandler("aabbcccdefg下载人数100")); var request1 = new Request("http://www.163.com/", null); Page page = downloader1.Download(request1, spider).Result; Assert.Single(page.TargetRequests); downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new RetryWhenContainsContentHandler("网易倒闭啦")); page = downloader1.Download(request1, spider).Result; Assert.Empty(page.TargetRequests); }
public void SubContentHandler() { var spider = new DefaultSpider("test", new Site()); TestDownloader downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c" }); var request1 = new Request("http://a.com/", null); Page page = downloader1.Download(request1, spider); Assert.AreEqual("aabbc", page.Content); downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c", EndOffset = 1 }); page = downloader1.Download(request1, spider); Assert.AreEqual("aabb", page.Content); downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c", StartOffset = 1 }); page = downloader1.Download(request1, spider); Assert.AreEqual("abbc", page.Content); downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c", StartOffset = 1, EndOffset = 1 }); page = downloader1.Download(request1, spider); Assert.AreEqual("abb", page.Content); downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c", StartOffset = 10 }); var downloader2 = downloader1; try { page = downloader2.Download(request1, spider); throw new System.Exception("test failed."); } catch (SpiderException exception) { Assert.AreEqual("Sub content failed. Please check your settings.", exception.Message); } downloader1 = new TestDownloader(); downloader1.AddAfterDownloadCompleteHandler(new SubContentHandler { Start = "a", End = "c", EndOffset = 20 }); try { page = downloader1.Download(request1, spider); throw new System.Exception("test failed."); } catch (SpiderException exception) { Assert.AreEqual("Sub content failed. Please check your settings.", exception.Message); } }
public void SubContentHandler() { var spider = new DefaultSpider("test", new Site()); TestDownloader downloader1 = new TestDownloader() { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c" } } }; var request1 = new Request("http://a.com/", 0, null); Page page = downloader1.Download(request1, spider); Assert.Equal("aabbc", page.Content); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c", EndOffset = 1 } } }; page = downloader1.Download(request1, spider); Assert.Equal("aabb", page.Content); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c", StartOffset = 1 } } }; page = downloader1.Download(request1, spider); Assert.Equal("abbc", page.Content); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c", StartOffset = 1, EndOffset = 1 } } }; page = downloader1.Download(request1, spider); Assert.Equal("abb", page.Content); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c", StartOffset = 10 } } }; var downloader2 = downloader1; var exception = Assert.Throws <SpiderException>(() => { page = downloader2.Download(request1, spider); }); Assert.Equal("Sub content failed. Please check your settings.", exception.Message); downloader1 = new TestDownloader { DownloadCompleteHandlers = new IDownloadCompleteHandler[] { new SubContentHandler { Start = "a", End = "c", EndOffset = 20 } } }; exception = Assert.Throws <SpiderException>(() => { page = downloader1.Download(request1, spider); }); Assert.Equal("Sub content failed. Please check your settings.", exception.Message); }