public void IncrementTargetUrls()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader = new TestDownloader();
downloader.AddAfterDownloadCompleteHandler(new IncrementTargetUrlsBuilder("&page=0", 2));
var request = new Request("http://a.com/?&page=0", null);
Page page = downloader.Download(request, spider).Result;
var request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
page = downloader.Download(request2, spider).Result;
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=4", request2.Url.ToString());
downloader = new TestDownloader();
downloader.AddAfterDownloadCompleteHandler(new RequestExtraTargetUrlsBuilder("&page=0", "page_index"));
request = new Request("http://a.com/?&page=0", new Dictionary <string, object>()
{
{ "page_index", 2 }
});
page = downloader.Download(request, spider).Result;
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
Assert.Equal(2, request2.GetExtra("page_index"));
}
public void PaggerStopper()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader = new TestDownloader();
downloader.AddAfterDownloadCompleteHandler(new IncrementTargetUrlsCreator("&page=0", 2)
{
Stopper = new PaggerStopper
{
CurrenctPageSelector = new BaseSelector
{
Expression = ""
}
}
});
var request = new Request("http://a.com/?&page=0", null);
Page page = downloader.Download(request, spider);
var request2 = page.TargetRequests.First();
Assert.AreEqual("http://a.com/?&page=2", request2.Url.ToString());
page = downloader.Download(request2, spider);
request2 = page.TargetRequests.First();
Assert.AreEqual("http://a.com/?&page=4", request2.Url.ToString());
}
public void IncrementTargetUrls()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader = new TestDownloader();
downloader.AddAfterDownloadCompleteHandler(new TargetUrlsHandler(new AutoIncrementTargetUrlsExtractor("&page=0", 2)));
var request = new Request("http://a.com/?&page=0", null);
Page page = downloader.Download(request, spider).Result;
var request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
page = downloader.Download(request2, spider).Result;
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=4", request2.Url.ToString());
}
public void IncrementTargetUrls()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader = new TestDownloader
{
DownloadCompleteHandlers = new IDownloadCompleteHandler[]
{
new IncrementTargetUrlsCreator("&page=0", 2)
}
};
var request = new Request("http://a.com/?&page=0", 0, null);
Page page = downloader.Download(request, spider);
var request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
page = downloader.Download(request2, spider);
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=4", request2.Url.ToString());
}
public void IncrementTargetUrls()
{
var spider = new DefaultSpider("test", new Site());
TestDownloader downloader = new TestDownloader
{
DownloadCompleteHandlers = new IDownloadCompleteHandler[]
{
new IncrementTargetUrlsCreator("&page=0", 2)
}
};
var request = new Request("http://a.com/?&page=0", null);
Page page = downloader.Download(request, spider);
var request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
page = downloader.Download(request2, spider);
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=4", request2.Url.ToString());
downloader = new TestDownloader
{
DownloadCompleteHandlers = new IDownloadCompleteHandler[]
{
new IncrementTargetUrlsCreator("&page=0", 2)
{
PageIndexKey = "page_index"
}
}
};
request = new Request("http://a.com/?&page=0", new Dictionary <string, object>()
{
{ "page_index", 1 }
});
page = downloader.Download(request, spider);
request2 = page.TargetRequests.First();
Assert.Equal("http://a.com/?&page=2", request2.Url.ToString());
Assert.Equal(2, request2.GetExtra("page_index"));
}
