public void Crawl_UsingOnDiskCrawledUrlRepository_VerifyCrawlResultIsAsExpected()
{
new PageRequester(new CrawlConfiguration {
UserAgentString = "aaa"
}).MakeRequest(new Uri("http://localhost:1111/PageGenerator/ClearCounters"));
using (var onDiskUrlRepo = new OnDiskCrawledUrlRepository(null, null, true))
{
Scheduler onDiskScheduler = new Scheduler(false, onDiskUrlRepo, new FifoPagesToCrawlRepository());
base.CrawlAndAssert(new PoliteWebCrawler(null, null, null, onDiskScheduler, null, null, null, null, null));
}
}
public void Dispose_DeletesCrawledUrlsDirectory()
{
DirectoryInfo directory = new DirectoryInfo("TestUrls");
using (OnDiskCrawledUrlRepository unitUnderTest = new OnDiskCrawledUrlRepository(new Md5HashGenerator(), directory, true))
{
Assert.IsTrue(Directory.Exists(directory.FullName));
}
Assert.IsFalse(Directory.Exists(directory.FullName));
}