/// <summary> /// Create custom repositories for scheduler or default if these are null /// </summary> /// <param name="allowUriRecrawling">Allow crawl this uri again if something was fail?</param> /// <param name="crawledUrlRepository">Crawled url repository</param> /// <param name="pagesToCrawlRepository">Pages to crawl repository</param> public Scheduler(bool allowUriRecrawling, ICrawledUrlRepository crawledUrlRepository, IQueueOfPagesToCrawlRepository pagesToCrawlRepository) { AllowUriRecrawling = allowUriRecrawling; CrawledUrlRepository = crawledUrlRepository ?? new CompactCrawledUrlRepository(); PagesToCrawlRepository = pagesToCrawlRepository ?? new QueueOfPagesToCrawlRepository(); }
public void NoFalseNegativesTest() { // create input collection List <Uri> inputs = GenerateRandomDataList(10000); // instantiate filter and populate it with the inputs using (ICrawledUrlRepository uut = GetInstance()) { //If all were unique then they should all return "true" for Contains() foreach (Uri input in inputs) { Assert.IsTrue(uut.AddIfNew(input)); } //If all were added successfully then they should all return "true" for Contains() foreach (Uri input in inputs) { if (!uut.Contains(input)) { Assert.Fail("False negative: {0}", input); } } } }
public MadBidScheduler(bool allowUriRecrawling, ICrawledUrlRepository crawledUrlRepo, IPagesToCrawlRepository pagesToCrawlRepo) { rand1 = new Random((int)DateTime.Now.Ticks & 0x0000FFFF); _allowUriRecrawling = allowUriRecrawling; _crawledUrlRepo = crawledUrlRepo ?? new InMemoryCrawledUrlRepository(); }
public Scheduler(bool allowUriRecrawling, ICrawledUrlRepository crawledUrlRepo, IPagesToCrawlRepository pagesToCrawlRepo) { _allowUriRecrawling = allowUriRecrawling; _crawledUrlRepo = crawledUrlRepo ?? new CompactCrawledUrlRepository(); _pagesToCrawlRepo = pagesToCrawlRepo ?? new FifoPagesToCrawlRepository(); }
public void SetUp() { _unitUnderTest = GetInstance(); _uri1 = new Uri("http://a.com"); _uri2 = new Uri("http://b.com"); }
public void SetUp() { _unitUnderTest = GetInstance(); _uri1 = new Uri("http://a.com"); _uri2 = new Uri("http://b.com"); }
public Scheduler(bool allowUriRecrawling, ICrawledUrlRepository crawledUrlRepo, IPagesToCrawlRepository pagesToCrawlRepo) { _allowUriRecrawling = allowUriRecrawling; _crawledUrlRepo = crawledUrlRepo ?? new CompactCrawledUrlRepository(); _pagesToCrawlRepo = pagesToCrawlRepo ?? new FifoPagesToCrawlRepository(); }
public Scheduler(ICrawledUrlRepository crawledUrlRepo, IPagesToCrawlRepository pagesToCrawlRepo) { _crawledUrlRepo = crawledUrlRepo ?? new InMemoryCrawledUrlStorage(); _pagesToCrawlRepo = pagesToCrawlRepo ?? new FifoPagesToCrawlStorage(); }