예제 #1
0
        public void Add_IEnumerableParam_IsUriRecrawlingIsTrue_DuplicateAdded()
        {
            _unitUnderTest = new FifoScheduler(true);//this is the default

            _unitUnderTest.Add(new List<PageToCrawl> { new PageToCrawl(new Uri("http://a.com/")), new PageToCrawl(new Uri("http://a.com/")), new PageToCrawl(new Uri("http://a.com/")) });

            Assert.AreEqual(3, _unitUnderTest.Count);
        }
예제 #2
0
        public void Add_IsUriRecrawlingIsFalse_DuplicateNotAdded()
        {
            _unitUnderTest = new FifoScheduler(false);//this is the default
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));

            Assert.AreEqual(1, _unitUnderTest.Count);
        }
예제 #3
0
        public void Add_IsUriRecrawlingIsTrue_DuplicateAdded()
        {
            _unitUnderTest = new FifoScheduler(true);

            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));

            Assert.AreEqual(3, _unitUnderTest.Count);
        }
예제 #4
0
        public void SetUp()
        {
            _fakeHyperLinkParser = new Mock<IHyperLinkParser>();
            _fakeHttpRequester = new Mock<IPageRequester>();
            _fakeCrawlDecisionMaker = new Mock<ICrawlDecisionMaker>();
            _fakeDomainRateLimiter = new Mock<IDomainRateLimiter>();
            _fakeMemoryManager = new Mock<IMemoryManager>();
            _fakeRobotsDotTextFinder = new Mock<IRobotsDotTextFinder>();
            _fakeRobotsDotText = new Mock<IRobotsDotText>();

            _dummyScheduler = new FifoScheduler();
            _dummyThreadManager = new ProducerConsumerThreadManager(1);
            _dummyConfiguration = new CrawlConfiguration();
            _dummyConfiguration.ConfigurationExtensions.Add("somekey", "someval");

            _rootUri = new Uri("http://a.com/");
        }
예제 #5
0
        public void Clear_RemovesAllPrevious()
        {
            _unitUnderTest = new FifoScheduler();
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://b.com/")));
            _unitUnderTest.Add(new PageToCrawl(new Uri("http://c.com/")));

            _unitUnderTest.Clear();

            Assert.AreEqual(0, _unitUnderTest.Count);
        }
예제 #6
0
 public void SetUp()
 {
     _unitUnderTest = new FifoScheduler();
 }
예제 #7
0
        public void SetUp()
        {
            _fakeHyperLinkParser = new Mock<IHyperLinkParser>();
            _fakeHttpRequester = new Mock<IPageRequester>();
            _fakeCrawlDecisionMaker = new Mock<ICrawlDecisionMaker>();
            _fakeMemoryManager = new Mock<IMemoryManager>();
            _fakeDomainRateLimiter = new Mock<IDomainRateLimiter>();
            _fakeRobotsDotTextFinder = new Mock<IRobotsDotTextFinder>();

            _dummyScheduler = new FifoScheduler();
            _dummyThreadManager = new ProducerConsumerThreadManager(10);
            _dummyConfiguration = new CrawlConfiguration();
            _dummyConfiguration.ConfigurationExtensions.Add("somekey", "someval");

            _unitUnderTest = new PoliteWebCrawler(_dummyConfiguration, _fakeCrawlDecisionMaker.Object, _dummyThreadManager, _dummyScheduler, _fakeHttpRequester.Object, _fakeHyperLinkParser.Object, _fakeMemoryManager.Object, _fakeDomainRateLimiter.Object, _fakeRobotsDotTextFinder.Object);
            _unitUnderTest.CrawlBag.SomeVal = "SomeVal";
            _unitUnderTest.CrawlBag.SomeList = new List<string>() { "a", "b" };
            _rootUri = new Uri("http://a.com/");
        }