public void CrawleOnePage() { //arrange var loader = new Mock<IHtmlDocumentLoader>(); var context = new Mock<ICrawlerRepository>(); var crawler = new CareersStackoverfowComCrawler(_logger); var document = new HtmlDocument(); document.Load(new FileStream("TestData/careers/careers.results.htm", FileMode.Open)); loader.Setup(l => l.LoadDocument("http://careers.stackoverflow.com/Jobs?searchTerm=.net,java,c%2B%2B&searchType=Any&location=&range=20&pg=1")).Returns(document); loader.Setup(l => l.LoadDocument("http://careers.stackoverflow.com/Jobs?searchTerm=.net,java,c%2B%2B&searchType=Any&location=&range=20&pg=2")).Returns(new HtmlDocument()); var vacancy = new HtmlDocument(); vacancy.Load(new FileStream("TestData/careers/vacancy.htm", FileMode.Open)); loader.Setup(l => l.LoadDocument(It.IsRegex(@"http://careers.stackoverflow.com/Jobs/(\d+)\?campaign=(\w+)"))). Returns(vacancy); var storage = new List<TddDemandRecord>(); context.Setup(c => c.Add(It.IsAny<TddDemandRecord>())).Callback((TddDemandRecord r) => storage.Add(r)); //act crawler.Crawle(loader.Object, context.Object); //assert context.Verify(c => c.SaveChanges()); Assert.That(storage.Count, Is.EqualTo(25), "Expected that all 25 jobs processed"); }
static void Main(string[] args) { var logger = new ConsoleLogger(); try { var loader = new HtmlDocumentLoader(); var repository = new CrawlerRepository(); //var crawler = new RabotaUaCrawler(logger); //var crawler = new PrgJobsComCrawler(logger); var crawler = new CareersStackoverfowComCrawler(logger); crawler.Crawle(loader, repository); } catch (Exception e) { logger.Log("FAILED exception caught in Main() method. Exception message: " + e.Message); logger.Log(e.InnerException.Message); logger.Log(e.StackTrace); } }