public void startCrawlingFtdTest()
{
Page page = new Page();
page.urlStr = "http://www.ftd.de";
CrawlerController crawlerController = new CrawlerController();
ICrawlerJobView crawlerJobView = crawlerController.getCrawlerJobView();
//
int peekProcessInformationCount = 0;
int updateEventCounter = 0;
int imageEventCounter = 0;
int trackedProcessInformationUpdateCounter = 0;
int trackedProcessInformationImageUpdateCounter = 0;
double trackedProcessInformationPercentagePeek = 0.0000;
//
CrawlerProcessInformation trackedProcessInformation = null;
crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
{
List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
updateEventCounter++;
//
if (trackedProcessInformation == null && crawlerProcessInformation != null)
{
trackedProcessInformation = crawlerProcessInformation;
trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information)
{
trackedProcessInformationUpdateCounter++;
trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
});
trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image)
{
trackedProcessInformationImageUpdateCounter++;
});
}
});
crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage)
{
imageEventCounter++;
});
//
crawlerController.startCrawling(page);
crawlerController.getFinishedEvent().WaitOne(System.TimeSpan.FromSeconds(5), false);
//
Assert.IsTrue(peekProcessInformationCount > 0);
Assert.IsTrue(updateEventCounter > 10);
Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
}
public static void Main(string[] args)
{
if (args.Length == 2 && args[0] != null && new DirectoryInfo(args[0]).Exists)
{
//
ICrawlerController crawlerController = new CrawlerController();
crawlerController.setImageLoggingFolder(args[0]);
//
crawlerController.startCrawling(new Page(args[1]));
//
Console.WriteLine("Crawling process is running. Press any key to stop crawling...");
Console.ReadLine();
//
crawlerController.stopCrawling();
}
else
{
Program.renderHelp();
}
}
public void startCrawlingTest()
{
//
Page page = new Page();
page.urlStr = "http://www.google.de";
CrawlerController crawlerController = new CrawlerController();
ICrawlerJobView crawlerJobView = crawlerController.getCrawlerJobView();
//
Assert.IsNotNull(crawlerJobView);
//
int peekProcessInformationCount = 0;
int updateEventCounter = 0;
int imageEventCounter = 0;
int trackedProcessInformationUpdateCounter = 0;
int trackedProcessInformationImageUpdateCounter = 0;
double trackedProcessInformationPercentagePeek = 0.0000;
//
CrawlerProcessInformation trackedProcessInformation = null;
crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
{
List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
updateEventCounter++;
//
if (trackedProcessInformation == null && crawlerProcessInformation != null)
{
trackedProcessInformation = crawlerProcessInformation;
trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information) {
trackedProcessInformationUpdateCounter++;
trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
});
trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image){
trackedProcessInformationImageUpdateCounter++;
});
}
});
crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage){
imageEventCounter++;
});
//
int crawlerJobCount = 100;
int crawlerJobPartCount = 10;
crawlerController.setInitialJobFactory(new InitialJobFactoryMock(crawlerJobCount, crawlerJobPartCount));
crawlerController.setMaximumCrawlerJobThreadCount(20);
crawlerController.startCrawling(page);
crawlerController.getFinishedEvent().WaitOne(Timeout.Infinite, false);
//
Assert.IsTrue(peekProcessInformationCount > 1);
Assert.IsTrue(updateEventCounter > crawlerJobCount * 10);
Assert.AreEqual(crawlerJobCount * crawlerJobPartCount, imageEventCounter);
Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
Assert.AreEqual(crawlerJobPartCount, trackedProcessInformationImageUpdateCounter);
}