public void startCrawlingFtdTest()
{
Page page = new Page();
page.urlStr = "http://www.ftd.de";
CrawlerController crawlerController = new CrawlerController();
ICrawlerJobView crawlerJobView = crawlerController.getCrawlerJobView();
//
int peekProcessInformationCount = 0;
int updateEventCounter = 0;
int imageEventCounter = 0;
int trackedProcessInformationUpdateCounter = 0;
int trackedProcessInformationImageUpdateCounter = 0;
double trackedProcessInformationPercentagePeek = 0.0000;
//
CrawlerProcessInformation trackedProcessInformation = null;
crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
{
List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
updateEventCounter++;
//
if (trackedProcessInformation == null && crawlerProcessInformation != null)
{
trackedProcessInformation = crawlerProcessInformation;
trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information)
{
trackedProcessInformationUpdateCounter++;
trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
});
trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image)
{
trackedProcessInformationImageUpdateCounter++;
});
}
});
crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage)
{
imageEventCounter++;
});
//
crawlerController.startCrawling(page);
crawlerController.getFinishedEvent().WaitOne(System.TimeSpan.FromSeconds(5), false);
//
Assert.IsTrue(peekProcessInformationCount > 0);
Assert.IsTrue(updateEventCounter > 10);
Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
}
private void crawlerJobProgressPercentageChangeEvent(ICrawlerJob crawlerJob, double progressPercentage)
{
if (!this.decoupleEvents)
{
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
crawlerProcessInformation.progressPercentage = progressPercentage;
crawlerProcessInformation.triggerCrawlerProcessInformationUpdateEvent();
}
}
}
private void generateAndRegisterInitialCrawlerJob()
{
ICrawlerJob crawlerJob = this.initialJobFactory.generateInitialCrawlerJob(this, this.pageBacklog);
crawlerJob.setCrawlerImageFilter(this.crawlerImageFilter);
crawlerJob.setCrawlerImageBacklog(this.crawlerImageBacklog);
crawlerJob.setPageFilter(this.pageFilter);
this.crawlerJobPool.addCrawlerJob(crawlerJob);
CrawlerProcessInformation crawlerProcessInformation = this.crawlerJobMonitor.registerCrawlerJob(crawlerJob);
this.crawlerJobView.registerCrawlerProcessInformation(crawlerProcessInformation);
}
private void crawlerJobProcessStateChangeEvent(ICrawlerJob crawlerJob, CrawlerJobProcessState processState)
{
if (!this.decoupleEvents)
{
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
//
crawlerProcessInformation.processState = this.determineProcessInformationProcessStateFromCrawlerJobProcessState(processState);
crawlerProcessInformation.pageUrlStr = this.determinePageUrlStrFromCrawlerJob(crawlerJob);
crawlerProcessInformation.triggerCrawlerProcessInformationUpdateEvent();
}
}
}
private CrawlerProcessInformation determineCrawlerProcessInformationForCrawlerJob(ICrawlerJob crawlerJob)
{
//
CrawlerProcessInformation crawlerProcessInformation = null;
//
if (crawlerJob != null)
{
lock (this.crawlerJobToProcessInformationDictionary)
{
this.crawlerJobToProcessInformationDictionary.TryGetValue(crawlerJob, out crawlerProcessInformation);
}
}
//
return(crawlerProcessInformation);
}
private void crawlerJobImageRetrievedEvent(ICrawlerJob crawlerJob, CrawlerImage crawlerImage)
{
if (!this.decoupleEvents && crawlerImage != null)
{
//processinformation -> inside the monitor
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
crawlerProcessInformation.triggerCrawlerProcessInformationNewImageEvent(crawlerImage);
}
//some other components of the controller -> outside of the monitor
if (this.imageRetrievedEvent != null)
{
this.imageRetrievedEvent(crawlerJob, crawlerImage.asEventArg());
}
}
}
public CrawlerProcessInformation registerCrawlerJob(ICrawlerJob crawlerJob)
{
//
CrawlerProcessInformation crawlerProcessInformation = new CrawlerProcessInformation();
//
if (crawlerJob != null)
{
//
crawlerJob.processStateChangeEvent += new CrawlerJobProcessStateChangeEvent(this.crawlerJobProcessStateChangeEvent);
crawlerJob.progressPercentageChangeEvent += new CrawlerJobProgressPercentageChangeEvent(this.crawlerJobProgressPercentageChangeEvent);
crawlerJob.crawlerJobImageRetrievedEvent += new CrawlerJobImageRetrievedEvent(crawlerJobImageRetrievedEvent);
//
lock (this.crawlerJobToProcessInformationDictionary)
{
this.crawlerJobToProcessInformationDictionary.Add(crawlerJob, crawlerProcessInformation);
}
}
return(crawlerProcessInformation);
}
public ProcessInformationPanel(CrawlerProcessInformation crawlerProcessInformation)
{
//
this.Controls.Add(this.labelStartTimestamp);
this.Controls.Add(this.labelRunningTimestamp);
this.Controls.Add(this.labelProcessState);
this.Controls.Add(this.labelUrl);
this.Controls.Add(this.progressBar);
//
this.labelStartTimestamp.Text = "";
this.labelStartTimestamp.AutoSize = true;
this.labelStartTimestamp.Padding = new Padding(2);
//
this.labelRunningTimestamp.Text = "";
this.labelRunningTimestamp.AutoSize = true;
this.labelRunningTimestamp.Padding = new Padding(2);
//
this.labelUrl.Text = "";
this.labelUrl.AutoSize = true;
this.labelUrl.Padding = new Padding(2);
this.labelUrl.LinkClicked += new LinkLabelLinkClickedEventHandler(delegate(object sender, LinkLabelLinkClickedEventArgs args)
{
System.Diagnostics.Process.Start(this.labelUrl.Text);
this.labelUrl.LinkVisited = true;
});
this.labelProcessState.Text = "";
this.labelProcessState.AutoSize = true;
this.labelProcessState.Padding = new Padding(2);
this.progressBar.Height = 12;
//
this.BackColor = Color.GhostWhite;
this.BorderStyle = BorderStyle.FixedSingle;
this.AutoSize = true;
this.Dock = DockStyle.Top;
this.Margin = new Padding(5);
//
if (crawlerProcessInformation != null)
{
System.Windows.Forms.Timer timer = new System.Windows.Forms.Timer();
timer.Interval = 1;
timer.Tick += new EventHandler(delegate(object sender, EventArgs args)
{
//
timer.Interval = 200;
//
if (crawlerProcessInformation.isActiveProcess())
{
//time throttle for process update
if (System.DateTime.Now.Subtract(this.lastUpdateTime).Milliseconds > 100)
{
try
{
this.Invoke(new MethodInvoker(delegate()
{
//
String startedAt = "Started at: " + crawlerProcessInformation.createdTime.ToLongTimeString();
String runningDuration = "Running: " + String.Format("{0:d3}s", crawlerProcessInformation.runningTime.Seconds);
String pageUrl = crawlerProcessInformation.pageUrlStr;
String processState = Enum.GetName(typeof(CrawlerJobProcessState), crawlerProcessInformation.processState);
int progressValue = (int)Math.Min(100, Math.Max(0, Math.Round(crawlerProcessInformation.progressPercentage * 100)));
//
if (!this.labelStartTimestamp.Text.Equals(startedAt))
{
this.labelStartTimestamp.Text = startedAt;
}
if (!this.labelRunningTimestamp.Text.Equals(runningDuration))
{
this.labelRunningTimestamp.Text = runningDuration;
}
if (!this.labelUrl.Text.Equals(pageUrl))
{
this.labelUrl.Text = pageUrl;
}
if (!this.labelProcessState.Text.Equals(processState))
{
this.labelProcessState.Text = processState;
}
if (this.progressBar.Value != progressValue)
{
this.progressBar.Value = progressValue;
}
}));
}
catch (Exception e) { }
//
this.lastUpdateTime = System.DateTime.Now;
}
}
else
{
try
{
this.Invoke(new MethodInvoker(delegate()
{
//
if (this.Parent != null)
{
this.Parent.Controls.Remove(this);
}
//
timer.Stop();
}));
}
catch (Exception e) { }
}
});
//
timer.Start();
}
}
public void startCrawlingTest()
{
//
Page page = new Page();
page.urlStr = "http://www.google.de";
CrawlerController crawlerController = new CrawlerController();
ICrawlerJobView crawlerJobView = crawlerController.getCrawlerJobView();
//
Assert.IsNotNull(crawlerJobView);
//
int peekProcessInformationCount = 0;
int updateEventCounter = 0;
int imageEventCounter = 0;
int trackedProcessInformationUpdateCounter = 0;
int trackedProcessInformationImageUpdateCounter = 0;
double trackedProcessInformationPercentagePeek = 0.0000;
//
CrawlerProcessInformation trackedProcessInformation = null;
crawlerJobView.updateProcessInformationEvent += new CrawlerJobViewUpdateEventProcessInformation(delegate(CrawlerProcessInformation crawlerProcessInformation)
{
List <CrawlerProcessInformation> crawlerProcessInformationList = crawlerJobView.getActiveCrawlerProcessInformationList();
peekProcessInformationCount = Math.Max(peekProcessInformationCount, crawlerProcessInformationList.Count);
updateEventCounter++;
//
if (trackedProcessInformation == null && crawlerProcessInformation != null)
{
trackedProcessInformation = crawlerProcessInformation;
trackedProcessInformation.crawlerProcessInformationUpdateEvent += new CrawlerProcessInformationUpdateEvent(delegate(CrawlerProcessInformation information) {
trackedProcessInformationUpdateCounter++;
trackedProcessInformationPercentagePeek = Math.Max(trackedProcessInformationPercentagePeek, information.progressPercentage);
});
trackedProcessInformation.crawlerProcessInformationNewImageEvent += new CrawlerProcessInformationNewImageEvent(delegate(CrawlerImage image){
trackedProcessInformationImageUpdateCounter++;
});
}
});
crawlerJobView.updateImageEvent += new CrawlerJobViewUpdateEventImage(delegate(CrawlerImage crawlerImage){
imageEventCounter++;
});
//
int crawlerJobCount = 100;
int crawlerJobPartCount = 10;
crawlerController.setInitialJobFactory(new InitialJobFactoryMock(crawlerJobCount, crawlerJobPartCount));
crawlerController.setMaximumCrawlerJobThreadCount(20);
crawlerController.startCrawling(page);
crawlerController.getFinishedEvent().WaitOne(Timeout.Infinite, false);
//
Assert.IsTrue(peekProcessInformationCount > 1);
Assert.IsTrue(updateEventCounter > crawlerJobCount * 10);
Assert.AreEqual(crawlerJobCount * crawlerJobPartCount, imageEventCounter);
Assert.IsTrue(trackedProcessInformationUpdateCounter > 10);
Assert.IsTrue(trackedProcessInformationPercentagePeek > 0.3);
Assert.AreEqual(crawlerJobPartCount, trackedProcessInformationImageUpdateCounter);
}
