/// <summary>
/// Removes a crawler job from the monitor.
/// </summary>
/// <param name="crawlerJob"></param>
public void unregisterCrawlerJob(ICrawlerJob crawlerJob)
{
//
lock (this.crawlerJobToProcessInformationDictionary)
{
this.crawlerJobToProcessInformationDictionary.Remove(crawlerJob);
}
}
public void addCrawlerJob(ICrawlerJob crawlerJob)
{
lock (this.availableCrawlerJobList)
{
this.availableCrawlerJobList.Add(crawlerJob);
crawlerJob.processStateChangeEvent += new CrawlerJobProcessStateChangeEvent(crawlerJob_finishedJobDelegate);
crawlerJob.crawlerJobSuspendEvent += new CrawlerJobSuspendEvent(crawlerJob_crawlerJobSuspendEvent);
}
}
public void removeCrawlerJob(ICrawlerJob crawlerJob)
{
lock (this.availableCrawlerJobList)
{
this.availableCrawlerJobList.Remove(crawlerJob);
}
if (this.crawlerJobPoolFinishedCrawlerJob != null)
{
this.crawlerJobPoolFinishedCrawlerJob(crawlerJob);
}
}
private void crawlerJobProgressPercentageChangeEvent(ICrawlerJob crawlerJob, double progressPercentage)
{
if (!this.decoupleEvents)
{
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
crawlerProcessInformation.progressPercentage = progressPercentage;
crawlerProcessInformation.triggerCrawlerProcessInformationUpdateEvent();
}
}
}
private void generateAndRegisterInitialCrawlerJob()
{
ICrawlerJob crawlerJob = this.initialJobFactory.generateInitialCrawlerJob(this, this.pageBacklog);
crawlerJob.setCrawlerImageFilter(this.crawlerImageFilter);
crawlerJob.setCrawlerImageBacklog(this.crawlerImageBacklog);
crawlerJob.setPageFilter(this.pageFilter);
this.crawlerJobPool.addCrawlerJob(crawlerJob);
CrawlerProcessInformation crawlerProcessInformation = this.crawlerJobMonitor.registerCrawlerJob(crawlerJob);
this.crawlerJobView.registerCrawlerProcessInformation(crawlerProcessInformation);
}
/// <summary>
/// tries to launch a new thread for another available worker. If successful returns true, otherwise false.
/// </summary>
/// <returns></returns>
protected Boolean launchNewThreadIfPossible()
{
//
Boolean retval = false;
//
ICrawlerJob crawlerJob = null;
lock (this.availableCrawlerJobList)
{
lock (this.runningCrawlerJobList)
{
try
{
if (this.availableCrawlerJobList.Count > 0 && this.runningCrawlerJobList.Count < this.threadCount)
{
//
crawlerJob = this.availableCrawlerJobList.ElementAt(0);
this.availableCrawlerJobList.Remove(crawlerJob);
this.runningCrawlerJobList.Add(crawlerJob);
//
Thread thread = new Thread(new ThreadStart(crawlerJob.run));
thread.Name = "worker" + ++workerThreadId;
thread.IsBackground = true;
thread.Start();
//
retval = true;
//
if (this.crawlerJobPoolLaunchingCrawlerJob != null)
{
this.crawlerJobPoolLaunchingCrawlerJob(crawlerJob);
}
//
if (this.crawlerJobPoolWorkerUpdate != null)
{
this.crawlerJobPoolWorkerUpdate(this.runningCrawlerJobList.Count);
}
}
}
catch (Exception e)
{ }
}
}
//
return(retval);
}
private void crawlerJobProcessStateChangeEvent(ICrawlerJob crawlerJob, CrawlerJobProcessState processState)
{
if (!this.decoupleEvents)
{
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
//
crawlerProcessInformation.processState = this.determineProcessInformationProcessStateFromCrawlerJobProcessState(processState);
crawlerProcessInformation.pageUrlStr = this.determinePageUrlStrFromCrawlerJob(crawlerJob);
crawlerProcessInformation.triggerCrawlerProcessInformationUpdateEvent();
}
}
}
private String determinePageUrlStrFromCrawlerJob(ICrawlerJob crawlerJob)
{
//
String retval = "";
//
if (crawlerJob != null)
{
Page page = crawlerJob.getPage();
if (page != null)
{
retval = page.urlStr;
}
}
//
return(retval);
}
private void crawlerJobImageRetrievedEvent(ICrawlerJob crawlerJob, CrawlerImage crawlerImage)
{
if (!this.decoupleEvents && crawlerImage != null)
{
//processinformation -> inside the monitor
CrawlerProcessInformation crawlerProcessInformation = this.determineCrawlerProcessInformationForCrawlerJob(crawlerJob);
if (crawlerProcessInformation != null)
{
crawlerProcessInformation.triggerCrawlerProcessInformationNewImageEvent(crawlerImage);
}
//some other components of the controller -> outside of the monitor
if (this.imageRetrievedEvent != null)
{
this.imageRetrievedEvent(crawlerJob, crawlerImage.asEventArg());
}
}
}
private void crawlerJob_finishedJobDelegate(ICrawlerJob sender, CrawlerJobProcessState processState)
{
if (sender != null && processState == CrawlerJobProcessState.Finished)
{
//
lock (this.runningCrawlerJobList)
{
//
this.runningCrawlerJobList.Remove(sender);
//
if (this.crawlerJobPoolWorkerUpdate != null)
{
this.crawlerJobPoolWorkerUpdate(this.runningCrawlerJobList.Count);
}
}
//
this.launcherThreadResetEvent.Set();
}
}
public CrawlerProcessInformation registerCrawlerJob(ICrawlerJob crawlerJob)
{
//
CrawlerProcessInformation crawlerProcessInformation = new CrawlerProcessInformation();
//
if (crawlerJob != null)
{
//
crawlerJob.processStateChangeEvent += new CrawlerJobProcessStateChangeEvent(this.crawlerJobProcessStateChangeEvent);
crawlerJob.progressPercentageChangeEvent += new CrawlerJobProgressPercentageChangeEvent(this.crawlerJobProgressPercentageChangeEvent);
crawlerJob.crawlerJobImageRetrievedEvent += new CrawlerJobImageRetrievedEvent(crawlerJobImageRetrievedEvent);
//
lock (this.crawlerJobToProcessInformationDictionary)
{
this.crawlerJobToProcessInformationDictionary.Add(crawlerJob, crawlerProcessInformation);
}
}
return(crawlerProcessInformation);
}
private CrawlerProcessInformation determineCrawlerProcessInformationForCrawlerJob(ICrawlerJob crawlerJob)
{
//
CrawlerProcessInformation crawlerProcessInformation = null;
//
if (crawlerJob != null)
{
lock (this.crawlerJobToProcessInformationDictionary)
{
this.crawlerJobToProcessInformationDictionary.TryGetValue(crawlerJob, out crawlerProcessInformation);
}
}
//
return(crawlerProcessInformation);
}
