public CrawlerJob(ICrawlerController crawlerController) : this()
{
this.setParentalCrawlerController(crawlerController);
}
public virtual ICrawlerJob generateInitialCrawlerJob(ICrawlerController crawlerController, IPageBacklog pageBacklog)
{
return(new CrawlerJob(crawlerController).addCrawlerJobPartInitialPageLock(pageBacklog));
}
public ICrawlerJob setParentalCrawlerController(ICrawlerController crawlerController)
{
this.crawlerJobContext.crawlerController = crawlerController;
return(this);
}
public override ICrawlerJob generateInitialCrawlerJob(ICrawlerController crawlerController, IPageBacklog pageBacklog)
{
return(new CrawlerJob().setParentalCrawlerController(crawlerController).addCrawlerJobPartAndSetPage(new CrawlerJobPartMock(this.jobPartCount, this.jobCount - 1), new Page("http://www.google.de")));
}
ICrawlerJob ICrawlerJob.setParentalCrawlerController(ICrawlerController crawlerController)
{
return(this);
}
public List <ICrawlerJobPart> run()
{
//
List <ICrawlerJobPart> crawlerJobPartList = new List <ICrawlerJobPart>();
//
if (this.progressPercentageChangeEvent != null)
{
this.progressPercentageChangeEvent(0.000);
}
//
if (this.crawlerJobContext != null && this.crawlerJobContext.page != null && this.crawlerJobContext.crawlerJob != null && !this.isAborting)
{
ICrawlerController crawlerController = this.crawlerJobContext.crawlerJob.getParentalCrawlerController();
if (this.crawlerJobContext.crawlerJob.getParentalCrawlerController() != null)
{
//
List <String> ankerHrefLinkList = new List <String>();
if (this.crawlerJobContext.page.ankerHrefLinkList != null)
{
ankerHrefLinkList.AddRange(this.crawlerJobContext.page.ankerHrefLinkList);
}
//
if (this.crawlerJobContext.page.redirectUrlStrList != null)
{
ankerHrefLinkList.AddRange(this.crawlerJobContext.page.redirectUrlStrList);
}
//
int counter = 0;
int counterMax = ankerHrefLinkList.Count;
foreach (String ankerHrefLink in ankerHrefLinkList)
{
//
if (this.isAborting)
{
break;
}
//
Page page = new Page();
//page.parent = this.page;
//this.page.childPageList.Add(page);
page.urlStr = ankerHrefLink;
//image is part of the url
if (Regex.IsMatch(ankerHrefLink, "image"))
{
page.priority = 10;
}
//image type is part of the url
if (Regex.IsMatch(ankerHrefLink, "jp.?g|gif|png|bmp"))
{
page.priority = 10;
}
/*
* end special priority behavior
*/
//
crawlerController.addPage(page);
//
counter++;
if (this.progressPercentageChangeEvent != null)
{
this.progressPercentageChangeEvent(counter * 1.0 / counterMax);
}
//
if (this.crawlerJobPartSuspendEvent != null)
{
this.crawlerJobPartSuspendEvent();
}
}
//
this.crawlerJobContext.page.ankerHrefLinkList = null;
this.crawlerJobContext.page.redirectUrlStrList = null;
}
}
//
if (this.progressPercentageChangeEvent != null)
{
this.progressPercentageChangeEvent(1.000);
}
//
return(crawlerJobPartList);
}
/// <summary>
/// Called right after the form is initialized in order to connect to the Remotable
/// object exposed by the Client Core.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void tmrInit_Tick(object sender, System.EventArgs e)
{
tmrInit.Enabled = false;
try
{
channel = new TcpChannel();
ChannelServices.RegisterChannel(channel, true);
core = (ICrawlerController) Activator.GetObject(
typeof(ICrawlerController),
"tcp://localhost:15460/Controller.rem");
tmrTimer.Enabled = true;
LoadSettings();
}
catch
{
MessageBox.Show("Failed to connect to Client Core. Please make sure it is running.", this.Text, MessageBoxButtons.OK, MessageBoxIcon.Error);
Application.Exit();
}
}
