/// <summary>
/// Creates and instance of the CrawlerWorkPool
/// </summary>
/// <param name="poolName">Pool name</param>
/// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param>
public WorkPool(string poolName, int numberOfWorkProcessors)
{
this.poolName = poolName;
this.workSource = new WorkSource();
this.numberOfWorkProcessors = numberOfWorkProcessors;
this.crawlers = new List <WorkProcessor>();
}
/// <summary>
/// Creates and instance of the CrawlerWorkPool
/// </summary>
/// <param name="poolName">Pool name</param>
/// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param>
public WorkPool(string poolName, int numberOfWorkProcessors)
{
this.poolName = poolName;
this.workSource = new WorkSource();
this.numberOfWorkProcessors = numberOfWorkProcessors;
this.crawlers = new List<WorkProcessor>();
}
/// <summary>
/// Stops the pool's crawlers
/// </summary>
public void Stop()
{
crawlers.ForEach(crawler => crawler.Stop());
crawlers.Clear();
if (workSource != null)
{
workSource.DisposeSource();
workSource = null;
}
}
/// <summary>
/// Stops the pool's crawlers
/// </summary>
public void Stop()
{
crawlers.ForEach(crawler => crawler.Stop());
crawlers.Clear();
if (workSource != null)
{
workSource.DisposeSource();
workSource = null;
}
}
/// <summary>
/// Starts dedicated crawler's thread which countinously takes and executes work.
/// </summary>
/// <param name="workSource">Crawler worksource</param>
public void StartCrawling(WorkSource workSource)
{
if (thread == null)
{
thread = new Thread(new ThreadStart(() =>
{
while (!shouldStop)
{
try
{
log.Debug("Getting available work...");
work = workSource.GetAvailableWork();
if (work != null)
{
log.DebugFormat("Executing work [{0}]", work);
work.Start();
log.InfoFormat("Work finished successfully. [{0}]", work);
workSource.ReturnFinishedWork(work);
log.DebugFormat("Work returned to the source. [{0}]", work);
}
}
catch (Exception ex)
{
log.ErrorException("Exception occured while executing a work. You should take care for all exceptions while you implement 'ICrawlerJob.Start()' method.", ex);
}
}
log.Info("Crowler was stopped.");
}));
thread.Name = name;
thread.Start();
}
else
{
log.FatalFormat("Crawler '{0}' is already running on another source.", name);
throw new InvalidOperationException(String.Format("Crawler '{0}' is already running on another source.", name));
}
}
