/// <summary>
/// Creates and instance of the CrawlerWorkPool
/// </summary>
/// <param name="poolName">Pool name</param>
/// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param>
public WorkPool(string poolName, int numberOfCrawlers)
{
this.poolName = poolName;
workSource = new WorkSource();
workersCount = numberOfCrawlers;
crawlers = new List<WorkProcessor>();
}
/// <summary>
/// Stops the pool's crawlers
/// </summary>
public void Stop()
{
foreach (var crawler in crawlers)
{
crawler.Stop();
}
crawlers.Clear();
if (workSource != null)
{
workSource.DisposeSource();
workSource = null;
}
}
/// <summary>
/// Starts a dedicated to the crawler's thread.Which countinously takes and executes work.
/// </summary>
/// <param name="workSource">Crawler worksource</param>
public void StartCrawling(WorkSource workSource)
{
if (thread == null)
{
thread = new Thread(new ThreadStart(() =>
{
while (!shouldStop)
{
IWork work;
try
{
work = workSource.GetAvailableWork();
if (work != null)
{
work.Start();
workSource.ReturnFinishedWork(work);
}
}
catch (Exception ex)
{
}
}
}));
thread.Name = name;
thread.Start();
}
else
{
throw new InvalidOperationException(String.Format("Crawler '{0}' is already running on another source.", name));
}
}