/// <summary> /// Creates and instance of the CrawlerWorkPool /// </summary> /// <param name="poolName">Pool name</param> /// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param> public WorkPool(string poolName, int numberOfCrawlers) { this.poolName = poolName; workSource = new WorkSource(); workersCount = numberOfCrawlers; crawlers = new List<WorkProcessor>(); }
/// <summary> /// Stops the pool's crawlers /// </summary> public void Stop() { foreach (var crawler in crawlers) { crawler.Stop(); } crawlers.Clear(); if (workSource != null) { workSource.DisposeSource(); workSource = null; } }
/// <summary> /// Starts a dedicated to the crawler's thread.Which countinously takes and executes work. /// </summary> /// <param name="workSource">Crawler worksource</param> public void StartCrawling(WorkSource workSource) { if (thread == null) { thread = new Thread(new ThreadStart(() => { while (!shouldStop) { IWork work; try { work = workSource.GetAvailableWork(); if (work != null) { work.Start(); workSource.ReturnFinishedWork(work); } } catch (Exception ex) { } } })); thread.Name = name; thread.Start(); } else { throw new InvalidOperationException(String.Format("Crawler '{0}' is already running on another source.", name)); } }