/// <summary> /// Creates and instance of the CrawlerWorkPool /// </summary> /// <param name="poolName">Pool name</param> /// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param> public WorkPool(string poolName, int numberOfWorkProcessors) { this.poolName = poolName; this.workSource = new WorkSource(); this.numberOfWorkProcessors = numberOfWorkProcessors; this.crawlers = new List <WorkProcessor>(); }
/// <summary> /// Creates and instance of the CrawlerWorkPool /// </summary> /// <param name="poolName">Pool name</param> /// <param name="numberOfCrawlers">Specifies the number of crawlers to serve the pool</param> public WorkPool(string poolName, int numberOfWorkProcessors) { this.poolName = poolName; this.workSource = new WorkSource(); this.numberOfWorkProcessors = numberOfWorkProcessors; this.crawlers = new List<WorkProcessor>(); }
/// <summary> /// Stops the pool's crawlers /// </summary> public void Stop() { crawlers.ForEach(crawler => crawler.Stop()); crawlers.Clear(); if (workSource != null) { workSource.DisposeSource(); workSource = null; } }
/// <summary> /// Starts dedicated crawler's thread which countinously takes and executes work. /// </summary> /// <param name="workSource">Crawler worksource</param> public void StartCrawling(WorkSource workSource) { if (thread == null) { thread = new Thread(new ThreadStart(() => { while (!shouldStop) { try { log.Debug("Getting available work..."); work = workSource.GetAvailableWork(); if (work != null) { log.DebugFormat("Executing work [{0}]", work); work.Start(); log.InfoFormat("Work finished successfully. [{0}]", work); workSource.ReturnFinishedWork(work); log.DebugFormat("Work returned to the source. [{0}]", work); } } catch (Exception ex) { log.ErrorException("Exception occured while executing a work. You should take care for all exceptions while you implement 'ICrawlerJob.Start()' method.", ex); } } log.Info("Crowler was stopped."); })); thread.Name = name; thread.Start(); } else { log.FatalFormat("Crawler '{0}' is already running on another source.", name); throw new InvalidOperationException(String.Format("Crawler '{0}' is already running on another source.", name)); } }