Пример #1
0
			public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
			{
				m_ThreadSafeCounter = threadSafeCounter;
				CrawlerQueueEntry = crawlerQueueEntry;
			}
Пример #2
0
 public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter)
 {
     m_ThreadSafeCounter = threadSafeCounter;
 }
Пример #3
0
 public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
 {
     m_ThreadSafeCounter = threadSafeCounter;
     CrawlerQueueEntry   = crawlerQueueEntry;
 }
Пример #4
0
        private void EndDownload(CrawlStep crawlStep, PropertyBag propertyBag, Exception exception,
            ThreadSafeCounter.ThreadSafeCounterCookie counterCookie)
        {
            using (counterCookie)
            {
                if (exception != null)
                {
                    OnDownloadException(exception, crawlStep);
                } else if (!propertyBag.IsNull())
                {
                    propertyBag.Referrer = crawlStep;

                    // Assign initial properties to propertybag
                    if (!counterCookie.CrawlerQueueEntry.Properties.IsNull())
                    {
                        counterCookie.CrawlerQueueEntry.Properties.
                            ForEach(key => propertyBag[key.Key].Value = key.Value);
                    }

                    if (OnAfterDownload(crawlStep, propertyBag))
                    {
                        // Executes all the pipelines sequentially for each downloaded content
                        // in the crawl process. Used to extract data from content, like which
                        // url's to follow, email addresses, aso.
                        Pipeline.ForEach(pipelineStep => ExecutePipeLineStep(pipelineStep, propertyBag));
                    }
                }
            }

            ProcessQueue();
        }