public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry) { m_ThreadSafeCounter = threadSafeCounter; CrawlerQueueEntry = crawlerQueueEntry; }
public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter) { m_ThreadSafeCounter = threadSafeCounter; }
private void EndDownload(CrawlStep crawlStep, PropertyBag propertyBag, Exception exception, ThreadSafeCounter.ThreadSafeCounterCookie counterCookie) { using (counterCookie) { if (exception != null) { OnDownloadException(exception, crawlStep); } else if (!propertyBag.IsNull()) { propertyBag.Referrer = crawlStep; // Assign initial properties to propertybag if (!counterCookie.CrawlerQueueEntry.Properties.IsNull()) { counterCookie.CrawlerQueueEntry.Properties. ForEach(key => propertyBag[key.Key].Value = key.Value); } if (OnAfterDownload(crawlStep, propertyBag)) { // Executes all the pipelines sequentially for each downloaded content // in the crawl process. Used to extract data from content, like which // url's to follow, email addresses, aso. Pipeline.ForEach(pipelineStep => ExecutePipeLineStep(pipelineStep, propertyBag)); } } } ProcessQueue(); }