public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
{
m_ThreadSafeCounter = threadSafeCounter;
CrawlerQueueEntry = crawlerQueueEntry;
}
public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter)
{
m_ThreadSafeCounter = threadSafeCounter;
}
public ThreadSafeCounterCookie(ThreadSafeCounter threadSafeCounter, CrawlerQueueEntry crawlerQueueEntry)
{
m_ThreadSafeCounter = threadSafeCounter;
CrawlerQueueEntry = crawlerQueueEntry;
}
private void EndDownload(CrawlStep crawlStep, PropertyBag propertyBag, Exception exception,
ThreadSafeCounter.ThreadSafeCounterCookie counterCookie)
{
using (counterCookie)
{
if (exception != null)
{
OnDownloadException(exception, crawlStep);
} else if (!propertyBag.IsNull())
{
propertyBag.Referrer = crawlStep;
// Assign initial properties to propertybag
if (!counterCookie.CrawlerQueueEntry.Properties.IsNull())
{
counterCookie.CrawlerQueueEntry.Properties.
ForEach(key => propertyBag[key.Key].Value = key.Value);
}
if (OnAfterDownload(crawlStep, propertyBag))
{
// Executes all the pipelines sequentially for each downloaded content
// in the crawl process. Used to extract data from content, like which
// url's to follow, email addresses, aso.
Pipeline.ForEach(pipelineStep => ExecutePipeLineStep(pipelineStep, propertyBag));
}
}
}
ProcessQueue();
}