protected void RunConsumer(CancellationTokenSource cancellationToken)
{
foreach (Domain domain in _domainsToCrawl.GetConsumingEnumerable())
{
DomainCrawlResult domainCrawlResult = null;
try
{
domainCrawlResult = _domainConsumer.Consume(domain, cancellationToken);
}
catch (Exception e)
{
_logger.ErrorFormat("Domain consumer [{0}] threw exception during Consume().", _domainConsumer.ToString());
_logger.Error(e);
Stop();
}
if (domainCrawlResult != null)
{
lock (_crawledDomains)
{
_crawledDomains.Add(domainCrawlResult);
}
}
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.Token.ThrowIfCancellationRequested();
}
}
}
protected void RunConsumers(CancellationTokenSource cancellationToken)
{
ManualResetEvent manualReset = new ManualResetEvent(true);
Object locker = new Object();
ParallelOptions options = new ParallelOptions
{
CancellationToken = cancellationToken.Token,
MaxDegreeOfParallelism = (_config.MaxConcurrentCrawls > 0) ? _config.MaxConcurrentCrawls : System.Environment.ProcessorCount
};
Parallel.ForEach(_domainsToCrawl.GetConsumingEnumerable(), options, domain =>
{
if (cancellationToken.IsCancellationRequested)
{
cancellationToken.Token.ThrowIfCancellationRequested();
}
DomainCrawlResult domainCrawlResult = null;
try
{
domainCrawlResult = _domainConsumer.Consume(domain, cancellationToken);
}
catch (Exception e)
{
_logger.ErrorFormat("Domain consumer [{0}] threw exception during Consume().", _domainConsumer.ToString());
_logger.Error(e);
Stop();
}
if (domainCrawlResult != null)
{
lock (_crawledDomains)
{
_crawledDomains.Add(domainCrawlResult);
}
}
});
}