async Task <IEnumerable <string> > GetLinksFromContent(CrawlingTask task)
{
await GetRandomDelay();
if (_contentEmulation.ContainsKey(task.UrlToCrawl))
{
return(_contentEmulation[task.UrlToCrawl]);
}
return(null);
}
async System.Threading.Tasks.Task Crawl(ConcurrentBag <CrawlingTask> bag, string crawlerName)
{
CrawlingTask task;
while (bag.TryTake(out task))
{
IEnumerable <string> urls = await GetLinksFromContent(task);
if (urls != null)
{
foreach (var url in urls)
{
var t = new CrawlingTask()
{
UrlToCrawl = url,
ProducerName = crawlerName
};
bag.Add(t);
}
}
Console.WriteLine($"Indexing url {task.UrlToCrawl} posted by {task.ProducerName} is comp {crawlerName}");
}
}