private static async Task <IEnumerable <string> > GetLinksFromContentAsync(CrawlingTask task) { await GetRandomDelay(); if (_contentEmulation.ContainsKey(task.UrlToCrawl)) { return(_contentEmulation[task.UrlToCrawl]); } return(null); }
private static async Task CrawlAsync(ConcurrentBag <CrawlingTask> bag, string crawlerName) { CrawlingTask task; while (bag.TryTake(out task)) { IEnumerable <string> urls = await GetLinksFromContentAsync(task); if (urls != null) { foreach (var url in urls) { var t = new CrawlingTask(url, crawlerName); bag.Add(t); } } WriteLine($"Indexing url {task.UrlToCrawl} posted by {task.ProducerName} is completed by {crawlerName}"); } }