示例#1
0
        async Task <IEnumerable <string> > GetLinksFromContent(CrawlingTask task)
        {
            await GetRandomDelay();

            if (_contentEmulation.ContainsKey(task.UrlToCrawl))
            {
                return(_contentEmulation[task.UrlToCrawl]);
            }
            return(null);
        }
示例#2
0
        async System.Threading.Tasks.Task Crawl(ConcurrentBag <CrawlingTask> bag, string crawlerName)
        {
            CrawlingTask task;

            while (bag.TryTake(out task))
            {
                IEnumerable <string> urls = await GetLinksFromContent(task);

                if (urls != null)
                {
                    foreach (var url in urls)
                    {
                        var t = new CrawlingTask()
                        {
                            UrlToCrawl   = url,
                            ProducerName = crawlerName
                        };
                        bag.Add(t);
                    }
                }
                Console.WriteLine($"Indexing url {task.UrlToCrawl} posted by {task.ProducerName} is comp {crawlerName}");
            }
        }