public static async Task <IEnumerable <string> > GetLinkFormatContent(CrawlingTask crawlingTask) { await GetRandomDleay(); if (dict.ContainsKey(crawlingTask.UrlToCrowl)) { return(dict[crawlingTask.UrlToCrowl]); } return(null); }
public static async Task crowl(ConcurrentBag <CrawlingTask> crawlingTasks, string crawlName) { CrawlingTask task; while (crawlingTasks.TryTake(out task)) { IEnumerable <string> urls = await GetLinkFormatContent(task); if (urls != null) { foreach (var item in urls) { var t = new CrawlingTask() { UrlToCrowl = item, ProducterName = crawlName }; crawlingTasks.Add(t); } } Console.WriteLine($"网页{task.UrlToCrowl}URL被放入队列{task.ProducterName},在{crawlName}消费者被完成"); } }