public static async Task <IEnumerable <string> > GetLinkFormatContent(CrawlingTask crawlingTask)
{
await GetRandomDleay();
if (dict.ContainsKey(crawlingTask.UrlToCrowl))
{
return(dict[crawlingTask.UrlToCrowl]);
}
return(null);
}
public static async Task crowl(ConcurrentBag <CrawlingTask> crawlingTasks, string crawlName)
{
CrawlingTask task;
while (crawlingTasks.TryTake(out task))
{
IEnumerable <string> urls = await GetLinkFormatContent(task);
if (urls != null)
{
foreach (var item in urls)
{
var t = new CrawlingTask()
{
UrlToCrowl = item,
ProducterName = crawlName
};
crawlingTasks.Add(t);
}
}
Console.WriteLine($"网页{task.UrlToCrowl}URL被放入队列{task.ProducterName},在{crawlName}消费者被完成");
}
}
