private void button3_Click(object sender, EventArgs e) { ProductService.Init();//初始化表 //得到商品类别 CategoryService catesService = new CategoryService(); var list = catesService.SelectAll().Where(u => u.Levels == 3); Stopwatch sw = new Stopwatch(); sw.Start(); List <Task> taskList = new List <Task>(); foreach (Category cg in list) { Task task = Task.Factory.StartNew(() => { Stopwatch swShop = new Stopwatch(); swShop.Start(); LogHelper.WriteLog($"*************商品名称:{cg.Name}开始爬取*************", ConsoleColor.Blue); int pageNum = ProductSearch.GetPageNum(cg.Url); for (int index = 1; index <= pageNum; index++) { LogHelper.WriteLog($"{cg.Name} 页码{index} 总页码{pageNum}"); //判断当前是否存在优化可能 var productList = ProductSearch.GetPageDataAsync(cg.Url, index, cg.Id); //分组,主要为了分表插入做准备 List <ProductGroupInput> groupList = productList.GroupBy(u => u.ProductId % PublicConst.SheetNum).Select(u => new ProductGroupInput { Id = u.Key, ProductUnits = u.OrderBy(p => p.ProductId).ToList() }).ToList(); ProductService.InsertGroupBulk(groupList); } swShop.Stop(); return(swShop.ElapsedMilliseconds); }).ContinueWith(t => { LogHelper.WriteLog($"*************商品名称:{cg.Name} 结束爬取 耗时:{t.Result / 1000 / 60}分钟*************", ConsoleColor.Blue); }); taskList.Add(task); if (taskList.Count > 20) { taskList = taskList.Where(t => !t.IsCompleted && !t.IsCanceled && !t.IsFaulted).ToList(); Task.WaitAny(taskList.ToArray()); } } Task.WaitAll(taskList.ToArray()); sw.Stop(); LogHelper.WriteLog($"*************商品爬取结束 耗时:{sw.ElapsedMilliseconds / 1000 / 60}分钟 {sw.ElapsedMilliseconds / 1000 / 60 / 24}小时*************"); }