private void button3_Click(object sender, EventArgs e)
{
ProductService.Init();//初始化表
//得到商品类别
CategoryService catesService = new CategoryService();
var list = catesService.SelectAll().Where(u => u.Levels == 3);
Stopwatch sw = new Stopwatch();
sw.Start();
List <Task> taskList = new List <Task>();
foreach (Category cg in list)
{
Task task = Task.Factory.StartNew(() =>
{
Stopwatch swShop = new Stopwatch();
swShop.Start();
LogHelper.WriteLog($"*************商品名称:{cg.Name}开始爬取*************", ConsoleColor.Blue);
int pageNum = ProductSearch.GetPageNum(cg.Url);
for (int index = 1; index <= pageNum; index++)
{
LogHelper.WriteLog($"{cg.Name} 页码{index} 总页码{pageNum}");
//判断当前是否存在优化可能
var productList = ProductSearch.GetPageDataAsync(cg.Url, index, cg.Id);
//分组，主要为了分表插入做准备
List <ProductGroupInput> groupList = productList.GroupBy(u => u.ProductId % PublicConst.SheetNum).Select(u => new ProductGroupInput
{
Id = u.Key,
ProductUnits = u.OrderBy(p => p.ProductId).ToList()
}).ToList();
ProductService.InsertGroupBulk(groupList);
}
swShop.Stop();
return(swShop.ElapsedMilliseconds);
}).ContinueWith(t =>
{
LogHelper.WriteLog($"*************商品名称:{cg.Name} 结束爬取 耗时:{t.Result / 1000 / 60}分钟*************", ConsoleColor.Blue);
});
taskList.Add(task);
if (taskList.Count > 20)
{
taskList = taskList.Where(t => !t.IsCompleted && !t.IsCanceled && !t.IsFaulted).ToList();
Task.WaitAny(taskList.ToArray());
}
}
Task.WaitAll(taskList.ToArray());
sw.Stop();
LogHelper.WriteLog($"*************商品爬取结束 耗时:{sw.ElapsedMilliseconds / 1000 / 60}分钟 {sw.ElapsedMilliseconds / 1000 / 60 / 24}小时*************");
}
