public static Dictionary <string, Int32> ReadCateFile() { var path = "D:\\tb\\淘宝网 - 淘!我喜欢.html"; var text = FileHelper.ReadBinaryFile(path); var pattern = "<a.+?href=\"(.+?)\".*>(.+)</a>"; // var catStr = "cat=(\\d{1,})[&|\"]"; Dictionary <string, Int32> dict = new Dictionary <string, Int32>(100); foreach (Match match in Regex.Matches(text, pattern, RegexOptions.IgnoreCase)) { //var groups = Regex.Matches(match.Groups[1].Value, catStr, RegexOptions.IgnoreCase); //if (groups.HasValue()) //{ // var cat = groups[0].Groups[1].Value; if (dict.ContainsKey(match.Groups[2].Value) == false) { dict[match.Groups[2].Value] = 0; } //} } var temp = dict.OrderBy(d => d.Value); foreach (var item in temp) { ItemCate tempItem = new ItemCate() { CateId = item.Value, CateName = item.Key, Status = 1 }; ItemCateBLL.Insert(tempItem); Console.WriteLine(item.Key + ":" + item.Value); } Console.WriteLine("Total:" + temp.Count()); return(dict); }
public static void InitCate() { List <ItemCate> cates = new List <ItemCate>() { new ItemCate() { CateId = 16, CateName = "女装" }, new ItemCate() { CateId = 50344007, CateName = "男装" }, new ItemCate() { CateId = 1625, CateName = "内衣" }, new ItemCate() { CateId = 50016853, CateName = "鞋靴" }, new ItemCate() { CateId = 50006842, CateName = "箱包" }, new ItemCate() { CateId = 50010404, CateName = "配件" }, new ItemCate() { CateId = 0, CateName = "童装玩具" }, new ItemCate() { CateId = 0, CateName = "孕产" }, new ItemCate() { CateId = 0, CateName = "用品" }, new ItemCate() { CateId = 0, CateName = "家电" }, new ItemCate() { CateId = 0, CateName = "数码" }, new ItemCate() { CateId = 0, CateName = "手机" }, new ItemCate() { CateId = 0, CateName = "美妆" }, new ItemCate() { CateId = 1801, CateName = "洗护" }, new ItemCate() { CateId = 0, CateName = "保健品" }, new ItemCate() { CateId = 50015926, CateName = "珠宝" }, new ItemCate() { CateId = 50015926, CateName = "眼镜" }, new ItemCate() { CateId = 0, CateName = "手表" }, new ItemCate() { CateId = 0, CateName = "运动" }, new ItemCate() { CateId = 0, CateName = "户外" }, new ItemCate() { CateId = 0, CateName = "乐器" }, new ItemCate() { CateId = 0, CateName = "游戏" }, new ItemCate() { CateId = 0, CateName = "动漫" }, new ItemCate() { CateId = 0, CateName = "影视" }, new ItemCate() { CateId = 0, CateName = "美食" }, new ItemCate() { CateId = 0, CateName = "生鲜" }, new ItemCate() { CateId = 0, CateName = "零食" }, new ItemCate() { CateId = 0, CateName = "鲜花" }, new ItemCate() { CateId = 0, CateName = "宠物" }, new ItemCate() { CateId = 0, CateName = "农资" }, new ItemCate() { CateId = 0, CateName = "工具" }, new ItemCate() { CateId = 50097129, CateName = "装修" }, new ItemCate() { CateId = 0, CateName = "建材" }, new ItemCate() { CateId = 0, CateName = "家具" }, new ItemCate() { CateId = 50008163, CateName = "家饰" }, new ItemCate() { CateId = 50008163, CateName = "家纺" }, new ItemCate() { CateId = 56974003, CateName = "汽车" }, new ItemCate() { CateId = 56974003, CateName = "二手车" }, new ItemCate() { CateId = 0, CateName = "用品" }, new ItemCate() { CateId = 0, CateName = "办公" }, new ItemCate() { CateId = 0, CateName = "DIY" }, new ItemCate() { CateId = 0, CateName = "五金电子" }, new ItemCate() { CateId = 0, CateName = "百货" }, new ItemCate() { CateId = 0, CateName = "餐厨" }, new ItemCate() { CateId = 0, CateName = "家庭保健" }, new ItemCate() { CateId = 0, CateName = "学习" }, new ItemCate() { CateId = 0, CateName = "卡券" }, new ItemCate() { CateId = 50097750, CateName = "本地服务" }, }; foreach (var item in cates) { item.Status = 1; ItemCateBLL.Insert(item); } }
/// <summary> /// 按关键词导入,每个关键词只能查询100条 /// </summary> public static void ImportAllTaobaoke() { var dt = DateTime.Now; PageInfo page = new PageInfo(); page.PageSize = 100; var cateItems = ItemCateBLL.GetData(); foreach (var item in cateItems) { if (item.LastUpdateTime > dt.AddDays(-10)) { continue; } Func <string, object> fun = (keyword) => { try { List <Items> updateItems = new List <Items>(200); List <Items> addItems = new List <Items>(200); var items = ItemsBLL.GetByTag(keyword); page.PageIndex = 1; var result = QueryCoupon(item.CateName); //Timeout timeout = new Timeout(); //timeout.Do = QueryCoupon(item.CateName); if (result == null || result.Count() == 0) { return(null); } foreach (var item2 in result) { var temp = items.FirstOrDefault(a => a.NumIid == item2.NumIid); if (temp != null) { item2.Id = temp.Id; temp = item2; if (updateItems.Exists(d => d.NumIid == temp.NumIid)) { updateItems.Add(temp); } } else { items.Add(item2); addItems.Add(item2); } } item.LastUpdateTime = dt; Task.Factory.StartNew(() => { if (addItems.Count > 0) { ItemsBLL.BatchInsert(addItems); } if (updateItems.Count > 0) { ItemsBLL.BatchUpdate(updateItems); } ItemCateBLL.Update(item); }); } catch (Exception ex) { Logger.WriteErrorLog(ex.Message); } return(null); }; FuncTimeout.EventNeedRun action = delegate(object[] param) { //调用自定义函数 return(fun(param[0].ToString())); }; FuncTimeout ft = new FuncTimeout(action, 1000 * 60 * 2);//超时时间2分钟 ft.doAction(item.CateName); Thread.Sleep(5000); } try { //ItemCateBLL.BatchUpdate(cateItems); ItemsBLL.ClearSameData(); Console.WriteLine("Done:" + DateTime.Now); } catch (Exception ex) { Logger.WriteErrorLog(ex.Message); } //TaskParallelHelper.ExecuteTask(actionItem); }