public static Dictionary <string, Int32> ReadCateFile()
{
var path = "D:\\tb\\淘宝网 - 淘！我喜欢.html";
var text = FileHelper.ReadBinaryFile(path);
var pattern = "<a.+?href=\"(.+?)\".*>(.+)</a>";
// var catStr = "cat=(\\d{1,})[&|\"]";
Dictionary <string, Int32> dict = new Dictionary <string, Int32>(100);
foreach (Match match in Regex.Matches(text, pattern, RegexOptions.IgnoreCase))
{
//var groups = Regex.Matches(match.Groups[1].Value, catStr, RegexOptions.IgnoreCase);
//if (groups.HasValue())
//{
// var cat = groups[0].Groups[1].Value;
if (dict.ContainsKey(match.Groups[2].Value) == false)
{
dict[match.Groups[2].Value] = 0;
}
//}
}
var temp = dict.OrderBy(d => d.Value);
foreach (var item in temp)
{
ItemCate tempItem = new ItemCate()
{
CateId = item.Value, CateName = item.Key, Status = 1
};
ItemCateBLL.Insert(tempItem);
Console.WriteLine(item.Key + ":" + item.Value);
}
Console.WriteLine("Total:" + temp.Count());
return(dict);
}
public static void InitCate()
{
List <ItemCate> cates = new List <ItemCate>()
{
new ItemCate()
{
CateId = 16, CateName = "女装"
},
new ItemCate()
{
CateId = 50344007, CateName = "男装"
},
new ItemCate()
{
CateId = 1625, CateName = "内衣"
},
new ItemCate()
{
CateId = 50016853, CateName = "鞋靴"
},
new ItemCate()
{
CateId = 50006842, CateName = "箱包"
},
new ItemCate()
{
CateId = 50010404, CateName = "配件"
},
new ItemCate()
{
CateId = 0, CateName = "童装玩具"
},
new ItemCate()
{
CateId = 0, CateName = "孕产"
},
new ItemCate()
{
CateId = 0, CateName = "用品"
},
new ItemCate()
{
CateId = 0, CateName = "家电"
},
new ItemCate()
{
CateId = 0, CateName = "数码"
},
new ItemCate()
{
CateId = 0, CateName = "手机"
},
new ItemCate()
{
CateId = 0, CateName = "美妆"
},
new ItemCate()
{
CateId = 1801, CateName = "洗护"
},
new ItemCate()
{
CateId = 0, CateName = "保健品"
},
new ItemCate()
{
CateId = 50015926, CateName = "珠宝"
},
new ItemCate()
{
CateId = 50015926, CateName = "眼镜"
},
new ItemCate()
{
CateId = 0, CateName = "手表"
},
new ItemCate()
{
CateId = 0, CateName = "运动"
},
new ItemCate()
{
CateId = 0, CateName = "户外"
},
new ItemCate()
{
CateId = 0, CateName = "乐器"
},
new ItemCate()
{
CateId = 0, CateName = "游戏"
},
new ItemCate()
{
CateId = 0, CateName = "动漫"
},
new ItemCate()
{
CateId = 0, CateName = "影视"
},
new ItemCate()
{
CateId = 0, CateName = "美食"
},
new ItemCate()
{
CateId = 0, CateName = "生鲜"
},
new ItemCate()
{
CateId = 0, CateName = "零食"
},
new ItemCate()
{
CateId = 0, CateName = "鲜花"
},
new ItemCate()
{
CateId = 0, CateName = "宠物"
},
new ItemCate()
{
CateId = 0, CateName = "农资"
},
new ItemCate()
{
CateId = 0, CateName = "工具"
},
new ItemCate()
{
CateId = 50097129, CateName = "装修"
},
new ItemCate()
{
CateId = 0, CateName = "建材"
},
new ItemCate()
{
CateId = 0, CateName = "家具"
},
new ItemCate()
{
CateId = 50008163, CateName = "家饰"
},
new ItemCate()
{
CateId = 50008163, CateName = "家纺"
},
new ItemCate()
{
CateId = 56974003, CateName = "汽车"
},
new ItemCate()
{
CateId = 56974003, CateName = "二手车"
},
new ItemCate()
{
CateId = 0, CateName = "用品"
},
new ItemCate()
{
CateId = 0, CateName = "办公"
},
new ItemCate()
{
CateId = 0, CateName = "DIY"
},
new ItemCate()
{
CateId = 0, CateName = "五金电子"
},
new ItemCate()
{
CateId = 0, CateName = "百货"
},
new ItemCate()
{
CateId = 0, CateName = "餐厨"
},
new ItemCate()
{
CateId = 0, CateName = "家庭保健"
},
new ItemCate()
{
CateId = 0, CateName = "学习"
},
new ItemCate()
{
CateId = 0, CateName = "卡券"
},
new ItemCate()
{
CateId = 50097750, CateName = "本地服务"
},
};
foreach (var item in cates)
{
item.Status = 1;
ItemCateBLL.Insert(item);
}
}
/// <summary>
/// 按关键词导入，每个关键词只能查询100条
/// </summary>
public static void ImportAllTaobaoke()
{
var dt = DateTime.Now;
PageInfo page = new PageInfo();
page.PageSize = 100;
var cateItems = ItemCateBLL.GetData();
foreach (var item in cateItems)
{
if (item.LastUpdateTime > dt.AddDays(-10))
{
continue;
}
Func <string, object> fun = (keyword) =>
{
try
{
List <Items> updateItems = new List <Items>(200);
List <Items> addItems = new List <Items>(200);
var items = ItemsBLL.GetByTag(keyword);
page.PageIndex = 1;
var result = QueryCoupon(item.CateName);
//Timeout timeout = new Timeout();
//timeout.Do = QueryCoupon(item.CateName);
if (result == null || result.Count() == 0)
{
return(null);
}
foreach (var item2 in result)
{
var temp = items.FirstOrDefault(a => a.NumIid == item2.NumIid);
if (temp != null)
{
item2.Id = temp.Id;
temp = item2;
if (updateItems.Exists(d => d.NumIid == temp.NumIid))
{
updateItems.Add(temp);
}
}
else
{
items.Add(item2);
addItems.Add(item2);
}
}
item.LastUpdateTime = dt;
Task.Factory.StartNew(() => {
if (addItems.Count > 0)
{
ItemsBLL.BatchInsert(addItems);
}
if (updateItems.Count > 0)
{
ItemsBLL.BatchUpdate(updateItems);
}
ItemCateBLL.Update(item);
});
}
catch (Exception ex)
{
Logger.WriteErrorLog(ex.Message);
}
return(null);
};
FuncTimeout.EventNeedRun action = delegate(object[] param)
{
//调用自定义函数
return(fun(param[0].ToString()));
};
FuncTimeout ft = new FuncTimeout(action, 1000 * 60 * 2);//超时时间2分钟
ft.doAction(item.CateName);
Thread.Sleep(5000);
}
try
{
//ItemCateBLL.BatchUpdate(cateItems);
ItemsBLL.ClearSameData();
Console.WriteLine("Done:" + DateTime.Now);
}
catch (Exception ex)
{
Logger.WriteErrorLog(ex.Message);
}
//TaskParallelHelper.ExecuteTask(actionItem);
}
