public static void Run()
{
//链接地址
string linkPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "SpiderLinks.txt");
Queue <string> linksQueue = new Queue <string>();
SysCatalogRepository SysCatalogDal = new SysCatalogRepository();
//判断文件是否存在。
if (File.Exists(linkPath))
{
foreach (var item in File.ReadAllLines(linkPath))
{
string link = item.Trim();
if (!string.IsNullOrEmpty(link))
{
linksQueue.Enqueue(link);
}
}
}
CasperJs casperJs = new CasperJs(DataReceived);
while (linksQueue.Count > 0)
{
string link = linksQueue.Dequeue();
SysCatalog entity = SysCatalogDal.FindSingle(o => o.Url == link);
if (entity == null)
{
Console.WriteLine($"执行获取分类：{link}");
casperJs.Exec(CatalogScriptPath, link);
}
}
}
public static void Run()
{
string updateString = DateTime.Now.ToString("yyyyMMdd");
SysCatalogRepository SysCatalogDal = new SysCatalogRepository();
List <SysCatalog> SysCatalogList = SysCatalogDal.Find(o => !o.IsDel && o.UpdateString != updateString);
while (SysCatalogList.Count > 0)
{
_run(SysCatalogList, updateString);
System.Threading.Thread.Sleep(500);
SysCatalogList = SysCatalogDal.Find(o => o.UpdateString != updateString);
}
//TaskExecute<SysCatalog> tasks = new TaskExecute<SysCatalog>(o =>
//{
// Console.WriteLine($"获取分类：{o.Url}");
// new CasperJs(s =>
// {
// DataReceived(s, o, SysCatalogDal, updateString);
// }).Exec(ScriptPath, o.Url);
//},5);
//SysCatalogList.ForEach(l =>
//{
// tasks.AddQueue(l);
// //Console.WriteLine($"获取分类：{l.Url}");
// //new CasperJs(o =>
// //{
// // DataReceived(o, l, SysCatalogDal, updateString);
// //}).Exec(ScriptPath, l.Url);
//});
//tasks.Run();
}
private static void DataReceived(string ouput, SysCatalog entity, SysCatalogRepository SysCatalogDal, string updateString)
{
Console.WriteLine(ouput);
if (!string.IsNullOrEmpty(ouput) &&
ouput.StartsWith("※"))
{
List <SysProduct> ProductList = JsonConvert.DeserializeObject <List <SysProduct> >(ouput.TrimStart('※'));
if (ProductList != null &&
ProductList.Count > 0)
{
Dictionary <string, string> dic = SysCatalogDal.GetCatalogProducts(entity.ID);
List <SysProduct> AddList = new List <SysProduct>();
List <CatalogProductViewModel> AddMappingList = new List <CatalogProductViewModel>();
ProductList.ForEach(l =>
{
l.ID = CommonMethods.NewGuidString;
l.CatalogID = entity.ID;
l.CreateTime = DateTime.Now;
if (!dic.ContainsKey(l.Asin))
{
AddList.Add(l);
}
else
{
l.ID = dic[l.Asin];
}
AddMappingList.Add(new CatalogProductViewModel()
{
ID = CommonMethods.NewGuidString,
CatalogID = entity.ID,
CreateTime = DateTime.Now,
ProductID = l.ID,
RankLevel = l.RankNumber,
RankTime = updateString
});
//SysCatalogDal.AddProductAndMapping(l,entity.CatalogProductTableName,updateString);
});
SysCatalogDal.AddProducts(AddList);
SysCatalogDal.AddCatalogProductMapping(entity.CatalogProductTableName, AddMappingList);
SysCatalogDal.UpdateCatalogUpdateString(entity.ID, updateString);
}
}
}
/// <summary>
/// cmd 消息接收事件
/// </summary>
/// <param name="ouput"></param>
private static void DataReceived(string ouput)
{
if (!string.IsNullOrEmpty(ouput) &&
ouput.StartsWith("※"))
{
SysCatalogRepository SysCatalogDal = new SysCatalogRepository();
SysCatalogViewModel entity = JsonConvert.DeserializeObject <SysCatalogViewModel>(ouput.TrimStart('※'));
if (entity != null)
{
List <SysCatalog> entities = new List <SysCatalog>();
string updateTime = DateTime.Now.AddDays(-5).ToString("yyyyMMdd");
entity.CatalogProductTableName = SysCatalogDal.CreateCatalogProductMapTable();
SysCatalog firstCatalog = new SysCatalog
{
ID = CommonMethods.NewGuidString,
UpdateString = updateTime,
CreateTime = DateTime.Now,
Name = entity.Name,
Url = entity.Url,
CatalogProductTableName = entity.CatalogProductTableName,
IsDel = false
};
entities.Add(firstCatalog);
entity.ChildCatalogs.ForEach(l =>
{
entities.Add(new SysCatalog
{
ID = CommonMethods.NewGuidString,
UpdateString = firstCatalog.UpdateString,
CreateTime = DateTime.Now,
Name = l.Name,
Url = l.Url,
CatalogProductTableName = firstCatalog.CatalogProductTableName,
ParentID = firstCatalog.ID,
IsDel = false
});
});
SysCatalogDal.AddRange(entities);
Console.WriteLine($"链接：{entity.Url} 分类数据添加完成！");
}
}
}
public void GetSysCatalogList(IJavascriptCallback javascriptCallback)
{
Task.Factory.StartNew(async() =>
{
using (javascriptCallback)
{
SysCatalogRepository dal = new SysCatalogRepository();
List <SysCatalog> list = dal.Find(o => !o.IsDel);
List <SysCatalog> parentList = list.Where(o => o.ParentID == null).ToList();
var response = parentList.ConvertAll(o => new
{
Name = o.Name,
ID = o.ID,
SubMenuList = list.Where(l => l.ParentID == o.ID).Select(l => new
{
l.Name,
l.ID
}).ToList()
});
await javascriptCallback.ExecuteAsync(JsonConvert.SerializeObject(response));
}
});
}
