public static void Run()
{
//链接地址
string linkPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "SpiderLinks.txt");
Queue <string> linksQueue = new Queue <string>();
SysCatalogRepository SysCatalogDal = new SysCatalogRepository();
//判断文件是否存在。
if (File.Exists(linkPath))
{
foreach (var item in File.ReadAllLines(linkPath))
{
string link = item.Trim();
if (!string.IsNullOrEmpty(link))
{
linksQueue.Enqueue(link);
}
}
}
CasperJs casperJs = new CasperJs(DataReceived);
while (linksQueue.Count > 0)
{
string link = linksQueue.Dequeue();
SysCatalog entity = SysCatalogDal.FindSingle(o => o.Url == link);
if (entity == null)
{
Console.WriteLine($"执行获取分类：{link}");
casperJs.Exec(CatalogScriptPath, link);
}
}
}
public void GetCatalogProductRankList(string obj, int start, int limit, IJavascriptCallback javascriptCallback)
{
Task.Factory.StartNew(async() =>
{
using (javascriptCallback)
{
Result <List <ProductListItemViewModel> > result = new Result <List <ProductListItemViewModel> >(true);
ProductListItemQueryViewModel queryEntity = JsonConvert.DeserializeObject <ProductListItemQueryViewModel>(obj);
if (queryEntity == null)
{
result.Success = false;
result.Msg = "数据转换失败！";
}
else
{
using (DbContext db = DbContextFactory.Context)
{
//找出mapName
var dbSet = db.Set <SysCatalog>();
SysCatalog entity = dbSet.First(o => o.ID == queryEntity.CatalogID);
List <SQLiteParameter> param = new List <SQLiteParameter>()
{
new SQLiteParameter("@CatalogID", queryEntity.CatalogID),
new SQLiteParameter("@RankLevel", queryEntity.RankNumber),
new SQLiteParameter("@StartDate", queryEntity.StartDate),
new SQLiteParameter("@EndDate", queryEntity.EndDate)
};
string sql, fromSql, whereSql, totalSql;
fromSql = $"from {entity.CatalogProductTableName} map join tb_Sys_Product p on map.ProductID = p.ID ";
whereSql = "where map.CatalogID = @CatalogID ";
if (!string.IsNullOrEmpty(queryEntity.Name))
{
whereSql += $"AND p.Name LIKE @Name ";
param.Add(new SQLiteParameter("@Name", $"%{queryEntity.Name}%"));
}
whereSql += $"AND map.RankLevel = @RankLevel ";
whereSql += $"AND map.CreateTime Between @StartDate AND @EndDate ";
sql = " select p.Name,p.ImgPath,map.RankTime,map.RankLevel "
+ fromSql
+ whereSql
+ "order by map.CreateTime desc, map.RankLevel asc "
+ $"limit {limit} offset {start}; ";
totalSql = " select count(1) " + fromSql + whereSql;
SQLiteParameter[] paramArray = param.ToArray();
result.Data = db.Database.SqlQuery <ProductListItemViewModel>(sql, paramArray).ToList();
result.Total = db.Database.SqlQuery <int>(totalSql, paramArray).FirstOrDefault();
}
}
await javascriptCallback.ExecuteAsync(JsonConvert.SerializeObject(result));
//javascriptCallback.ExecuteAsync(JsonConvert.SerializeObject(result));
}
});
}
private static void DataReceived(string ouput, SysCatalog entity, SysCatalogRepository SysCatalogDal, string updateString)
{
Console.WriteLine(ouput);
if (!string.IsNullOrEmpty(ouput) &&
ouput.StartsWith("※"))
{
List <SysProduct> ProductList = JsonConvert.DeserializeObject <List <SysProduct> >(ouput.TrimStart('※'));
if (ProductList != null &&
ProductList.Count > 0)
{
Dictionary <string, string> dic = SysCatalogDal.GetCatalogProducts(entity.ID);
List <SysProduct> AddList = new List <SysProduct>();
List <CatalogProductViewModel> AddMappingList = new List <CatalogProductViewModel>();
ProductList.ForEach(l =>
{
l.ID = CommonMethods.NewGuidString;
l.CatalogID = entity.ID;
l.CreateTime = DateTime.Now;
if (!dic.ContainsKey(l.Asin))
{
AddList.Add(l);
}
else
{
l.ID = dic[l.Asin];
}
AddMappingList.Add(new CatalogProductViewModel()
{
ID = CommonMethods.NewGuidString,
CatalogID = entity.ID,
CreateTime = DateTime.Now,
ProductID = l.ID,
RankLevel = l.RankNumber,
RankTime = updateString
});
//SysCatalogDal.AddProductAndMapping(l,entity.CatalogProductTableName,updateString);
});
SysCatalogDal.AddProducts(AddList);
SysCatalogDal.AddCatalogProductMapping(entity.CatalogProductTableName, AddMappingList);
SysCatalogDal.UpdateCatalogUpdateString(entity.ID, updateString);
}
}
}
/// <summary>
/// cmd 消息接收事件
/// </summary>
/// <param name="ouput"></param>
private static void DataReceived(string ouput)
{
if (!string.IsNullOrEmpty(ouput) &&
ouput.StartsWith("※"))
{
SysCatalogRepository SysCatalogDal = new SysCatalogRepository();
SysCatalogViewModel entity = JsonConvert.DeserializeObject <SysCatalogViewModel>(ouput.TrimStart('※'));
if (entity != null)
{
List <SysCatalog> entities = new List <SysCatalog>();
string updateTime = DateTime.Now.AddDays(-5).ToString("yyyyMMdd");
entity.CatalogProductTableName = SysCatalogDal.CreateCatalogProductMapTable();
SysCatalog firstCatalog = new SysCatalog
{
ID = CommonMethods.NewGuidString,
UpdateString = updateTime,
CreateTime = DateTime.Now,
Name = entity.Name,
Url = entity.Url,
CatalogProductTableName = entity.CatalogProductTableName,
IsDel = false
};
entities.Add(firstCatalog);
entity.ChildCatalogs.ForEach(l =>
{
entities.Add(new SysCatalog
{
ID = CommonMethods.NewGuidString,
UpdateString = firstCatalog.UpdateString,
CreateTime = DateTime.Now,
Name = l.Name,
Url = l.Url,
CatalogProductTableName = firstCatalog.CatalogProductTableName,
ParentID = firstCatalog.ID,
IsDel = false
});
});
SysCatalogDal.AddRange(entities);
Console.WriteLine($"链接：{entity.Url} 分类数据添加完成！");
}
}
}
public void GetCatalogProductList(string obj, IJavascriptCallback javascriptCallback)
{
Task.Factory.StartNew(async() =>
{
using (javascriptCallback)
{
Result <List <CatalogProductListViewModel> > result = new Result <List <CatalogProductListViewModel> >(true);
try
{
ProductListItemQueryViewModel queryEntity = JsonConvert.DeserializeObject <ProductListItemQueryViewModel>(obj);
if (queryEntity == null)
{
result.Success = false;
result.Msg = "查询参数转换失败！";
}
else
{
using (DbContext db = DbContextFactory.Context)
{
//找出mapName
var dbSet = db.Set <SysCatalog>();
SysCatalog entity = dbSet.First(o => o.ID == queryEntity.CatalogID);
List <SQLiteParameter> param = new List <SQLiteParameter>()
{
new SQLiteParameter("@CatalogID", queryEntity.CatalogID)
};
string sql, whereSql, innerWhereSql;
whereSql = " WHERE p.CatalogID = @CatalogID ";
if (!string.IsNullOrEmpty(queryEntity.Name))
{
whereSql += "AND p.Name LIKE @Name ";
param.Add(new SQLiteParameter("@Name", $"%{queryEntity.Name}%"));
}
innerWhereSql = " WHERE map.CatalogID = p.CatalogID AND map.ProductID = p.ID ";
//AND map.ProductID = p.ID AND map.CreateTime BETWEEN @StartDate AND @EndDate
if (queryEntity.StartDate.HasValue && queryEntity.EndDate.HasValue)
{
innerWhereSql += " AND map.CreateTime BETWEEN @StartDate AND @EndDate ";
param.Add(new SQLiteParameter("@StartDate", queryEntity.StartDate.Value));
param.Add(new SQLiteParameter("@EndDate", queryEntity.EndDate.Value));
}
//先找出分类下的产品
sql = string.Format(@"SELECT p.Name AS 'ProductName',p.ImgPath AS 'ProductImgPath'
,(SELECT COUNT(1) FROM {0} map {2} ) AS 'TotalCount'
FROM tb_Sys_Product p {1} ", entity.CatalogProductTableName, whereSql, innerWhereSql);
List <CatalogProductListViewModel> list = db.Database.SqlQuery <CatalogProductListViewModel>(sql, param.ToArray()).OrderByDescending(o => o.TotalCount).ToList();
list.RemoveAll(o => o.TotalCount <= 0);
result.Data = list;
result.Total = list.Sum(o => o.TotalCount);
}
}
}
catch (Exception ex)
{
result.Success = false;
result.Msg = ex.Message;
}
await javascriptCallback.ExecuteAsync(JsonConvert.SerializeObject(result));
}
});
}
