public void UpdateProinfo(SiteProInfo pro)
{
try
{
var fquery = Query.EQ("_id", pro.SiteSkuId);
var update = Update.Set("SpPrice", (double) pro.FloorPrice);
update.Set("SpName", pro.SpName);
update.Set("UpdateTime", pro.UpdateTime);
update.Set("FloorPrice", (double)pro.FloorPrice);
update.Set("ClassId", pro.ClassId);
update.Set("ProUrl", pro.ProUrl);
update.Set("CommenCount", pro.CommenCount);
update.Set("Promotions", pro.Promotions ?? "");
update.Set("SellType", pro.SellType);
update.Set("ShopName", pro.ShopName ?? "");
update.Set("SiteCat", pro.SiteCat);
update.Set("SiteProId", pro.SiteProId ?? "");
Collection.Update(fquery, update);
}
catch (Exception ex)
{
LogServer.WriteLog(ex, "DBError");
}
//Collection.FindAndModify(fquery,new IMongoSortBy(), new SiteProInfo { SpPrice = pro.SpPrice });
}
public void AddHissiteproinfo(SiteProInfo pro)
{
try
{
Collection.Insert(pro);
}
catch (Exception ex)
{
LogServer.WriteLog(ex, "DBError");
}
}
public void AddSiteProInfo(SiteProInfo pro)
{
try
{
Collection.Save(pro);
}
catch (Exception ex)
{
LogServer.WriteLog(ex, "DBError");
}
}
public void AddSiteProInfo(SiteProInfo pro)
{
try
{
using (var db = _dbFactory.OpenDbConnection())
{
db.Insert(pro);
}
}
catch (Exception ex)
{
LogServer.WriteLog(ex, "DBError");
}
}
public void AddHissiteproinfo(SiteProInfo pro)
{
new HisSiteProInfoDb().AddHissiteproinfo(pro);
}
private void SaveProduct(IProList proListManager)
{
string pageUrl = proListManager.getListUrl();
//var paNum = proListManager.Reginfo.PageStart + proListManager.Reginfo.PageStep;
//string tempPageUrl = pageUrl.Replace("$page", paNum.ToString());
//string pageHtml = proListManager.DownLoadPage(tempPageUrl, proListManager.AreaInfoId);
//if (string.IsNullOrEmpty(pageHtml))
//{
// if (proListManager.ErrorInfo != null)
// new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
// return;
//}
int maxPage = 1;
List<SiteProInfo> list = new List<SiteProInfo>();
SiteProInfoBll probll = new SiteProInfoBll();
for (int i = 0; i < maxPage; i++)
{
if (Quitout)
break;
int paNum = proListManager.Reginfo.PageStart + proListManager.Reginfo.PageStep * i;
string tempPageUrl = pageUrl.Replace("$page", paNum.ToString());
Stopwatch downLoadtime = new Stopwatch();
downLoadtime.Start();
string pageHtml = proListManager.DownLoadPage(tempPageUrl, proListManager.AreaInfoId);
downLoadtime.Stop();
if (string.IsNullOrEmpty(pageHtml))
{
if (proListManager.ErrorInfo != null)
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
continue;
}
if (i == 0)
{
maxPage = proListManager.getPageNum(pageHtml);
if (!proListManager.HasProducts(pageHtml))
{
if (proListManager.ErrorInfo != null)
{
proListManager.ErrorInfo.UrlPath = pageUrl;
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
return;
}
}
}
string content = string.IsNullOrEmpty(proListManager.Reginfo.ListsReg) ? pageHtml : proListManager.GetProConent(pageHtml);
MatchCollection mcsingle = proListManager.GetSigleProduct(content);
if(mcsingle== null)
{
MessageCenter.ShowBox("[线程" + Thread.CurrentThread.Name + "]" + proListManager.SiteName + " " + proListManager.CatInfo.CatName + " 第" + (i + 1) + "页 单个产品匹配错误" , 2);
continue;
}
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
continue;
}
int tempUpdate = 0;
int temperror = 0;
int tempadd = 0;
Stopwatch detialtime = new Stopwatch();
detialtime.Start();
foreach (Match item in mcsingle)
{
SiteProInfo pro = new SiteProInfo();
string singlePro = item.ToString();
try
{
pro = GetProduct(singlePro, proListManager);
}
catch (Exception ex)
{
LogServer.WriteLog(ex);
continue;
}
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
temperror++;
continue;
}
if (pro == null)
{
temperror++;
continue;
}
var oldpro = probll.FindOne(pro.SiteSkuId);
if (oldpro != null)
{
if (pro.SpPrice != oldpro.SpPrice || pro.SpName != oldpro.SpName)
{
new HisSiteProInfoBll().AddHissiteproinfo(pro);
}
if (oldpro.FloorPrice < pro.FloorPrice)
{
pro.FloorPrice = oldpro.FloorPrice;
}
pro.CreateDate = oldpro.CreateDate;
probll.UpdateProinfo(pro);
tempUpdate++;
}
else
{
pro.ClassId = proListManager.CatInfo.Id;
//detial
getProDetial(pro, proListManager);
list.Add(pro);
new HisSiteProInfoBll().AddHissiteproinfo(pro);
tempadd++;
}
}
detialtime.Stop();
probll.addSiteProinfo(list);
list.Clear();
MessageCenter.ShowBox("[线程" + Thread.CurrentThread.Name + "]" + proListManager.SiteName + " " + proListManager.CatInfo.CatName + " 第" + (i + 1) + "/" + maxPage + "页" + tempUpdate + "/" + tempadd + "/" + temperror + " t1:" + downLoadtime.Elapsed.TotalSeconds.ToString("0") + "s " + "t2:" + detialtime.Elapsed.TotalSeconds.ToString("0") + "s", 2);
}
if (list.Count == 0)
return;
}
private SiteProInfo GetProduct(string singleHtml,IProList proListManager)
{
SiteProInfo item = new SiteProInfo();
item.ProUrl = proListManager.GetSpUrl(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SiteSkuId = proListManager.GetItemSku(item.ProUrl);
item.SpName = proListManager.GetSpName(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SpPrice = proListManager.GetSpPrice(singleHtml, item.SiteSkuId);
if (item.SpPrice <= 0)
return null;
item.FloorPrice = item.SpPrice;
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.smallPic = proListManager.GetSmallPic(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.CommenCount = proListManager.GetComments(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SiteCat = proListManager.SiteCatInfo.ClassId;
item.SiteId = proListManager.SiteCatInfo.SiteId;
item.CreateDate = DateTime.Now;
item.UpdateTime = DateTime.Now;
return item;
}
private void getProDetial(SiteProInfo pro, IProList proListManager)
{
string detial = proListManager.DownLoadPage(pro.ProUrl, proListManager.AreaInfoId);
pro.SellType = proListManager.GetSellType(detial);
pro.BigPic = proListManager.GetBigPic(detial);
pro.spBrand = proListManager.GetBrand(detial);
pro.spSkuDes = proListManager.GetSkuDes(detial);
pro.Otherpic = proListManager.GetOtherpic(detial);
pro.ShopName = proListManager.GetShopName(detial);
}
private void AddProInfo(List<ItemEF> proList)
{
if (proList == null || proList.Count == 0)
return;
List<SiteProInfo> list = new List<SiteProInfo>();
foreach (ItemEF item in proList)
{
string smallpic = "";
string otherpic = "";
if (item.picture_url == null)
item.picture_url = "";
var pictures = item.picture_url.Split(',');
if (pictures.Length > 0 && item.picture_url != "")
{
string bigpic = pictures[0];
smallpic = pictures[0].Replace("_800", "_160");
otherpic = item.picture_url.Replace(bigpic, "").TrimStart(',');
}
SiteProInfo tempPro = new SiteProInfo
{
SpName=item.sku_name,
SpPrice = item.sale_price==0?item.list_price:item.sale_price,
ProUrl = item.product_url,
smallPic = smallpic,
Promotions = item.promo_desc,
spBrand = item.brand,
SingleDesc=item.service_desc,
IsSell=item.stock_status,
SiteCat = item.category_id,
CommenCount=0,
SiteProId=item.product_id,
CommentUrl=item.product_url,
BigPic = item.picture_url,
Otherpic = otherpic,
spSkuDes="",
SellType=item.operating_model=="1"?1:2,
ClassId = 0,
SiteId = Baseinfo.SiteId,
SiteSkuId = Baseinfo.SiteId + "|" + item.sku_id,
FloorPrice = item.sale_price == 0 ? item.list_price : item.sale_price,
UpdateTime=DateTime.Now,
CreateDate =DateTime.Now
};
list.Add(tempPro);
}
new SiteProInfoDB().AddSiteProInfo(list);
}
public void UpdateProinfo(SiteProInfo pro)
{
try
{
using (var db = _dbFactory.OpenDbConnection())
{
db.UpdateNonDefaults(new SiteProInfo
{
SpPrice = pro.FloorPrice,
SpName=pro.SpName,
UpdateTime=pro.UpdateTime,
FloorPrice=pro.FloorPrice,
ClassId=pro.ClassId,
ProUrl=pro.ProUrl,
CommenCount=pro.CommenCount,
Promotions=pro.Promotions,
SellType=pro.SellType,
ShopName=pro.ShopName,
SiteCat=pro.SiteCat,
SiteProId = pro.SiteProId ?? ""
}, p => p.SiteSkuId == pro.SiteSkuId);
}
}
catch (Exception ex)
{
LogServer.WriteLog(ex, "DBError");
}
}
private TempSiteProInfo Tempchangobj(SiteProInfo pro)
{
TempSiteProInfo temp = new TempSiteProInfo
{
AreaPriceDetial = pro.AreaPriceDetial,
BigPic = pro.BigPic,
ClassId = pro.ClassId,
BrandId = pro.BrandId,
SiteId = pro.SiteId,
SingleDesc = pro.SingleDesc,
ShopName = pro.ShopName,
SellType = pro.SellType,
SearchFiles = pro.SearchFiles,
smallPic = pro.smallPic,
spBrand = pro.spBrand,
spSkuDes = pro.spSkuDes,
SearchSkuJson = pro.SearchSkuJson,
SiteCat = pro.SiteCat,
SiteProId = pro.SiteProId,
SiteSkuId = pro.SiteSkuId,
SpName = pro.SpName,
SpPrice = pro.SpPrice,
IsSell = pro.IsSell,
QzSort = pro.QzSort,
CommenCount = pro.CommenCount,
CommentUrl = pro.CommentUrl,
CreateDate = pro.CreateDate,
FloorPrice = pro.FloorPrice,
Otherpic = pro.Otherpic,
Promotions = pro.Promotions,
ProUrl = pro.ProUrl,
UpdateTime = pro.UpdateTime
};
return temp;
}
