private SiteProInfo GetProduct(string singleHtml,IProList proListManager)
{
SiteProInfo item = new SiteProInfo();
item.ProUrl = proListManager.GetSpUrl(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SiteSkuId = proListManager.GetItemSku(item.ProUrl);
item.SpName = proListManager.GetSpName(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SpPrice = proListManager.GetSpPrice(singleHtml, item.SiteSkuId);
if (item.SpPrice <= 0)
return null;
item.FloorPrice = item.SpPrice;
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.smallPic = proListManager.GetSmallPic(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.CommenCount = proListManager.GetComments(singleHtml);
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
return null;
}
item.SiteCat = proListManager.SiteCatInfo.ClassId;
item.SiteId = proListManager.SiteCatInfo.SiteId;
item.CreateDate = DateTime.Now;
item.UpdateTime = DateTime.Now;
return item;
}
private void SaveProduct(IProList proListManager)
{
string pageUrl = proListManager.getListUrl();
//var paNum = proListManager.Reginfo.PageStart + proListManager.Reginfo.PageStep;
//string tempPageUrl = pageUrl.Replace("$page", paNum.ToString());
//string pageHtml = proListManager.DownLoadPage(tempPageUrl, proListManager.AreaInfoId);
//if (string.IsNullOrEmpty(pageHtml))
//{
// if (proListManager.ErrorInfo != null)
// new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
// return;
//}
int maxPage = 1;
List<SiteProInfo> list = new List<SiteProInfo>();
SiteProInfoBll probll = new SiteProInfoBll();
for (int i = 0; i < maxPage; i++)
{
if (Quitout)
break;
int paNum = proListManager.Reginfo.PageStart + proListManager.Reginfo.PageStep * i;
string tempPageUrl = pageUrl.Replace("$page", paNum.ToString());
Stopwatch downLoadtime = new Stopwatch();
downLoadtime.Start();
string pageHtml = proListManager.DownLoadPage(tempPageUrl, proListManager.AreaInfoId);
downLoadtime.Stop();
if (string.IsNullOrEmpty(pageHtml))
{
if (proListManager.ErrorInfo != null)
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
continue;
}
if (i == 0)
{
maxPage = proListManager.getPageNum(pageHtml);
if (!proListManager.HasProducts(pageHtml))
{
if (proListManager.ErrorInfo != null)
{
proListManager.ErrorInfo.UrlPath = pageUrl;
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
return;
}
}
}
string content = string.IsNullOrEmpty(proListManager.Reginfo.ListsReg) ? pageHtml : proListManager.GetProConent(pageHtml);
MatchCollection mcsingle = proListManager.GetSigleProduct(content);
if(mcsingle== null)
{
MessageCenter.ShowBox("[线程" + Thread.CurrentThread.Name + "]" + proListManager.SiteName + " " + proListManager.CatInfo.CatName + " 第" + (i + 1) + "页 单个产品匹配错误" , 2);
continue;
}
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
continue;
}
int tempUpdate = 0;
int temperror = 0;
int tempadd = 0;
Stopwatch detialtime = new Stopwatch();
detialtime.Start();
foreach (Match item in mcsingle)
{
SiteProInfo pro = new SiteProInfo();
string singlePro = item.ToString();
try
{
pro = GetProduct(singlePro, proListManager);
}
catch (Exception ex)
{
LogServer.WriteLog(ex);
continue;
}
if (proListManager.ErrorInfo != null)
{
new SaveSpiderErrorBll().SaveSpiderError(proListManager.ErrorInfo);
proListManager.ErrorInfo = null;
temperror++;
continue;
}
if (pro == null)
{
temperror++;
continue;
}
var oldpro = probll.FindOne(pro.SiteSkuId);
if (oldpro != null)
{
if (pro.SpPrice != oldpro.SpPrice || pro.SpName != oldpro.SpName)
{
new HisSiteProInfoBll().AddHissiteproinfo(pro);
}
if (oldpro.FloorPrice < pro.FloorPrice)
{
pro.FloorPrice = oldpro.FloorPrice;
}
pro.CreateDate = oldpro.CreateDate;
probll.UpdateProinfo(pro);
tempUpdate++;
}
else
{
pro.ClassId = proListManager.CatInfo.Id;
//detial
getProDetial(pro, proListManager);
list.Add(pro);
new HisSiteProInfoBll().AddHissiteproinfo(pro);
tempadd++;
}
}
detialtime.Stop();
probll.addSiteProinfo(list);
list.Clear();
MessageCenter.ShowBox("[线程" + Thread.CurrentThread.Name + "]" + proListManager.SiteName + " " + proListManager.CatInfo.CatName + " 第" + (i + 1) + "/" + maxPage + "页" + tempUpdate + "/" + tempadd + "/" + temperror + " t1:" + downLoadtime.Elapsed.TotalSeconds.ToString("0") + "s " + "t2:" + detialtime.Elapsed.TotalSeconds.ToString("0") + "s", 2);
}
if (list.Count == 0)
return;
}
private void getProDetial(SiteProInfo pro, IProList proListManager)
{
string detial = proListManager.DownLoadPage(pro.ProUrl, proListManager.AreaInfoId);
pro.SellType = proListManager.GetSellType(detial);
pro.BigPic = proListManager.GetBigPic(detial);
pro.spBrand = proListManager.GetBrand(detial);
pro.spSkuDes = proListManager.GetSkuDes(detial);
pro.Otherpic = proListManager.GetOtherpic(detial);
pro.ShopName = proListManager.GetShopName(detial);
}
