private void AnalysicProduct(string urlCurrent, GABIZ.Base.HtmlAgilityPack.HtmlDocument doc)
{
if (IsDetailUrl(urlCurrent, _detailLinkRegexs))
{
if (_company.Status == Common.CompanyStatus.TIN)
{
Product product = new Product();
product.Analytics(doc, urlCurrent, _config, false, _company.Domain);
}
else
{
var pt = new Product();
pt.Analytics(doc, urlCurrent, _config, false, _company.Domain);
if (pt.IsSuccessData(this._config.CheckPrice))
{
pt.Valid = false;
if (!IsExistsProduct(pt.ID))
{
if (!this._hsHashDuplicate.ContainsKey(pt.GetHashCheckDuplicate()))
{
_totalProductBefore++;
_productAdapter.InsertProduct(pt);
_redisLastUpdateProduct.UpdateBathLastUpdateProduct(this._companyId, new List <long> {
pt.ID
}, DateTime.Now);
_cacheProductHash.SetCacheProductHash(_companyId, new List <QT.Entities.CrawlerProduct.Cache.ProductHash>
{
new ProductHash()
{
HashChange = pt.GetHashChange(),
HashDuplicate = pt.GetHashCheckDuplicate(),
Id = pt.ID,
Price = pt.Price,
url = pt.DetailUrl,
HashImage = pt.GetHashImage(),
}
});
//_productAdapter.PushMQChangeImage(new List<long> { pt.ID });
_mqLogChangePrice.PushQueueChangePriceLog(
new JobRabbitChangePrice()
{
Name = pt.Name,
OldPrice = 0,
NewPrice = pt.Price,
ProductID = pt.ID,
CompanyID = pt.IDCongTy
});
AddToDuplicate(pt.GetHashCheckDuplicate(), pt.ID);
}
else
{
_log.Info("Duplicate data");
}
}
}
}
}
}
public void Analysic()
{
try
{
long productID = 0;
ProductAdapter productAdapter = new ProductAdapter(sqldb);
Configuration configXPath = new Configuration(companyID);
QT.Entities.Company company = new Entities.Company(companyID);
DataTable tblProduct = sqldb.GetTblData("Select ID,DetailUrl From Product Where Company = @CompanyID", CommandType.Text,
new SqlParameter[] {
SqlDb.CreateParamteterSQL("@CompanyID", companyID, SqlDbType.BigInt)
}, null, true);
foreach (DataRow rowInfo in tblProduct.Rows)
{
productID = QT.Entities.Common.Obj2Int64(rowInfo["ID"]);
if (!DicDetailUrl.ContainsKey(QT.Entities.Common.Obj2String(rowInfo["DetailUrl"])))
{
sqldb.RunQuery("update Product set Valid = 0 where Company = @CompanyID and ID = @productID", CommandType.Text, new SqlParameter[] {
sqldb.CreateParamteter("@CompanyID", companyID, SqlDbType.BigInt),
sqldb.CreateParamteter("@productID", productID, SqlDbType.BigInt)
});
}
}
foreach (var DetailUrl in DicDetailUrl)
{
string strDetailUrl = DetailUrl.Key.ToString();
productID = QT.Entities.Common.GetIDProduct(strDetailUrl);
if (this.bDeleteProductData)
{
sqldb.RunQuery("delete product where id = @id", CommandType.Text, new SqlParameter[] {
SqlDb.CreateParamteterSQL("@id", productID, SqlDbType.BigInt)
});
}
Product pt = new Product();
GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(GetHtmlCode(strDetailUrl, false));
pt.Analytics(doc, strDetailUrl, configXPath, false, company.Domain, null);
if (pt.IsSuccessData(this.config.CheckPrice))
{
productAdapter.InsertProduct(pt);
lstProductIDChangeImage.Add(productID);
this.Invoke(new Action(() =>
{
richTextBox1.AppendText("\r\nSuccess link: " + strDetailUrl);
}));
}
else
{
this.Invoke(new Action(() =>
{
richTextBox1.AppendText("\r\nNo product link: " + strDetailUrl);
}));
}
}
}
catch (Exception ex)
{
this.Invoke(new Action(() =>
{
MessageBox.Show(ex.Message + ex.StackTrace);
}));
}
MessageBox.Show(string.Format("Crawler {0} \nSuccess Link {1} \nFail link {2}", DicDetailUrl.Count, lstProductIDChangeImage.Count, (DicDetailUrl.Count - lstProductIDChangeImage.Count)));
}
public override void ProcessMessage(BasicDeliverEventArgs message)
{
try
{
var pt = ProductEntity.GetFromJson(message.Body);
if (pt.StatusChange.IsDelete)
{
string sql = string.Format(@"select top 1 p.DetailUrl, p.Name, p.Price, p.CategoryID, p.LastUpdate, p.ImagePath, p.ImageUrls, p.ImageId, p.ID
from Product p
where p.ID = {0} " , pt.ID);
DataTable tbl = _productAdapter.GetSqlDb().GetTblData(sql, CommandType.Text, null);
if (tbl.Rows.Count > 0 && _productAdapter.DeleteProduct(pt.ID))
{
_log.Info("Deleted Success product: " + pt.ID + pt.DetailUrl);
var row1 = tbl.Rows[0];
string imgId = Common.Obj2String(row1["ImageId"]);
var objBackUp = new JobBackupProductToDel()
{
Id = Common.Obj2Int64(row1["ID"]),
Price = Common.Obj2Int64(row1["Price"]),
ImageId = imgId,
ImageUrl = Common.Obj2String(row1["ImageUrls"]),
Name = Common.Obj2String(row1["Name"]),
ProductUrl = Common.Obj2String("DetailUrl")
};
RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>()
{
pt.ID
});
if (!string.IsNullOrEmpty(imgId))
{
_producerDelImgImbo.PublishString(imgId);
}
_producerLogDelProduct.PublishString(objBackUp.ToJson());
}
}
else if (pt.StatusChange.IsDuplicate)
{
if (_productAdapter.DeleteProduct(pt.ID))
{
_log.Info("Deleted duplicate: " + pt.ID + pt.DetailUrl);
RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>()
{
pt.ID
});
}
}
else if (pt.StatusChange.IsNew)
{
if (_productAdapter.InsertProduct(pt))
{
_jobClient.PublishJob(new Websosanh.Core.JobServer.Job()
{
Data = ImageProductInfo.GetMessage(new ImageProductInfo(pt.ID, pt.Name, pt.DetailUrl, pt.ImageUrl, true))
});
_producerLogAddProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(new JobRabbitAddProduct()
{
DateAdd = DateTime.Now,
DetailUrl = pt.DetailUrl,
IDCompnay = pt.CompanyId,
Name = pt.Name,
ProductID = pt.ID
}));
_log.Info(string.Format("Company: {0} Inserted product: {1}", pt.CompanyId, pt.ID));
}
}
else
{
if (_productAdapter.UpdateProduct(pt))
{
_log.Info(string.Format("Company: {0} Updated product: {1}", pt.CompanyId, pt.ID));
if (pt.StatusChange.IsChangeImage)
{
_jobClient.PublishJob(new Websosanh.Core.JobServer.Job()
{
Data = ImageProductInfo.GetMessage(new ImageProductInfo(pt.ID, pt.Name, pt.DetailUrl, pt.ImageUrl, false))
});
}
RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>()
{
pt.ID
});
}
}
}
catch (Exception ex)
{
_log.Error(ex.Message);
}
GetChannel().BasicAck(message.DeliveryTag, true);
}