private void AnalysicProduct(string urlCurrent, GABIZ.Base.HtmlAgilityPack.HtmlDocument doc) { if (IsDetailUrl(urlCurrent, _detailLinkRegexs)) { if (_company.Status == Common.CompanyStatus.TIN) { Product product = new Product(); product.Analytics(doc, urlCurrent, _config, false, _company.Domain); } else { var pt = new Product(); pt.Analytics(doc, urlCurrent, _config, false, _company.Domain); if (pt.IsSuccessData(this._config.CheckPrice)) { pt.Valid = false; if (!IsExistsProduct(pt.ID)) { if (!this._hsHashDuplicate.ContainsKey(pt.GetHashCheckDuplicate())) { _totalProductBefore++; _productAdapter.InsertProduct(pt); _redisLastUpdateProduct.UpdateBathLastUpdateProduct(this._companyId, new List <long> { pt.ID }, DateTime.Now); _cacheProductHash.SetCacheProductHash(_companyId, new List <QT.Entities.CrawlerProduct.Cache.ProductHash> { new ProductHash() { HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashCheckDuplicate(), Id = pt.ID, Price = pt.Price, url = pt.DetailUrl, HashImage = pt.GetHashImage(), } }); //_productAdapter.PushMQChangeImage(new List<long> { pt.ID }); _mqLogChangePrice.PushQueueChangePriceLog( new JobRabbitChangePrice() { Name = pt.Name, OldPrice = 0, NewPrice = pt.Price, ProductID = pt.ID, CompanyID = pt.IDCongTy }); AddToDuplicate(pt.GetHashCheckDuplicate(), pt.ID); } else { _log.Info("Duplicate data"); } } } } } }
public void Analysic() { try { long productID = 0; ProductAdapter productAdapter = new ProductAdapter(sqldb); Configuration configXPath = new Configuration(companyID); QT.Entities.Company company = new Entities.Company(companyID); DataTable tblProduct = sqldb.GetTblData("Select ID,DetailUrl From Product Where Company = @CompanyID", CommandType.Text, new SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyID", companyID, SqlDbType.BigInt) }, null, true); foreach (DataRow rowInfo in tblProduct.Rows) { productID = QT.Entities.Common.Obj2Int64(rowInfo["ID"]); if (!DicDetailUrl.ContainsKey(QT.Entities.Common.Obj2String(rowInfo["DetailUrl"]))) { sqldb.RunQuery("update Product set Valid = 0 where Company = @CompanyID and ID = @productID", CommandType.Text, new SqlParameter[] { sqldb.CreateParamteter("@CompanyID", companyID, SqlDbType.BigInt), sqldb.CreateParamteter("@productID", productID, SqlDbType.BigInt) }); } } foreach (var DetailUrl in DicDetailUrl) { string strDetailUrl = DetailUrl.Key.ToString(); productID = QT.Entities.Common.GetIDProduct(strDetailUrl); if (this.bDeleteProductData) { sqldb.RunQuery("delete product where id = @id", CommandType.Text, new SqlParameter[] { SqlDb.CreateParamteterSQL("@id", productID, SqlDbType.BigInt) }); } Product pt = new Product(); GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(GetHtmlCode(strDetailUrl, false)); pt.Analytics(doc, strDetailUrl, configXPath, false, company.Domain, null); if (pt.IsSuccessData(this.config.CheckPrice)) { productAdapter.InsertProduct(pt); lstProductIDChangeImage.Add(productID); this.Invoke(new Action(() => { richTextBox1.AppendText("\r\nSuccess link: " + strDetailUrl); })); } else { this.Invoke(new Action(() => { richTextBox1.AppendText("\r\nNo product link: " + strDetailUrl); })); } } } catch (Exception ex) { this.Invoke(new Action(() => { MessageBox.Show(ex.Message + ex.StackTrace); })); } MessageBox.Show(string.Format("Crawler {0} \nSuccess Link {1} \nFail link {2}", DicDetailUrl.Count, lstProductIDChangeImage.Count, (DicDetailUrl.Count - lstProductIDChangeImage.Count))); }
public override void ProcessMessage(BasicDeliverEventArgs message) { try { var pt = ProductEntity.GetFromJson(message.Body); if (pt.StatusChange.IsDelete) { string sql = string.Format(@"select top 1 p.DetailUrl, p.Name, p.Price, p.CategoryID, p.LastUpdate, p.ImagePath, p.ImageUrls, p.ImageId, p.ID from Product p where p.ID = {0} " , pt.ID); DataTable tbl = _productAdapter.GetSqlDb().GetTblData(sql, CommandType.Text, null); if (tbl.Rows.Count > 0 && _productAdapter.DeleteProduct(pt.ID)) { _log.Info("Deleted Success product: " + pt.ID + pt.DetailUrl); var row1 = tbl.Rows[0]; string imgId = Common.Obj2String(row1["ImageId"]); var objBackUp = new JobBackupProductToDel() { Id = Common.Obj2Int64(row1["ID"]), Price = Common.Obj2Int64(row1["Price"]), ImageId = imgId, ImageUrl = Common.Obj2String(row1["ImageUrls"]), Name = Common.Obj2String(row1["Name"]), ProductUrl = Common.Obj2String("DetailUrl") }; RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>() { pt.ID }); if (!string.IsNullOrEmpty(imgId)) { _producerDelImgImbo.PublishString(imgId); } _producerLogDelProduct.PublishString(objBackUp.ToJson()); } } else if (pt.StatusChange.IsDuplicate) { if (_productAdapter.DeleteProduct(pt.ID)) { _log.Info("Deleted duplicate: " + pt.ID + pt.DetailUrl); RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>() { pt.ID }); } } else if (pt.StatusChange.IsNew) { if (_productAdapter.InsertProduct(pt)) { _jobClient.PublishJob(new Websosanh.Core.JobServer.Job() { Data = ImageProductInfo.GetMessage(new ImageProductInfo(pt.ID, pt.Name, pt.DetailUrl, pt.ImageUrl, true)) }); _producerLogAddProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(new JobRabbitAddProduct() { DateAdd = DateTime.Now, DetailUrl = pt.DetailUrl, IDCompnay = pt.CompanyId, Name = pt.Name, ProductID = pt.ID })); _log.Info(string.Format("Company: {0} Inserted product: {1}", pt.CompanyId, pt.ID)); } } else { if (_productAdapter.UpdateProduct(pt)) { _log.Info(string.Format("Company: {0} Updated product: {1}", pt.CompanyId, pt.ID)); if (pt.StatusChange.IsChangeImage) { _jobClient.PublishJob(new Websosanh.Core.JobServer.Job() { Data = ImageProductInfo.GetMessage(new ImageProductInfo(pt.ID, pt.Name, pt.DetailUrl, pt.ImageUrl, false)) }); } RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>() { pt.ID }); } } } catch (Exception ex) { _log.Error(ex.Message); } GetChannel().BasicAck(message.DeliveryTag, true); }