private void SaveUpdate() { lock (_productWaitUpdateGroup) { if (_productWaitUpdateGroup.Count > 0) { var lstProductIdUpdate = new List <long>(); var lstUpdateToChacheHash = new List <ProductHash>(); foreach (var product in _productWaitUpdateGroup) { lstProductIdUpdate.Add(product.ID); lstUpdateToChacheHash.Add(new ProductHash() { HashChange = product.GetHashChange(), HashDuplicate = product.GetHashCheckDuplicate(), HashImage = product.GetHashImage(), Id = product.ID, Price = product.Price, url = product.DetailUrl }); } _productAdapter.UpdateProductsChangeToDb(_productWaitUpdateGroup); _redisProductHash.SetCacheProductHash(_companyId, lstUpdateToChacheHash); _cacheCheckDelete.SetBathScore(_companyId, lstProductIdUpdate, 0); _log.Info(string.Format("Save changed {0} product", _productWaitUpdateGroup.Count)); _productWaitUpdateGroup.Clear(); } } }
private void AnalysicProduct(string urlCurrent, GABIZ.Base.HtmlAgilityPack.HtmlDocument doc) { if (IsDetailUrl(urlCurrent, _detailLinkRegexs)) { if (_company.Status == Common.CompanyStatus.TIN) { Product product = new Product(); product.Analytics(doc, urlCurrent, _config, false, _company.Domain); } else { var pt = new Product(); pt.Analytics(doc, urlCurrent, _config, false, _company.Domain); if (pt.IsSuccessData(this._config.CheckPrice)) { pt.Valid = false; if (!IsExistsProduct(pt.ID)) { if (!this._hsHashDuplicate.ContainsKey(pt.GetHashCheckDuplicate())) { _totalProductBefore++; _productAdapter.InsertProduct(pt); _redisLastUpdateProduct.UpdateBathLastUpdateProduct(this._companyId, new List <long> { pt.ID }, DateTime.Now); _cacheProductHash.SetCacheProductHash(_companyId, new List <QT.Entities.CrawlerProduct.Cache.ProductHash> { new ProductHash() { HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashCheckDuplicate(), Id = pt.ID, Price = pt.Price, url = pt.DetailUrl, HashImage = pt.GetHashImage(), } }); //_productAdapter.PushMQChangeImage(new List<long> { pt.ID }); _mqLogChangePrice.PushQueueChangePriceLog( new JobRabbitChangePrice() { Name = pt.Name, OldPrice = 0, NewPrice = pt.Price, ProductID = pt.ID, CompanyID = pt.IDCongTy }); AddToDuplicate(pt.GetHashCheckDuplicate(), pt.ID); } else { _log.Info("Duplicate data"); } } } } } }
private static void ResetCacheProduct(long companyID) { CacheProductHash cashProductHash = CacheProductHash.Instance(); RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance(); List <ProductHash> lst = new List <ProductHash>(); List <long> lstLastUpdate = new List <long>(); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); Company cmp = new Company(companyID); productAdapter.DeleteProductUnvalidOfCOmpany(companyID); DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID); foreach (DataRow rowProduct in tbl.Rows) { long ProductID = Common.Obj2Int64(rowProduct["ID"]); string Name = rowProduct["Name"].ToString(); long Price = Common.Obj2Int64(rowProduct["Price"]); string ImageUrl = Convert.ToString(rowProduct["ImageUrls"]); string DetailUrl = Convert.ToString(rowProduct["DetailUrl"]); int InStock = Common.Obj2Int(rowProduct["InStock"]); bool Valid = Common.Obj2Bool(rowProduct["Valid"]); string shortDescription = Common.CellToString(rowProduct["ShortDescription"], ""); bool IsDeal = Common.Obj2Bool(rowProduct["IsDeal"]); long CategoryID = rowProduct["ClassificationID"] == DBNull.Value ? 0 : Common.Obj2Int64(rowProduct["ClassificationID"]); long HashChange = Product.GetHashChangeInfo(InStock, Valid, Price, Name, ImageUrl, CategoryID, shortDescription); long HashDuplicate = Product.GetHashCheckDuplicate(cmp.Domain, Price, Name, ImageUrl); long HashImage = Product.GetHashImageInfo(ImageUrl); lst.Add(new ProductHash() { HashChange = HashChange, HashDuplicate = HashDuplicate, HashImage = HashImage, Id = ProductID, Price = Price, url = DetailUrl }); lstLastUpdate.Add(ProductID); } cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID); cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1)); productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count); lst.Clear(); lstLastUpdate.Clear(); }
public void Run(System.Threading.CancellationToken token) { ProductAdapter productAdapter = new ProductAdapter(new SqlDb("Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200")); log.InfoFormat("Start run at {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture)); CacheProductHash cashProductHash = CacheProductHash.Instance(); RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance(); int countProduct = 0; try { var lstFn = productAdapter.GetAllCompanyIdCrawlerFindNew(); var lstRl = productAdapter.GetAllCompanyIdCrawlerReload(); RedisCompanyWaitCrawler redisCache = RedisCompanyWaitCrawler.Instance(); redisCache.SyncCompanyFindNew(lstFn); redisCache.SyncCompanyReload(lstRl); } catch (Exception ex) { log.Error(ex); } var lst = new List <QT.Entities.CrawlerProduct.Cache.ProductHash>(); var lstLastUpdate = new List <long>(); var lstCompany = productAdapter.GetAllCompanyIdCrawler(); foreach (var companyID in lstCompany) { Company cmp = new Company(companyID); productAdapter.DeleteProductUnvalidOfCOmpany(companyID); DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID); foreach (DataRow rowProduct in tbl.Rows) { long productId = QT.Entities.Common.Obj2Int64(rowProduct["ID"]); long originPrice = QT.Entities.Common.Obj2Int64(rowProduct["OriginPrice"]); string name = rowProduct["Name"].ToString(); long price = QT.Entities.Common.Obj2Int64(rowProduct["Price"]); string imageUrl = Convert.ToString(rowProduct["ImageUrls"]); string detailUrl = Convert.ToString(rowProduct["DetailUrl"]); int inStock = QT.Entities.Common.Obj2Int(rowProduct["InStock"]); bool valid = QT.Entities.Common.Obj2Bool(rowProduct["Valid"]); string shortDescription = QT.Entities.Common.CellToString(rowProduct["ShortDescription"], ""); long categoryId = rowProduct["ClassificationID"] == DBNull.Value ? 0 : QT.Entities.Common.Obj2Int64(rowProduct["ClassificationID"]); long hashChange = ProductEntity.GetHashChangeInfo(inStock, valid, price, name, imageUrl, categoryId, shortDescription, originPrice); long hashDuplicate = Product.GetHashDuplicate(cmp.Domain, price, name, imageUrl); long hashImage = Product.GetHashImageInfo(imageUrl); lst.Add(new QT.Entities.CrawlerProduct.Cache.ProductHash() { HashChange = hashChange, HashDuplicate = hashDuplicate, HashImage = hashImage, Id = productId, Price = price, url = detailUrl }); lstLastUpdate.Add(productId); } cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID); cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1)); productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count); lst.Clear(); lstLastUpdate.Clear(); log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count)); } log.Info("Complete all company"); NextRun = DateTime.Now.AddHours(MAX_HOUR_LOOP); log.InfoFormat("End at {0}", DateTime.Now.ToString()); }
public override void ProcessMessage(BasicDeliverEventArgs message) { try { var pt = ProductEntity.GetFromJson(message.Body); _producerClassification.Publish(new Classification() { CompanyId = pt.CompanyId, Id = pt.ID, Category = pt.GetCategoryString() }.GetArbyteJson()); if (pt.StatusChange.IsNew) { _chLastProduct.UpdateBathLastUpdateProduct(pt.CompanyId, new List <long> { pt.ID }, DateTime.Now); _chProductHash.SetCacheProductHash(pt.CompanyId, new ProductHash() { url = pt.DetailUrl, Id = pt.ID, Price = pt.Price, HashImage = pt.GetHashImage(), HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashDuplicate() }); } else if (pt.StatusChange.IsDelete) { _chLastProduct.RemoveProduct(pt.CompanyId, pt.ID); _chProductHash.RemoveProduct(pt.CompanyId, pt.ID); } else if (pt.StatusChange.IsDuplicate) { _chLastProduct.RemoveProduct(pt.CompanyId, pt.ID); _chProductHash.RemoveProduct(pt.CompanyId, pt.CompanyId); } else if (pt.StatusChange.IsChangeBasic) { _chLastProduct.UpdateBathLastUpdateProduct(pt.CompanyId, new List <long> { pt.ID }, DateTime.Now); _chProductHash.SetCacheProductHash(pt.CompanyId, new ProductHash() { url = pt.DetailUrl, Id = pt.ID, Price = pt.Price, HashImage = pt.GetHashImage(), HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashDuplicate() }); } if (pt.StatusChange.IsChangeDesc) { var jobDescription = new JobMqChangeDesc() { Id = pt.ID, FullDesc = pt.FullDescHtml, ShortDesc = pt.ShortDescHtml, SpecDesc = pt.SpecsDescHtml, VideoDesc = pt.VideoDescHtml }; _producerDescriptionQuang.Publish(UtilZipFile.Zip(jobDescription.GetJSON())); _producerChangeDesc.Publish(UtilZipFile.Zip(jobDescription.GetJSON())); _chProductDesciptioHash.SetHashDesc(pt.CompanyId, new Tuple <long, long>(pt.ID, pt.GetHashDesc())); } if (pt.StatusChange.IsChangePrice) { _jobPushChangePrice.PushQueueChangePriceLog(new JobRabbitChangePrice() { Name = pt.Name, ProductID = pt.ID, CompanyID = pt.CompanyId, NewPrice = pt.Price, OldPrice = pt.OldPrice }); } _log.Info(string.Format("Saved for pt: {0}", pt.ID)); } catch (Exception ex01) { _log.Error(ex01); } this.GetChannel().BasicAck(message.DeliveryTag, true); }