private void SaveUpdate()
{
lock (_productWaitUpdateGroup)
{
if (_productWaitUpdateGroup.Count > 0)
{
var lstProductIdUpdate = new List <long>();
var lstUpdateToChacheHash = new List <ProductHash>();
foreach (var product in _productWaitUpdateGroup)
{
lstProductIdUpdate.Add(product.ID);
lstUpdateToChacheHash.Add(new ProductHash()
{
HashChange = product.GetHashChange(),
HashDuplicate = product.GetHashCheckDuplicate(),
HashImage = product.GetHashImage(),
Id = product.ID,
Price = product.Price,
url = product.DetailUrl
});
}
_productAdapter.UpdateProductsChangeToDb(_productWaitUpdateGroup);
_redisProductHash.SetCacheProductHash(_companyId, lstUpdateToChacheHash);
_cacheCheckDelete.SetBathScore(_companyId, lstProductIdUpdate, 0);
_log.Info(string.Format("Save changed {0} product", _productWaitUpdateGroup.Count));
_productWaitUpdateGroup.Clear();
}
}
}
private void AnalysicProduct(string urlCurrent, GABIZ.Base.HtmlAgilityPack.HtmlDocument doc)
{
if (IsDetailUrl(urlCurrent, _detailLinkRegexs))
{
if (_company.Status == Common.CompanyStatus.TIN)
{
Product product = new Product();
product.Analytics(doc, urlCurrent, _config, false, _company.Domain);
}
else
{
var pt = new Product();
pt.Analytics(doc, urlCurrent, _config, false, _company.Domain);
if (pt.IsSuccessData(this._config.CheckPrice))
{
pt.Valid = false;
if (!IsExistsProduct(pt.ID))
{
if (!this._hsHashDuplicate.ContainsKey(pt.GetHashCheckDuplicate()))
{
_totalProductBefore++;
_productAdapter.InsertProduct(pt);
_redisLastUpdateProduct.UpdateBathLastUpdateProduct(this._companyId, new List <long> {
pt.ID
}, DateTime.Now);
_cacheProductHash.SetCacheProductHash(_companyId, new List <QT.Entities.CrawlerProduct.Cache.ProductHash>
{
new ProductHash()
{
HashChange = pt.GetHashChange(),
HashDuplicate = pt.GetHashCheckDuplicate(),
Id = pt.ID,
Price = pt.Price,
url = pt.DetailUrl,
HashImage = pt.GetHashImage(),
}
});
//_productAdapter.PushMQChangeImage(new List<long> { pt.ID });
_mqLogChangePrice.PushQueueChangePriceLog(
new JobRabbitChangePrice()
{
Name = pt.Name,
OldPrice = 0,
NewPrice = pt.Price,
ProductID = pt.ID,
CompanyID = pt.IDCongTy
});
AddToDuplicate(pt.GetHashCheckDuplicate(), pt.ID);
}
else
{
_log.Info("Duplicate data");
}
}
}
}
}
}
private static void ResetCacheProduct(long companyID)
{
CacheProductHash cashProductHash = CacheProductHash.Instance();
RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance();
List <ProductHash> lst = new List <ProductHash>();
List <long> lstLastUpdate = new List <long>();
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
Company cmp = new Company(companyID);
productAdapter.DeleteProductUnvalidOfCOmpany(companyID);
DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID);
foreach (DataRow rowProduct in tbl.Rows)
{
long ProductID = Common.Obj2Int64(rowProduct["ID"]);
string Name = rowProduct["Name"].ToString();
long Price = Common.Obj2Int64(rowProduct["Price"]);
string ImageUrl = Convert.ToString(rowProduct["ImageUrls"]);
string DetailUrl = Convert.ToString(rowProduct["DetailUrl"]);
int InStock = Common.Obj2Int(rowProduct["InStock"]);
bool Valid = Common.Obj2Bool(rowProduct["Valid"]);
string shortDescription = Common.CellToString(rowProduct["ShortDescription"], "");
bool IsDeal = Common.Obj2Bool(rowProduct["IsDeal"]);
long CategoryID = rowProduct["ClassificationID"] == DBNull.Value ? 0 : Common.Obj2Int64(rowProduct["ClassificationID"]);
long HashChange = Product.GetHashChangeInfo(InStock, Valid, Price, Name, ImageUrl, CategoryID, shortDescription);
long HashDuplicate = Product.GetHashCheckDuplicate(cmp.Domain, Price, Name, ImageUrl);
long HashImage = Product.GetHashImageInfo(ImageUrl);
lst.Add(new ProductHash()
{
HashChange = HashChange,
HashDuplicate = HashDuplicate,
HashImage = HashImage,
Id = ProductID,
Price = Price,
url = DetailUrl
});
lstLastUpdate.Add(ProductID);
}
cashProductHash.SetCacheProductHash(companyID, lst, 100);
cacheLastUpdateProduct.RemoveAllLstProduct(companyID);
cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1));
productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count);
lst.Clear();
lstLastUpdate.Clear();
}
public void Run(System.Threading.CancellationToken token)
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb("Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"));
log.InfoFormat("Start run at {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture));
CacheProductHash cashProductHash = CacheProductHash.Instance();
RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance();
int countProduct = 0;
try
{
var lstFn = productAdapter.GetAllCompanyIdCrawlerFindNew();
var lstRl = productAdapter.GetAllCompanyIdCrawlerReload();
RedisCompanyWaitCrawler redisCache = RedisCompanyWaitCrawler.Instance();
redisCache.SyncCompanyFindNew(lstFn);
redisCache.SyncCompanyReload(lstRl);
}
catch (Exception ex)
{
log.Error(ex);
}
var lst = new List <QT.Entities.CrawlerProduct.Cache.ProductHash>();
var lstLastUpdate = new List <long>();
var lstCompany = productAdapter.GetAllCompanyIdCrawler();
foreach (var companyID in lstCompany)
{
Company cmp = new Company(companyID);
productAdapter.DeleteProductUnvalidOfCOmpany(companyID);
DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID);
foreach (DataRow rowProduct in tbl.Rows)
{
long productId = QT.Entities.Common.Obj2Int64(rowProduct["ID"]);
long originPrice = QT.Entities.Common.Obj2Int64(rowProduct["OriginPrice"]);
string name = rowProduct["Name"].ToString();
long price = QT.Entities.Common.Obj2Int64(rowProduct["Price"]);
string imageUrl = Convert.ToString(rowProduct["ImageUrls"]);
string detailUrl = Convert.ToString(rowProduct["DetailUrl"]);
int inStock = QT.Entities.Common.Obj2Int(rowProduct["InStock"]);
bool valid = QT.Entities.Common.Obj2Bool(rowProduct["Valid"]);
string shortDescription = QT.Entities.Common.CellToString(rowProduct["ShortDescription"], "");
long categoryId = rowProduct["ClassificationID"] == DBNull.Value ? 0 : QT.Entities.Common.Obj2Int64(rowProduct["ClassificationID"]);
long hashChange = ProductEntity.GetHashChangeInfo(inStock, valid, price, name, imageUrl, categoryId, shortDescription, originPrice);
long hashDuplicate = Product.GetHashDuplicate(cmp.Domain, price, name, imageUrl);
long hashImage = Product.GetHashImageInfo(imageUrl);
lst.Add(new QT.Entities.CrawlerProduct.Cache.ProductHash()
{
HashChange = hashChange,
HashDuplicate = hashDuplicate,
HashImage = hashImage,
Id = productId,
Price = price,
url = detailUrl
});
lstLastUpdate.Add(productId);
}
cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID);
cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1));
productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count);
lst.Clear();
lstLastUpdate.Clear();
log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count));
}
log.Info("Complete all company");
NextRun = DateTime.Now.AddHours(MAX_HOUR_LOOP);
log.InfoFormat("End at {0}", DateTime.Now.ToString());
}
public override void ProcessMessage(BasicDeliverEventArgs message)
{
try
{
var pt = ProductEntity.GetFromJson(message.Body);
_producerClassification.Publish(new Classification()
{
CompanyId = pt.CompanyId, Id = pt.ID, Category = pt.GetCategoryString()
}.GetArbyteJson());
if (pt.StatusChange.IsNew)
{
_chLastProduct.UpdateBathLastUpdateProduct(pt.CompanyId, new List <long> {
pt.ID
}, DateTime.Now);
_chProductHash.SetCacheProductHash(pt.CompanyId,
new ProductHash()
{
url = pt.DetailUrl, Id = pt.ID, Price = pt.Price, HashImage = pt.GetHashImage(), HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashDuplicate()
});
}
else if (pt.StatusChange.IsDelete)
{
_chLastProduct.RemoveProduct(pt.CompanyId, pt.ID);
_chProductHash.RemoveProduct(pt.CompanyId, pt.ID);
}
else if (pt.StatusChange.IsDuplicate)
{
_chLastProduct.RemoveProduct(pt.CompanyId, pt.ID);
_chProductHash.RemoveProduct(pt.CompanyId, pt.CompanyId);
}
else if (pt.StatusChange.IsChangeBasic)
{
_chLastProduct.UpdateBathLastUpdateProduct(pt.CompanyId, new List <long> {
pt.ID
}, DateTime.Now);
_chProductHash.SetCacheProductHash(pt.CompanyId,
new ProductHash()
{
url = pt.DetailUrl, Id = pt.ID, Price = pt.Price, HashImage = pt.GetHashImage(), HashChange = pt.GetHashChange(), HashDuplicate = pt.GetHashDuplicate()
});
}
if (pt.StatusChange.IsChangeDesc)
{
var jobDescription = new JobMqChangeDesc()
{
Id = pt.ID, FullDesc = pt.FullDescHtml, ShortDesc = pt.ShortDescHtml, SpecDesc = pt.SpecsDescHtml, VideoDesc = pt.VideoDescHtml
};
_producerDescriptionQuang.Publish(UtilZipFile.Zip(jobDescription.GetJSON()));
_producerChangeDesc.Publish(UtilZipFile.Zip(jobDescription.GetJSON()));
_chProductDesciptioHash.SetHashDesc(pt.CompanyId, new Tuple <long, long>(pt.ID, pt.GetHashDesc()));
}
if (pt.StatusChange.IsChangePrice)
{
_jobPushChangePrice.PushQueueChangePriceLog(new JobRabbitChangePrice()
{
Name = pt.Name, ProductID = pt.ID, CompanyID = pt.CompanyId, NewPrice = pt.Price, OldPrice = pt.OldPrice
});
}
_log.Info(string.Format("Saved for pt: {0}", pt.ID));
}
catch (Exception ex01)
{
_log.Error(ex01);
}
this.GetChannel().BasicAck(message.DeliveryTag, true);
}