private static void ResetCacheProductInfo() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); CacheProductInfo c = new CacheProductInfo(new SqlDb(Server.ConnectionString)); Console.WriteLine("1- Reset all cache by company (domain)"); Console.WriteLine("2- Reset all cache"); Console.WriteLine("3- Reset only 'product_info' all company"); Console.WriteLine("4- Reset cache ProductInfo by domain"); Console.Write("Input choose:"); int i = Convert.ToInt32(Console.ReadLine()); switch (i) { case 1: { Console.Write("Input domain:"); string strDomain = Console.ReadLine().Trim(); c.RefreshAllCacheAllProduct(productAdapter.GetCompanyIDFromDomain(strDomain)); } break; case 2: { List <long> lstCompany = productAdapter.GetAllCompanyIdCrawler(); for (int j = 0; j < lstCompany.Count; j++) { c.RefreshAllCacheAllProduct(lstCompany[j]); log.Info(string.Format("Update company {0}/{1}", j, lstCompany.Count - 1)); } } break; case 3: { List <long> lstCompany = productAdapter.GetAllCompanyIdCrawler(); for (int j = 0; j < lstCompany.Count; j++) { c.ResetAllCacheProductInfo(lstCompany[j]); log.Info(string.Format("Update company {0}/{1}", j, lstCompany.Count - 1)); } } break; case 4: { Console.Write("Input domain:"); string strDomain = Console.ReadLine().Trim(); c.ResetForCompany(productAdapter.GetCompanyIDFromDomain(strDomain)); } break; } }
public void Start() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); List <long> lstCOmpanyDownloaded = productAdapter.GetCOmpanyDownloadedHTML(); foreach (long companyID in productAdapter.GetAllCompanyIdCrawler()) { if (!lstCOmpanyDownloaded.Contains(companyID)) { lst.Enqueue(companyID); } } int MaxThread = 100; for (int i = 0; i < MaxThread; i++) { int jThread = i; Task.Factory.StartNew(() => { WorkerExportHtml worker = new WorkerExportHtml(jThread); worker.eventGetCompanyID += new WorkerExportHtml.GetCompanyId(GetCompanyCrawler); worker.eventProcessHtml += new WorkerExportHtml.DelegateProcessHtml(SaveHtml); worker.eventWhenFinish += new WorkerExportHtml.DelegateWhenFinish(EventWhenFinsih); worker.Start(); }); } }
private static void ResesCacheProduct() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); Console.WriteLine("1- Reset all Companies)"); Console.WriteLine("2- Reset for Company"); Console.Write("Input choose:"); int i = Convert.ToInt32(Console.ReadLine()); switch (i) { case 1: { int countProduct = 0; var lstCompany = productAdapter.GetAllCompanyIdCrawler(); foreach (var companyID in lstCompany) { ResetCacheProduct(companyID); log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count)); } log.Info("Complete all company"); } break; case 2: { Console.Write("Input domain:"); string strDomain = Console.ReadLine().Trim(); long companyID = productAdapter.GetCompanyIdByDomain(strDomain); ResetCacheProduct(companyID); log.Info(string.Format("Complete Company: {0}", companyID)); } break; } }
public void Start() { ProductAdapter pa = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct)); QT.Entities.Server.ConnectionString = ConfigCrawler.ConnectProduct; List <long> companyIds = pa.GetAllCompanyIdCrawler(); for (int i = 0; i < companyIds.Count; i++) { long companyId = companyIds[i]; Company company = new Company(companyId); Configuration configuration = new Configuration(companyId); ProductParse parse = new ProductParse(); ProductEntity productEntity = new ProductEntity(); HtmlDocument document = new HtmlDocument(); string url = configuration.LinkTest; string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(url, 45, 2); if (!string.IsNullOrEmpty(html)) { document.LoadHtml(html); try { parse.Analytics(productEntity, document, configuration.LinkTest, configuration, company.Domain); if (!productEntity.IsSuccessData(configuration.CheckPrice)) { pa.GetSqlDb() .RunQuery("insert into Company_FailConfig (CompanyId) Values (@CompanyId)", CommandType.Text, new SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyId", companyId, SqlDbType.BigInt) }); } } catch (Exception ex) { pa.GetSqlDb() .RunQuery("insert into Company_FailConfig (CompanyId, Error) Values (@CompanyId, @Error)", CommandType.Text, new SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyId", companyId, SqlDbType.BigInt), SqlDb.CreateParamteterSQL("@Error", ex.Message + "\n" + ex.StackTrace, SqlDbType.NVarChar) }); } _log.Info(string.Format("Run data {0}/ {1}", i, companyIds.Count)); } } }
private static void DeleteAllUnvalidProduct() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); List <long> lstCompanyCrawler = productAdapter.GetAllCompanyIdCrawler(); for (int i = 0; i < lstCompanyCrawler.Count; i++) { productAdapter.DeleteProductUnvalidOfCOmpany(lstCompanyCrawler[i]); log.Info(string.Format("Delete unvalid product for company {0}/{1}", i, lstCompanyCrawler.Count - 1)); } log.Info("Delete unvalid product for company"); }
private static void ManagerCacheLastTime() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); RedisLastUpdateProduct redisLastUpdate = RedisLastUpdateProduct.Instance(); Console.WriteLine("1- Reset cache by company (domain)"); Console.WriteLine("2- Cache for AllCompany"); Console.Write("Input choose:"); int i = Convert.ToInt32(Console.ReadLine()); switch (i) { case 1: { Console.Write("Input domain you need reset cache:"); string strDomain = Console.ReadLine().Trim(); long CompanyID = productAdapter.GetCompanyIDFromDomain(strDomain); redisLastUpdate.RemoveAllLstProduct(CompanyID); DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt) }); List <long> productIDs = new List <long>(); foreach (DataRow rowInfo in tbl.Rows) { productIDs.Add(Convert.ToInt64(rowInfo["id"])); } redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, new DateTime(1900, 1, 1, 0, 0, 0)); Console.WriteLine("Reseted all product LastUpdate in cache"); } break; case 2: { List <long> companyCrawelrs = productAdapter.GetAllCompanyIdCrawler(); int Count = 0; foreach (long CompanyID in companyCrawelrs) { Count++; DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt) }); List <long> productIDs = new List <long>(); foreach (DataRow rowInfo in tbl.Rows) { productIDs.Add(Convert.ToInt64(rowInfo["id"])); } redisLastUpdate.RemoveAllLstProduct(CompanyID); redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, new DateTime(1990, 1, 1, 0, 0, 0)); log.Info(string.Format("Reseted all products for company {0}/{1} LastUpdate in cache", Count, companyCrawelrs.Count)); } } break; } }
private static void ResetAllCopmanyInfo() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); RedisCacheCompanyCrawler redisCompanyInfo = RedisCacheCompanyCrawler.Instance(); List <long> lstCompanyCrawler = productAdapter.GetAllCompanyIdCrawler(); for (int i = 0; i < lstCompanyCrawler.Count; i++) { Company comp = new Company(lstCompanyCrawler[i]); redisCompanyInfo.SetCompanyInfo(comp.ID, comp.Domain, 12, 12); log.Info(string.Format("Pushed companyInfo {0}/{1}", i, lstCompanyCrawler.Count - 1)); } log.Info("Success push companyInfo"); }
private static void DeleteUnvalidProductCrawler() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); int COunt = 0; List <long> lst = productAdapter.GetAllCompanyIdCrawler(); for (int i = 0; i < lst.Count; i++) { COunt++; productAdapter.sqlDb.RunQuery("delete product where company = @company_id and valid = 0 and isnull(IsBlackList,0) = 0", CommandType.Text, new System.Data.SqlClient.SqlParameter[] { SqlDb.CreateParamteterSQL("@company_id", lst[i], SqlDbType.BigInt) }, true, 1000); log.Info(string.Format("Delete unvalid product for {0}/{1}", i, lst.Count - 1)); } }
public void Run(System.Threading.CancellationToken token) { List <long> companyCrawelrs = productAdapter.GetAllCompanyIdCrawler(); int Count = 0; foreach (long CompanyID in companyCrawelrs) { Count++; DataTable tbl = productAdapter.sqlDb.GetTblData("select id from product where company = @CompanyID", CommandType.Text, new System.Data.SqlClient.SqlParameter[] { SqlDb.CreateParamteterSQL("@CompanyID", CompanyID, SqlDbType.BigInt) }); List <long> productIDs = new List <long>(); foreach (DataRow rowInfo in tbl.Rows) { productIDs.Add(Convert.ToInt64(rowInfo["id"])); } redisLastUpdate.RemoveAllLstProduct(CompanyID); redisLastUpdate.UpdateBathLastUpdateProduct(CompanyID, productIDs, DateTime.Now); log.Info(string.Format("Reseted all products for company {0}/{1} LastUpdate in cache", Count, companyCrawelrs.Count)); } }
private static void ResetAllCacheProduct() { Console.Write("Input sleep:"); int iSleep = Convert.ToInt32(Console.ReadLine()); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); RedisCheckDuplicateAdapter redisCheckDuplicate = RedisCheckDuplicateAdapter.Instace(); RedisLastUpdateProduct redisLastUpdate = RedisLastUpdateProduct.Instance(); CacheProductInfo cacheProductInfo = new CacheProductInfo(productAdapter.sqlDb); int count = 0; var companyIDs = productAdapter.GetAllCompanyIdCrawler(); foreach (long companyID in companyIDs) { Company cmp = new Company(companyID); redisCheckDuplicate.ResetForCompany(companyID, cmp.Domain, productAdapter.GetSqlDb()); redisLastUpdate.ResetForCompany(companyID, productAdapter.sqlDb); cacheProductInfo.ResetForCompany(companyID); log.Info(string.Format("Reset cache company {0}/{1}-{2} {3} products", count++, companyIDs.Count, cmp.Domain, cmp.TotalProduct)); Thread.Sleep(iSleep * 1000); } Console.WriteLine("Success reset data!"); }
public void Run(System.Threading.CancellationToken token) { ProductAdapter productAdapter = new ProductAdapter(new SqlDb("Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200")); log.InfoFormat("Start run at {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture)); CacheProductHash cashProductHash = CacheProductHash.Instance(); RedisLastUpdateProduct cacheLastUpdateProduct = RedisLastUpdateProduct.Instance(); int countProduct = 0; try { var lstFn = productAdapter.GetAllCompanyIdCrawlerFindNew(); var lstRl = productAdapter.GetAllCompanyIdCrawlerReload(); RedisCompanyWaitCrawler redisCache = RedisCompanyWaitCrawler.Instance(); redisCache.SyncCompanyFindNew(lstFn); redisCache.SyncCompanyReload(lstRl); } catch (Exception ex) { log.Error(ex); } var lst = new List <QT.Entities.CrawlerProduct.Cache.ProductHash>(); var lstLastUpdate = new List <long>(); var lstCompany = productAdapter.GetAllCompanyIdCrawler(); foreach (var companyID in lstCompany) { Company cmp = new Company(companyID); productAdapter.DeleteProductUnvalidOfCOmpany(companyID); DataTable tbl = productAdapter.GetProductResetColumnDuplicateAndChange(companyID); foreach (DataRow rowProduct in tbl.Rows) { long productId = QT.Entities.Common.Obj2Int64(rowProduct["ID"]); long originPrice = QT.Entities.Common.Obj2Int64(rowProduct["OriginPrice"]); string name = rowProduct["Name"].ToString(); long price = QT.Entities.Common.Obj2Int64(rowProduct["Price"]); string imageUrl = Convert.ToString(rowProduct["ImageUrls"]); string detailUrl = Convert.ToString(rowProduct["DetailUrl"]); int inStock = QT.Entities.Common.Obj2Int(rowProduct["InStock"]); bool valid = QT.Entities.Common.Obj2Bool(rowProduct["Valid"]); string shortDescription = QT.Entities.Common.CellToString(rowProduct["ShortDescription"], ""); long categoryId = rowProduct["ClassificationID"] == DBNull.Value ? 0 : QT.Entities.Common.Obj2Int64(rowProduct["ClassificationID"]); long hashChange = ProductEntity.GetHashChangeInfo(inStock, valid, price, name, imageUrl, categoryId, shortDescription, originPrice); long hashDuplicate = Product.GetHashDuplicate(cmp.Domain, price, name, imageUrl); long hashImage = Product.GetHashImageInfo(imageUrl); lst.Add(new QT.Entities.CrawlerProduct.Cache.ProductHash() { HashChange = hashChange, HashDuplicate = hashDuplicate, HashImage = hashImage, Id = productId, Price = price, url = detailUrl }); lstLastUpdate.Add(productId); } cashProductHash.SetCacheProductHash(companyID, lst, 100); cacheLastUpdateProduct.RemoveAllLstProduct(companyID); cacheLastUpdateProduct.UpdateBathLastUpdateProduct(companyID, lstLastUpdate, DateTime.Now.AddDays(-1)); productAdapter.UpdateCountProductForCompany(companyID, lstLastUpdate.Count, lstLastUpdate.Count); lst.Clear(); lstLastUpdate.Clear(); log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count)); } log.Info("Complete all company"); NextRun = DateTime.Now.AddHours(MAX_HOUR_LOOP); log.InfoFormat("End at {0}", DateTime.Now.ToString()); }