public void Run() { Task.Factory.StartNew(() => { SqlDb sqlDb = new QT.Entities.Data.SqlDb(@"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"); QT.Moduls.CrawlerProduct.Cache.CacheProductInfo cacheProductInfo = new QT.Moduls.CrawlerProduct.Cache.CacheProductInfo(sqlDb); while (true) { BasicGetResult result = this.GetMessage(); if (result != null) { string str = System.Text.Encoding.UTF8.GetString(result.Body); if (str != "") { MssRefreshCacheProductInfo job = MssRefreshCacheProductInfo.FromJSON(str); log.Info(string.Format("Get Message For Company:{0} {1}", job.CompanyID, job.Domain)); RedisCacheProductInfo rediscacheProductForCompany = RedisCacheProductInfo.Instance(); int ProductCache = cacheProductInfo.RefreshAllCacheAllProduct(job.CompanyID); log.Info(string.Format("Company:{0} NumberProductCache:{1}", job.CompanyID, job.Domain)); } } this.tokenStop.WaitHandle.WaitOne(10000); } }); }
protected override void OnStart(string[] args) { log.Info("Start service"); try { InitializeComponent(); cancelTokenSource = new CancellationTokenSource(); Server.LogConnectionString = ConfigurationManager.AppSettings["LogConnectionString"]; QT.Entities.Server.ConnectionString = connectionString; string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; workers = new Worker[workerCount]; rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); for (int i = 0; i < workerCount; i++) { log.InfoFormat("Start worker {i}", i.ToString()); var worker = new Worker(refreshCacheProductInfoJobName, false, rabbitMQServer); workers[i] = worker; var token = this.cancelTokenSource.Token; Task workerTask = new Task(() => { var db = new QT.Entities.Data.SqlDb(this.connectionString); RedisCacheProductInfo rediscacheProductForCompany = new RedisCacheProductInfo(); QT.Moduls.CrawlerProduct.Cache.CacheProductInfo cacheProductInfo = new QT.Moduls.CrawlerProduct.Cache.CacheProductInfo(db); worker.JobHandler = (jobMss) => { try { token.ThrowIfCancellationRequested(); QT.Entities.CrawlerProduct.RabbitMQ.MssRefreshCacheProductInfo mss = QT.Entities.CrawlerProduct.RabbitMQ.MssRefreshCacheProductInfo.FromJSON(QT.Entities.Common.ByteToString(jobMss.Data)); log.InfoFormat("Start run refresh company {0} : {1}", mss.CompanyID, mss.Domain); int numberProduct = cacheProductInfo.RefreshAllCacheAllProduct(mss.CompanyID); log.InfoFormat("End refresh company {0} : {1} {2} products", mss.CompanyID, mss.Domain, numberProduct); return(true); } catch (OperationCanceledException opc) { log.Info("End worker"); return(false); } }; worker.Start(); }, token); workerTask.Start(); log.InfoFormat("Worker {0} started", i); } } catch (Exception ex) { log.Error("Start error", ex); throw; } }
private void Init() { _redisDesHash = CacheProductDesciptioHash.Instance(); _jobClientLogChangePrice = new MqLogChangePrice(); _jobClientLogChangeProduct = MQLogChangeProduct.Instance(); _dicDuplicate = new Dictionary <long, long>(); _dicCacheProduct = new Dictionary <long, ProductCache>(); _linksQueue = new Queue <NSCrawler.Job>(); _productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); _redisWaitCrawler = RedisCompanyWaitCrawler.Instance(); _redisProduct = RedisCacheProductInfo.Instance(); _redisProductLastUpdate = RedisLastUpdateProduct.Instance(); _publiserDesciption = new PublisherDesciption(); }
private void btnRemoveHashChange_Click(object sender, EventArgs e) { RedisLastUpdateProduct redisLastUpdateProduct = RedisLastUpdateProduct.Instance(); RedisCacheProductInfo redisCacheProductInfo = RedisCacheProductInfo.Instance(); int iRowHandle = this.gridView1.FocusedRowHandle; if (iRowHandle >= 0) { long CompanyID = Convert.ToInt64(this.gridView1.GetRowCellValue(iRowHandle, "Id")); List <long> lstProduct = redisLastUpdateProduct.GetAllProduct(CompanyID); foreach (long product in lstProduct) { redisCacheProductInfo.ResetHashChange(CompanyID, product); } } }
public int ReloadChaceForAllCompany(CancellationToken CancellationToken = new CancellationToken()) { log.Info(string.Format("Started Push For AllCompany")); DataTable tblCompany = sqlDb.GetTblData("Select ID,Domain From Company Where Status = 1 And DatafeedType=0"); int iCount = 0; foreach (DataRow rowInfo in tblCompany.Rows) { iCount++; long Company = QT.Entities.Common.Obj2Int64(rowInfo["ID"]); string Domain = QT.Entities.Common.Obj2String(rowInfo["Domain"]); RedisCacheProductInfo rediscacheProductForCompany = RedisCacheProductInfo.Instance(); int iProductPush = this.RefreshAllCacheAllProduct(Company); log.Info(string.Format("Pushed C {0} {3}/{2} : {1}", Company, iProductPush, tblCompany.Rows.Count, iCount)); CancellationToken.ThrowIfCancellationRequested(); } log.Info(string.Format("End Push For All {0}", tblCompany.Rows.Count)); return(tblCompany.Rows.Count); }
private void PushProduct() { while (true) { try { var productAdapter = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString)); DataTable tbl = productAdapter.GetListCompanyIDToPush(); foreach (DataRow row in tbl.Rows) { long companyID = Convert.ToInt64(row["ID"]); string domain = Convert.ToString(row["Domain"]); this.Invoke(new Action(() => { lblCompany.Text = domain; })); int iPage = 0; while (true) { DataTable tblProduct = productAdapter.GetTblProductPushRedisForCompany(companyID, iPage); if (tblProduct.Rows.Count == 0) { break; } iPage++; this.Invoke(new Action(() => { lblPageIDProduct.Text = iPage.ToString(); })); foreach (DataRow rowProduct in tblProduct.Rows) { long ProductID = Convert.ToInt64(rowProduct["ID"]); long CompanyID = Convert.ToInt64(rowProduct["Company"]); string ProductName = Common.Obj2String(rowProduct["Name"]); int ProductPrice = Common.Obj2Int(rowProduct["Price"]); bool Valid = Common.Obj2Bool(rowProduct["Valid"]); string keyRedis = "product:" + ProductID.ToString(); int Status = Common.Obj2Int(rowProduct["Status"]); string ImageUrl = Common.Obj2String(rowProduct["ImageUrls"]); bool IsDea = Common.Obj2Bool(rowProduct["IsDeal"]); long IDCategory = Common.Obj2Int64(rowProduct["ClassificationID"]); if (!RedisCacheProductInfo.Instance().ExistsInCache(companyID, ProductID)) { int count = 0; while (true) { try { break; } catch (Exception ex) { count++; if (count > 5) { break; } else { Thread.Sleep(1000); } } } } } } productAdapter.UpdateLastSyncProductRedisForCompany(companyID); } } catch (ThreadAbortException ex) { return; } catch (Exception ex1) { Thread.Sleep(1000); } } }