public void PushJob(bool useQuery, string sql, bool allowSaveOld, bool allowSaveNew , bool bIsTest, int typeCrawler, int numberJob , DataTable tblOut = null, string queueName = "crawler_product_company_wait", int iTimeReset = 10) { try { RedisSession redisSession = new RedisSession(); SqlDb sqlDb = new SqlDb(QT.Entities.Server.ConnectionStringCrawler); DataTable tbl = (!useQuery) ? sqlDb.GetTblData(sql, CommandType.Text, null) : tblOut; foreach (DataRow row in tbl.Rows) { long companyID = Convert.ToInt64(row["ID"]); int iWait = redisSession.CheckRunning(companyID, typeCrawler, sqlDb.GetCurrent(), iTimeReset); if (iWait <= 0) { string url = QT.Entities.Common.Obj2String(row["Website"]); //Xóa Cache dữ liệu liên quan session. QueueWaitRedis queueWait = new QueueWaitRedis(companyID, typeCrawler); queueWait.Clean(); SetAddedQueueRedis setAddedQueu = new SetAddedQueueRedis(companyID, typeCrawler); setAddedQueu.Clean(); var a = Websosanh.Core.Common.BAL.ProtobufTool.Serialize(new QT.Entities.CrawlerProduct.MQCompanyWaitCrawler { CompanyID = companyID, IsTest = bIsTest, TypeCrawler = typeCrawler, datePush = DateTime.Now, ExtractionLink = (typeCrawler != 1) }); QueueWaitRedis queueWaitRedis = new QueueWaitRedis(companyID, typeCrawler); queueWaitRedis.PushJob(new Job() { deep = 0, url = url }); redisSession.InitSession(companyID, typeCrawler); this.WriteLog(string.Format("Pushed job Company:{0}. TypeCrawler:{1}", companyID, typeCrawler)); } else { this.WriteLog(string.Format("Skip company running company:{0}. TypeCrawler:{1}. Wait: {2}", companyID, typeCrawler, iWait)); } } } catch (Exception ex) { this.WriteLog("EXCEPTION POST JOB:" + ex.Message); } finally { } }
public void CleanSession(long company, int typeCrawler) { //Xoa SetAddesQueue. SetAddedQueueRedis setAdded = new SetAddedQueueRedis(company, typeCrawler); setAdded.Clean(); //Xóa queueWait. QueueWaitRedis queueWait = new QueueWaitRedis(company, typeCrawler); queueWait.Clean(); //Xoa session string key = PrefixSession + typeCrawler + ":" + company.ToString(); this.redisDb.HashSet(key, "is_end", "True"); this.redisDb.HashSet(key, "product", "0"); this.redisDb.HashSet(key, "length_queue", "0"); this.redisDb.HashSet(key, "visited", "0"); }