private void ReportToRedis(ReportSessionRunning re) { _redisWaitCrawler.SetRunningCrawler(re.CompanyId); if (re.Type == "Reload") { _redisWaitCrawler.SetNexReload(re.CompanyId, 1); } else if (re.Type == "FindNew") { _redisWaitCrawler.SetNexFindNew(re.CompanyId, 1); } }
private static void SyncCompanyCrawler() { int count = 0; RedisCacheCompanyCrawler redisCompany = RedisCacheCompanyCrawler.Instance(); RedisCompanyWaitCrawler redisCompanyWaitCrawler = RedisCompanyWaitCrawler.Instance(); RedisLastUpdateProduct redisLstProduct = RedisLastUpdateProduct.Instance(); CacheProductInfo cacheProductInfo = new CacheProductInfo(new SqlDb(Server.ConnectionString)); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); List <long> lstCrawler = productAdapter.GetAllCompanyIdCrawlerReload(); for (int i = 0; i < lstCrawler.Count; i++) { redisCompanyWaitCrawler.SetNexReload(lstCrawler[i], -10); //long companyID = lstCrawler[i]; //if (!redisCompanyWaitCrawler.CheckHaveItemReload(companyID)) //{ // count++; // redisCompanyWaitCrawler.SetNexReload(companyID, 1); // Company cmp = new Company(companyID); // redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1); // cacheProductInfo.RefeshCacheProductInfo(companyID); // redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID)); //} log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count)); } List <long> lstCrawlerFindNew = productAdapter.GetAllCompanyIdCrawlerFindNew(); for (int i = 0; i < lstCrawler.Count; i++) { redisCompanyWaitCrawler.SetNexFindNew(lstCrawler[i], -10); long companyID = lstCrawler[i]; //if (!redisCompanyWaitCrawler.CheckHaveItemFindNew(companyID)) //{ // count++; // redisCompanyWaitCrawler.SetNexFindNew(companyID, 1); // Company cmp = new Company(companyID); // redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1); // cacheProductInfo.RefeshCacheProductInfo(companyID); // redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID)); //} log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count)); } Console.WriteLine("Success sync company crawl!"); }
public void CrawlForCompany(long CompanyID) { try { this._companyId = CompanyID; if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlFindNew(_companyId)) { if (!_redisWaitCrawler.CheckHaveItemFindNew(_companyId)) { _redisWaitCrawler.SetNexFindNew(_companyId, 1); var company = new QT.Entities.Company(this._companyId); _redisCacheCompanyCrawler.SetCompanyInfo(_companyId, company.Domain, 24, 24); } Crawl(); } else { _redisWaitCrawler.SetNexFindNew(_companyId, 1); } } catch (System.OperationCanceledException) { if (this._company != null) { this._redisQueueFindNew.SaveQueue(this._companyId, this._linkQueue.ToArray()); this._redisWaitCrawler.SetRemoveRunningCrawler(_companyId); this._redisWaitCrawler.SetNexFindNew(this._companyId, 1); this._redisCrcVisited.SetForCompany(_companyId, new List <long>(this._visitedCrc.Keys)); } throw; } catch (Exception ex02) { _log.Error(ex02); } }
public void End() { string strLog = string.Format("End crawler {0}", _typeEnd); if (EventReportRun != null) { EventReportRun(strLog); } _log.Info(strLog); this.UpdateEndCrawl(new CrawlerSessionLog() { CompanyId = _companyId, Domain = _company.Domain, StartAt = _timeStart, EndAt = DateTime.Now, CountVisited = _countVisited, Ip = Dns.GetHostName(), Session = _session, TypeRun = "AUTO", TypeCrawler = 0, CountChange = _countNewProduct, TotalProduct = _company.TotalProduct, CountProduct = 0, TypeEnd = _typeEnd.ToString(), NumberDuplicateProduct = _hsDuplicateProduct.Count }); if (_typeEnd != TypeEnd.Success) { SaveChange(); } else { var queueTableAdapter = new QueueTableAdapter(); queueTableAdapter.Connection.ConnectionString = ConfigCrawler.ConnectionCrawlerQueue; queueTableAdapter.DeleteByCompanyId(_companyId); queueTableAdapter.Connection.Close(); _cacheCrcVisited.RemoveCrcVited(_companyId); } _cacheWaitCrawler.SetRemoveRunningCrawler(_companyId); _cacheWaitCrawler.SetNexFindNew(_companyId, 1); _tokenSource.Cancel(); }
private void btnPush_Click(object sender, EventArgs e) { int hours = QT.Entities.Common.Obj2Int(txtHours.Text); redisWaitCrawler = QT.Moduls.CrawlerProduct.Cache.RedisCompanyWaitCrawler.Instance(); lstDomain = QT.Entities.Common.GetListXPathFromString(richTextBox1.Text); foreach (string strDomain in lstDomain) { tblCompany = sqldb.GetTblData("Select ID from company where Domain = @Domain and Status = 1 and DataFeedType = 0", CommandType.Text, new System.Data.SqlClient.SqlParameter[] { sqldb.CreateParamteter("@Domain", strDomain, SqlDbType.NVarChar) }); if (tblCompany.Rows.Count > 0) { CompanyID = QT.Entities.Common.Obj2Int64(tblCompany.Rows[0]["ID"]); lstCompanyID.Add(CompanyID); } } if (rdbCkFN.Checked == true) { foreach (long CompanyID in lstCompanyID) { redisWaitCrawler.SetNexFindNew(CompanyID, hours); } lstCompanyID.Clear(); } else { foreach (long CompanyID in lstCompanyID) { redisWaitCrawler.SetNexReload(CompanyID, hours); } lstCompanyID.Clear(); } MessageBox.Show(string.Format("Pushed: {0} company", lstDomain.Count)); lstDomain.Clear(); }