private void InitSession() { _session = Guid.NewGuid().ToString(); if (!_productAdapter.AllowCrawlReload(this._companyId)) { return; } _config = new Configuration(_companyId, true); _company = new Entities.Company(_companyId); _productAdapter.UpdateLastJobForDb(_company.ID); _productAdapter.UpdateLastReloadCrawler(_company.ID); _startCrawler = DateTime.Now; _crawlerRegexs = _config.VisitUrlsRegex; _noCrawlerRegexs = _config.NoVisitUrlRegex; _countChange = 0; _countLink = 0; _countVisited = 0; _countProduct = 0; _totalProductBefore = _company.TotalProduct; _productWaitInsertGroup.Clear(); _productIdWaitChangeImage.Clear(); _linksQueue.Clear(); _crawlerRegexs.Clear(); _noCrawlerRegexs.Clear(); _dicCacheProduct.Clear(); _dicDuplicate.Clear(); _dicTrackDie = _cacheCheckDelete.GetDicTrackOfCompany(_companyId); lock (_productWaitUpdateGroup) { _productWaitUpdateGroup.Clear(); } LoadQueue(); }
public void CrawlForCompany(long companyId) { try { _companyId = companyId; if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlReload(_companyId)) { InitSession(); if (_config.NumberThreadCrawler > 1) { Crawl(_config.NumberThreadCrawler); } else { Crawl("NO"); } EndSession(); } else { _redisWaitCrawler.SetNexReload(_companyId, 1); } } catch (OperationCanceledException) { if (_company == null) { throw; } _redisWaitCrawler.SetRemoveRunningCrawler(_companyId); _redisWaitCrawler.SetNexReload(_companyId, 1); throw; } catch (Exception ex02) { _log.Error(ex02); } }