private void InitSession()
{
_session = Guid.NewGuid().ToString();
if (!_productAdapter.AllowCrawlReload(this._companyId))
{
return;
}
_config = new Configuration(_companyId, true);
_company = new Entities.Company(_companyId);
_productAdapter.UpdateLastJobForDb(_company.ID);
_productAdapter.UpdateLastReloadCrawler(_company.ID);
_startCrawler = DateTime.Now;
_crawlerRegexs = _config.VisitUrlsRegex;
_noCrawlerRegexs = _config.NoVisitUrlRegex;
_countChange = 0;
_countLink = 0;
_countVisited = 0;
_countProduct = 0;
_totalProductBefore = _company.TotalProduct;
_productWaitInsertGroup.Clear();
_productIdWaitChangeImage.Clear();
_linksQueue.Clear();
_crawlerRegexs.Clear();
_noCrawlerRegexs.Clear();
_dicCacheProduct.Clear();
_dicDuplicate.Clear();
_dicTrackDie = _cacheCheckDelete.GetDicTrackOfCompany(_companyId);
lock (_productWaitUpdateGroup)
{
_productWaitUpdateGroup.Clear();
}
LoadQueue();
}
public void CrawlForCompany(long companyId)
{
try
{
_companyId = companyId;
if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlReload(_companyId))
{
InitSession();
if (_config.NumberThreadCrawler > 1)
{
Crawl(_config.NumberThreadCrawler);
}
else
{
Crawl("NO");
}
EndSession();
}
else
{
_redisWaitCrawler.SetNexReload(_companyId, 1);
}
}
catch (OperationCanceledException)
{
if (_company == null)
{
throw;
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexReload(_companyId, 1);
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
