public void Start()
{
try
{
if (_config.NumberThreadCrawler > 1)
{
Crawl(_config.NumberThreadCrawler);
}
else
{
Crawl("NoName");
}
}
catch (OperationCanceledException)
{
if (_company == null)
{
throw;
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexReload(this._companyId, 1);
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
public void End()
{
string strLog = string.Format("End crawler {1} {0}", _session, _typeEnd);
if (EventReportRun != null)
{
EventReportRun(strLog);
}
_log.Info(strLog);
UpdateEndCrawl(new CrawlerSessionLog()
{
CompanyId = _companyId,
Domain = _company.Domain,
StartAt = _timeStart,
EndAt = DateTime.Now,
CountVisited = _countVisited,
Ip = Dns.GetHostName(),
Session = _session,
TypeRun = "AUTO",
TypeCrawler = 1,
CountChange = _countChange,
TotalProduct = _company.TotalProduct,
CountProduct = _countProduct,
TypeEnd = _typeEnd.ToString()
});
_cacheWaitCrawler.SetNexReload(_companyId, _config.MinHourReload);
_cacheWaitCrawler.SetRemoveRunningCrawler(_companyId);
TokenSource.Cancel();
}
public void CrawlForCompany(long companyId)
{
try
{
_companyId = companyId;
if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlReload(_companyId))
{
InitSession();
if (_config.NumberThreadCrawler > 1)
{
Crawl(_config.NumberThreadCrawler);
}
else
{
Crawl("NO");
}
EndSession();
}
else
{
_redisWaitCrawler.SetNexReload(_companyId, 1);
}
}
catch (OperationCanceledException)
{
if (_company == null)
{
throw;
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexReload(_companyId, 1);
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
public void End()
{
string strLog = string.Format("End crawler {0}", _typeEnd);
if (EventReportRun != null)
{
EventReportRun(strLog);
}
_log.Info(strLog);
this.UpdateEndCrawl(new CrawlerSessionLog()
{
CompanyId = _companyId,
Domain = _company.Domain,
StartAt = _timeStart,
EndAt = DateTime.Now,
CountVisited = _countVisited,
Ip = Dns.GetHostName(),
Session = _session,
TypeRun = "AUTO",
TypeCrawler = 0,
CountChange = _countNewProduct,
TotalProduct = _company.TotalProduct,
CountProduct = 0,
TypeEnd = _typeEnd.ToString(),
NumberDuplicateProduct = _hsDuplicateProduct.Count
});
if (_typeEnd != TypeEnd.Success)
{
SaveChange();
}
else
{
var queueTableAdapter = new QueueTableAdapter();
queueTableAdapter.Connection.ConnectionString = ConfigCrawler.ConnectionCrawlerQueue;
queueTableAdapter.DeleteByCompanyId(_companyId);
queueTableAdapter.Connection.Close();
_cacheCrcVisited.RemoveCrcVited(_companyId);
}
_cacheWaitCrawler.SetRemoveRunningCrawler(_companyId);
_cacheWaitCrawler.SetNexFindNew(_companyId, 1);
_tokenSource.Cancel();
}
private void EndSession()
{
if (this._linkQueue.Count == 0)
{
_redisCrcVisited.RemoveCrcVited(_companyId);
}
else
{
_redisCrcVisited.SetForCompany(_companyId, new List <long>(_visitedCrc.Keys));
_redisQueueFindNew.SaveQueue(_companyId, _linkQueue.ToArray());
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexFindNew(_companyId, _config.MinHourFindNew);
_productAdapter.UpdateEndCrawl(_company.ID, 0, startCrawler, DateTime.Now, _countLink, _countVisited, _totalProductBefore, _countChange, _session, _totalProduct + _totalProductBefore, QT.Entities.Server.IPHost, _company.Domain);
LogData(string.Format(" >>>>>>> END SESSION {0} - Vs:{1}", _company.Domain, _countVisited));
this._visitedCrc.Clear();
this._linkQueue.Clear();
this._companyId = 0;
this._company = null;
}