public void Start()
{
try
{
if (_config.NumberThreadCrawler > 1)
{
Crawl(_config.NumberThreadCrawler);
}
else
{
Crawl("NoName");
}
}
catch (OperationCanceledException)
{
if (_company == null)
{
throw;
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexReload(this._companyId, 1);
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
public void End()
{
string strLog = string.Format("End crawler {1} {0}", _session, _typeEnd);
if (EventReportRun != null)
{
EventReportRun(strLog);
}
_log.Info(strLog);
UpdateEndCrawl(new CrawlerSessionLog()
{
CompanyId = _companyId,
Domain = _company.Domain,
StartAt = _timeStart,
EndAt = DateTime.Now,
CountVisited = _countVisited,
Ip = Dns.GetHostName(),
Session = _session,
TypeRun = "AUTO",
TypeCrawler = 1,
CountChange = _countChange,
TotalProduct = _company.TotalProduct,
CountProduct = _countProduct,
TypeEnd = _typeEnd.ToString()
});
_cacheWaitCrawler.SetNexReload(_companyId, _config.MinHourReload);
_cacheWaitCrawler.SetRemoveRunningCrawler(_companyId);
TokenSource.Cancel();
}
private bool CheckOtherRunning(long companyId)
{
if (_redisWaitCrawler.CheckRunningCrawler(companyId))
{
_redisWaitCrawler.SetNexReload(companyId, 1);
return(true);
}
else
{
return(false);
}
}
private void ReportToRedis(ReportSessionRunning re)
{
_redisWaitCrawler.SetRunningCrawler(re.CompanyId);
if (re.Type == "Reload")
{
_redisWaitCrawler.SetNexReload(re.CompanyId, 1);
}
else if (re.Type == "FindNew")
{
_redisWaitCrawler.SetNexFindNew(re.CompanyId, 1);
}
}
public void CrawlForCompany(long companyId)
{
try
{
_companyId = companyId;
if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlReload(_companyId))
{
InitSession();
if (_config.NumberThreadCrawler > 1)
{
Crawl(_config.NumberThreadCrawler);
}
else
{
Crawl("NO");
}
EndSession();
}
else
{
_redisWaitCrawler.SetNexReload(_companyId, 1);
}
}
catch (OperationCanceledException)
{
if (_company == null)
{
throw;
}
_redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
_redisWaitCrawler.SetNexReload(_companyId, 1);
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
private static void SyncCompanyCrawler()
{
int count = 0;
RedisCacheCompanyCrawler redisCompany = RedisCacheCompanyCrawler.Instance();
RedisCompanyWaitCrawler redisCompanyWaitCrawler = RedisCompanyWaitCrawler.Instance();
RedisLastUpdateProduct redisLstProduct = RedisLastUpdateProduct.Instance();
CacheProductInfo cacheProductInfo = new CacheProductInfo(new SqlDb(Server.ConnectionString));
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
List <long> lstCrawler = productAdapter.GetAllCompanyIdCrawlerReload();
for (int i = 0; i < lstCrawler.Count; i++)
{
redisCompanyWaitCrawler.SetNexReload(lstCrawler[i], -10);
//long companyID = lstCrawler[i];
//if (!redisCompanyWaitCrawler.CheckHaveItemReload(companyID))
//{
// count++;
// redisCompanyWaitCrawler.SetNexReload(companyID, 1);
// Company cmp = new Company(companyID);
// redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
// cacheProductInfo.RefeshCacheProductInfo(companyID);
// redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));
//}
log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
}
List <long> lstCrawlerFindNew = productAdapter.GetAllCompanyIdCrawlerFindNew();
for (int i = 0; i < lstCrawler.Count; i++)
{
redisCompanyWaitCrawler.SetNexFindNew(lstCrawler[i], -10);
long companyID = lstCrawler[i];
//if (!redisCompanyWaitCrawler.CheckHaveItemFindNew(companyID))
//{
// count++;
// redisCompanyWaitCrawler.SetNexFindNew(companyID, 1);
// Company cmp = new Company(companyID);
// redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
// cacheProductInfo.RefeshCacheProductInfo(companyID);
// redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));
//}
log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
}
Console.WriteLine("Success sync company crawl!");
}
private void btnPush_Click(object sender, EventArgs e)
{
int hours = QT.Entities.Common.Obj2Int(txtHours.Text);
redisWaitCrawler = QT.Moduls.CrawlerProduct.Cache.RedisCompanyWaitCrawler.Instance();
lstDomain = QT.Entities.Common.GetListXPathFromString(richTextBox1.Text);
foreach (string strDomain in lstDomain)
{
tblCompany = sqldb.GetTblData("Select ID from company where Domain = @Domain and Status = 1 and DataFeedType = 0", CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
sqldb.CreateParamteter("@Domain", strDomain, SqlDbType.NVarChar)
});
if (tblCompany.Rows.Count > 0)
{
CompanyID = QT.Entities.Common.Obj2Int64(tblCompany.Rows[0]["ID"]);
lstCompanyID.Add(CompanyID);
}
}
if (rdbCkFN.Checked == true)
{
foreach (long CompanyID in lstCompanyID)
{
redisWaitCrawler.SetNexFindNew(CompanyID, hours);
}
lstCompanyID.Clear();
}
else
{
foreach (long CompanyID in lstCompanyID)
{
redisWaitCrawler.SetNexReload(CompanyID, hours);
}
lstCompanyID.Clear();
}
MessageBox.Show(string.Format("Pushed: {0} company", lstDomain.Count));
lstDomain.Clear();
}