private void ReportToRedis(ReportSessionRunning re)
{
_redisWaitCrawler.SetRunningCrawler(re.CompanyId);
if (re.Type == "Reload")
{
_redisWaitCrawler.SetNexReload(re.CompanyId, 1);
}
else if (re.Type == "FindNew")
{
_redisWaitCrawler.SetNexFindNew(re.CompanyId, 1);
}
}
private static void SyncCompanyCrawler()
{
int count = 0;
RedisCacheCompanyCrawler redisCompany = RedisCacheCompanyCrawler.Instance();
RedisCompanyWaitCrawler redisCompanyWaitCrawler = RedisCompanyWaitCrawler.Instance();
RedisLastUpdateProduct redisLstProduct = RedisLastUpdateProduct.Instance();
CacheProductInfo cacheProductInfo = new CacheProductInfo(new SqlDb(Server.ConnectionString));
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
List <long> lstCrawler = productAdapter.GetAllCompanyIdCrawlerReload();
for (int i = 0; i < lstCrawler.Count; i++)
{
redisCompanyWaitCrawler.SetNexReload(lstCrawler[i], -10);
//long companyID = lstCrawler[i];
//if (!redisCompanyWaitCrawler.CheckHaveItemReload(companyID))
//{
// count++;
// redisCompanyWaitCrawler.SetNexReload(companyID, 1);
// Company cmp = new Company(companyID);
// redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
// cacheProductInfo.RefeshCacheProductInfo(companyID);
// redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));
//}
log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
}
List <long> lstCrawlerFindNew = productAdapter.GetAllCompanyIdCrawlerFindNew();
for (int i = 0; i < lstCrawler.Count; i++)
{
redisCompanyWaitCrawler.SetNexFindNew(lstCrawler[i], -10);
long companyID = lstCrawler[i];
//if (!redisCompanyWaitCrawler.CheckHaveItemFindNew(companyID))
//{
// count++;
// redisCompanyWaitCrawler.SetNexFindNew(companyID, 1);
// Company cmp = new Company(companyID);
// redisCompany.SetCompanyInfo(companyID, cmp.Domain, 1, 1);
// cacheProductInfo.RefeshCacheProductInfo(companyID);
// redisLstProduct.ResetLastUpdateForCompany(companyID, productAdapter.GetListProductIdOfCompany(companyID));
//}
log.Info(string.Format("sync company {0} {1}/{2}", count, i, lstCrawler.Count));
}
Console.WriteLine("Success sync company crawl!");
}
public void CrawlForCompany(long CompanyID)
{
try
{
this._companyId = CompanyID;
if ((!_bCheckOtherRunning || !CheckOtherRunning()) && _productAdapter.AllowCrawlFindNew(_companyId))
{
if (!_redisWaitCrawler.CheckHaveItemFindNew(_companyId))
{
_redisWaitCrawler.SetNexFindNew(_companyId, 1);
var company = new QT.Entities.Company(this._companyId);
_redisCacheCompanyCrawler.SetCompanyInfo(_companyId, company.Domain, 24, 24);
}
Crawl();
}
else
{
_redisWaitCrawler.SetNexFindNew(_companyId, 1);
}
}
catch (System.OperationCanceledException)
{
if (this._company != null)
{
this._redisQueueFindNew.SaveQueue(this._companyId, this._linkQueue.ToArray());
this._redisWaitCrawler.SetRemoveRunningCrawler(_companyId);
this._redisWaitCrawler.SetNexFindNew(this._companyId, 1);
this._redisCrcVisited.SetForCompany(_companyId, new List <long>(this._visitedCrc.Keys));
}
throw;
}
catch (Exception ex02)
{
_log.Error(ex02);
}
}
public void End()
{
string strLog = string.Format("End crawler {0}", _typeEnd);
if (EventReportRun != null)
{
EventReportRun(strLog);
}
_log.Info(strLog);
this.UpdateEndCrawl(new CrawlerSessionLog()
{
CompanyId = _companyId,
Domain = _company.Domain,
StartAt = _timeStart,
EndAt = DateTime.Now,
CountVisited = _countVisited,
Ip = Dns.GetHostName(),
Session = _session,
TypeRun = "AUTO",
TypeCrawler = 0,
CountChange = _countNewProduct,
TotalProduct = _company.TotalProduct,
CountProduct = 0,
TypeEnd = _typeEnd.ToString(),
NumberDuplicateProduct = _hsDuplicateProduct.Count
});
if (_typeEnd != TypeEnd.Success)
{
SaveChange();
}
else
{
var queueTableAdapter = new QueueTableAdapter();
queueTableAdapter.Connection.ConnectionString = ConfigCrawler.ConnectionCrawlerQueue;
queueTableAdapter.DeleteByCompanyId(_companyId);
queueTableAdapter.Connection.Close();
_cacheCrcVisited.RemoveCrcVited(_companyId);
}
_cacheWaitCrawler.SetRemoveRunningCrawler(_companyId);
_cacheWaitCrawler.SetNexFindNew(_companyId, 1);
_tokenSource.Cancel();
}
private void btnPush_Click(object sender, EventArgs e)
{
int hours = QT.Entities.Common.Obj2Int(txtHours.Text);
redisWaitCrawler = QT.Moduls.CrawlerProduct.Cache.RedisCompanyWaitCrawler.Instance();
lstDomain = QT.Entities.Common.GetListXPathFromString(richTextBox1.Text);
foreach (string strDomain in lstDomain)
{
tblCompany = sqldb.GetTblData("Select ID from company where Domain = @Domain and Status = 1 and DataFeedType = 0", CommandType.Text, new System.Data.SqlClient.SqlParameter[] {
sqldb.CreateParamteter("@Domain", strDomain, SqlDbType.NVarChar)
});
if (tblCompany.Rows.Count > 0)
{
CompanyID = QT.Entities.Common.Obj2Int64(tblCompany.Rows[0]["ID"]);
lstCompanyID.Add(CompanyID);
}
}
if (rdbCkFN.Checked == true)
{
foreach (long CompanyID in lstCompanyID)
{
redisWaitCrawler.SetNexFindNew(CompanyID, hours);
}
lstCompanyID.Clear();
}
else
{
foreach (long CompanyID in lstCompanyID)
{
redisWaitCrawler.SetNexReload(CompanyID, hours);
}
lstCompanyID.Clear();
}
MessageBox.Show(string.Format("Pushed: {0} company", lstDomain.Count));
lstDomain.Clear();
}