public static void PushDownloadHtml(IEnumerable <string> domains)
{
SqlDb sql = new SqlDb(ConfigStatic.ProductConnection);
ProductAdapter productAdapter = new ProductAdapter(sql);
foreach (var domain in domains)
{
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
string queryData = string.Format(@"
Select Id, DetailUrl
From Product
Where Company = {0}
Order by Id
"
, productAdapter.GetCompanyIdByDomain(domain));
sql.ProcessDataTableLarge(queryData,
10000, (Row, iRow) =>
{
producerBasic.PublishString(new JobDownloadHtml()
{
ProductId = Convert.ToInt64(Row["Id"]),
DetailUrl = Convert.ToString(Row["DetailUrl"]),
Domain = domain
}.GetJson());
});
}
}
public Runner()
{
_connectinString = ConfigurationManager.AppSettings["ConnectionString"];
_producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer("rabbitMqCrlProperties"), "Product.AdsScore.Deleted");
MAX_HOUR_LOOP = CommonConvert.Obj2Int(ConfigurationManager.AppSettings["MAX_HOUR_LOOP"]);
HOUR_RUN = CommonConvert.Obj2Int(ConfigurationManager.AppSettings["HOUR_RUN"]);
}
public static bool DownloadImageProduct(ImageProductInfo imageProductInfo, ProducerBasic producerUpdateImageIdSql, ref string messageError)
{
bool result = false;
try
{
var idImbo = Common.DownloadImageProductWithImboServer(imageProductInfo.ImageUrls, ConfigImbo.PublicKey, ConfigImbo.PrivateKey, "wss", ConfigImbo.Host, ConfigImbo.Port);
if (!string.IsNullOrEmpty(idImbo))
{
UpdateImageIdSqlService(imageProductInfo.Id, idImbo, producerUpdateImageIdSql);
Log.Info(string.Format("Product: ID = {0} download image success!", imageProductInfo.Id));
messageError = "";
result = true;
}
else
{
Log.Info(string.Format("Product: ID = {0} download image fails idImbo null!", imageProductInfo.Id));
messageError = "IDImbo null";
result = false;
}
}
catch (Exception exception)
{
Log.Error(string.Format("Product: ID = {0}. ImageUrl: {1} . DetailUrl: {2}", imageProductInfo.Id, imageProductInfo.ImageUrls, imageProductInfo.DetailUrl), exception);
messageError = exception.ToString();
}
return(result);
}
private bool DownloadImageRootProduct(ImageProductInfo imageProductInfo, ProducerBasic producerUpdateImageIdSql)
{
bool result = false;
try
{
var idImbo = Common.DownloadImageProductWithImboServer(imageProductInfo.ImageUrls, ConfigImbo.PublicKey, ConfigImbo.PrivateKey, "root_product", ConfigImbo.Host, ConfigImbo.Port);
if (!string.IsNullOrEmpty(idImbo))
{
UpdateImageIdSqlService(imageProductInfo.Id, idImbo, producerUpdateImageIdSql);
//ThumbImageService(imageProductInfo.Id, idImbo, producerThumbImage);
Log.Info(string.Format("RootProduct: ID = {0} download image success!", imageProductInfo.Id));
//InsertLogDownloadImageProduct(imageProductInfo.Id);
result = true;
}
else
{
imageProductInfo.ErrorMessage = "IDImbo = null";
SendErrorDownloadImageToService(imageProductInfo);
}
}
catch (Exception exception)
{
Log.Error(string.Format("Product: ID = {0}. ImageUrl: {1} . DetailUrl: {2}", imageProductInfo.Id, imageProductInfo.ImageUrls, imageProductInfo.DetailUrl), exception);
imageProductInfo.ErrorMessage = exception.ToString();
SendErrorDownloadImageToService(imageProductInfo);
}
return(result);
}
public static void PushParseFromNoSql(IEnumerable <string> domains)
{
NoSqlAdapter noSqlAdapter = NoSqlAdapter.GetInstance();
foreach (var domain in domains)
{
Task.Factory.StartNew(() =>
{
int iCount = 0;
var producerBasicWaitPs = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(domain));
noSqlAdapter.ProcessAllIdProductByCompany(domain, (obj, productId) =>
{
iCount++;
producerBasicWaitPs.PublishString(new JobParse()
{
Id = productId
}.GetJson());
if (iCount % 10 == 0)
{
log.Info(string.Format("Pushed {0} mss of {1}", iCount, domain));
}
});
log.Info(string.Format("Pushed all data for company {0} {1}", domain, iCount));
});
}
Thread.Sleep(10000000);
}
//public void ThumbImageService(long productId, string idImbo, ProducerBasic producerThumbImage)
//{
// int index = 0;
// while (_isRunning)
// {
// try
// {
// producerThumbImage.PublishString(new JobWaitThumb()
// {
// ImageId = idImbo,
// Sizes = _widthHeightImages
// }.ToJson());
// break;
// }
// catch (Exception exception)
// {
// Thread.Sleep(600000);
// Log.Error(
// string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
// productId), exception);
// if (index == 5)
// break;
// else
// index++;
// }
// }
//}
public void UpdateImageIdSqlService(long productId, string idImageImbo, ProducerBasic producerUpdateImageIdSql)
{
int index = 0;
while (_isRunning)
{
try
{
producerUpdateImageIdSql.PublishString(new JobUploadedImg()
{
ImageId = idImageImbo,
ProductId = productId
}.ToJson());
break;
}
catch (Exception exception)
{
Thread.Sleep(600000);
Log.Error(
string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
productId), exception);
if (index == 5)
{
break;
}
else
{
index++;
}
}
}
}
internal void PushQueueUpdateAlexaSeoSystem()
{
SqlDb _sqlDb =
new SqlDb("Data Source=192.168.100.178;Initial Catalog=QA_System;User ID=sa;Password=123456a@");
string queryCompany = @"
select ID, Name as Domain
From Web
where LastUpdateAlexa is null
--where IsCrawler = 1
";
ProducerBasic producerBasic =
new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct),
"UpdateAlexaCompanySeo");
DataTable tblCompanyTable = _sqlDb.GetTblData(queryCompany, CommandType.Text, null);
foreach (DataRow variable in tblCompanyTable.Rows)
{
producerBasic.PublishString(
Newtonsoft.Json.JsonConvert.SerializeObject(new MssUpdateAlexaCompany()
{
CompanyId = Convert.ToInt64(variable["ID"]),
Domain = Convert.ToString(variable["Domain"])
}));
}
_log.Info("Pushed all job");
}
public static void RePushThumb()
{
SqlDb sqlDb = new SqlDb(ConfigImbo.ConnectionProduct);
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo),
"Img.Product.Thumb");
sqlDb.ProcessDataTableLarge(
@"select ImageId, Id
from product
where valid = 1
order by Id", 10000, (rowData, iRow) =>
{
string imgId = Common.Obj2String(rowData["ImageId"]);
long Id = Common.Obj2Int64(rowData["Id"]);
if (!string.IsNullOrEmpty(imgId))
{
producerBasic.PublishString(new JobUploadedImg()
{
ImageId = imgId,
ProductId = Id
}.ToJson());
}
return(true);
});
}
public void SendImageIdToDelImageService(long productId, string idImageImbo, ProducerBasic producerDelByImageId)
{
int index = 0;
while (true)
{
try
{
producerDelByImageId.PublishString(idImageImbo);
break;
}
catch (Exception exception)
{
Thread.Sleep(1000);
Log.Error(
string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
productId), exception);
if (index == 5)
{
break;
}
else
{
index++;
}
}
}
}
public void Download()
{
_workers = new Worker[_workerProduct + _workerCompany + 1];
_rabbitMqServer = RabbitMQManager.GetRabbitMQServer(ConfigImages.RabbitMqServerName);
_checkErrorJobClient = new JobClient(ConfigImages.ImboExchangeImages, GroupType.Topic, ConfigImages.ImboRoutingKeyCheckErrorDownload, true, _rabbitMqServer);
for (var i = 0; i < _workerProduct; i++)
{
var worker = new Worker(ConfigImages.ImboQueueDownloadImageProduct, false, _rabbitMqServer);
_workers[i] = worker;
var workerTask = new Task(() =>
{
//JobClient send message to service upload sql & thumb của tráng
var producerUpdateImageIdSql = new ProducerBasic(_rabbitMqServer, ConfigImages.ImboExchangeImages, ConfigImages.ImboRoutingKeyUploadImageIdSql);
//var producerDelImage = new ProducerBasic(_rabbitMqServer, ConfigImages.ImboExchangeImages, ConfigImages.ImboRoutingKeyThumbImage);
worker.JobHandler = (downloadImageJob) =>
{
try
{
DownloadImageProduct(ImageProductInfo.GetDataFromMessage(downloadImageJob.Data), producerUpdateImageIdSql);
}
catch (Exception exception)
{
Log.Error("Execute Job Error.", exception);
}
return(true);
};
worker.Start();
});
workerTask.Start();
Log.InfoFormat("Worker {0} started", i);
}
}
public ConsumerDownloadHtml(RabbitMQServer rabbitmqServer, string domain) : base(rabbitmqServer, Config.QueueWaitDownloadHtml + "." + domain, false)
{
_noSqlHtml = NoSqlCommentSystem.Instance();
_queueAS = Config.QueueWaitAsComment + "." + domain;
_producerAfterDownload = new ProducerBasic(rabbitmqServer, Config.QueueWaitAsComment + "." + domain);
}
public ConsumerProductChangeToCache()
: base(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueProductChangeToCache, false)
{
_log.Info("Constructor object ConsumerProductChangeToCache");
_producerDescriptionQuang = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqProduct), ConfigRun.QueueChangeDescription);
_producerChangeDesc = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeChangeDesc, ConfigCrawler.RoutingkeyChangeDesc);
_producerClassification = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueAddClassification);
_log.Info("Success constructor object ConsumerProductChangeToCache");
}
private void btnPushCompanyInfo_Reset_Click(object sender, EventArgs e)
{
long companyInfo = QT.Entities.Common.Obj2Int64((this.gridView1.GetRow(this.gridView1.FocusedRowHandle) as DataRowView)["ID"]);
string Domain = QT.Entities.Common.Obj2String((this.gridView1.GetRow(this.gridView1.FocusedRowHandle) as DataRowView)["Domain"]);
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct), ConfigRun.QueueUpdateCompanyInfoToWeb);
producerBasic.PublishString(companyInfo.ToString());
MessageBox.Show(string.Format("Pushed company: {0}", Domain));
}
public HandlerClick(IProductProcess productProcess, ICheckLast checkLast, IProductAdapter productAdapter, IProducerService producerService,
WSS.Service.Report.ProductOnClick.Error.Model.ISettingRepository settingRepository, ICheckLinkValid checkLinkValid)
{
_productProcess = productProcess;
_checkLast = checkLast;
_productAdapter = productAdapter;
_producerService = producerService;
_checkLinkValid = checkLinkValid;
_producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(settingRepository.RabbitMq), "Product.ErrorRequest");
}
public void PushJobToTest()
{
var producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueVipCompanyFindNew);
producerBasic.PublishString(new JobCompanyCrawler()
{
CheckRunning = true,
CompanyId = 2102811518945449457
}.GetJSon());
}
private void PushResetCacheCompany(object sender, EventArgs e)
{
var lstCompany = eventGetCompanys();
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueResetCacheProduct);
foreach (var item in lstCompany)
{
producerBasic.PublishString(item.ToString());
}
MessageBox.Show(string.Format("Pushed {0} companys", lstCompany.Count));
}
private void PushResetDuplicateCompany(object sender, EventArgs e)
{
var lstCompany = eventGetCompanys();
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeResetDuplicate, ConfigCrawler.RoutingkeyResetDuplicate);
foreach (var item in lstCompany)
{
((ProducerBasic)producerBasic).PublishString(item.ToString());
}
MessageBox.Show(string.Format("Pushed {0} companys", lstCompany.Count));
}
private void ClickUpdateCompanyInfoToWeb(object sender, EventArgs e)
{
var companyIds = eventGetCompanys();
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct), "UpdateCompany.ToWeb");
foreach (var companyId in companyIds)
{
producer.PublishString(companyId.ToString());
}
MessageBox.Show(string.Format("Pushed {0}!", companyIds.Count));
}
private void PushQueueCrawlerReload(object sender, EventArgs e)
{
ProducerBasic producerReloadCompany = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeCompanyReload, ConfigCrawler.RoutingkeyCompanyReload);
foreach (var item in eventGetCompanys())
{
producerReloadCompany.PublishString(new JobCompanyCrawler()
{
CompanyId = item, CheckRunning = false
}.GetJSon());
}
}
public void PushResetCompany()
{
string arCOmpanyId = GetStrCompany;
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqCrawler), ConfigCrawler.QueueResetCacheProduct);
foreach (var VARIABLE in arCOmpanyId.Split(Common.arSplitToList, StringSplitOptions.RemoveEmptyEntries))
{
producerBasic.PublishString(new JobCompanyCrawler()
{
CheckRunning = false, CompanyId = Convert.ToInt64(VARIABLE)
}.GetJSon());
}
}
public ConsumerAsComment(RabbitMQServer rabbitmqServer)
: base(rabbitmqServer,
Config.QueueWaitAsComment, false)
{
string domain = "lazada.vn";
_noSqlHtml = NoSqlCommentSystem.Instance();
_producerComment = new ProducerBasic(rabbitmqServer, "Comment", "Comment.CommentUser.#");
using (var db = new DsConfigurationCommentTableAdapters.Configuration_CommentTableAdapter())
{
DsConfigurationComment.Configuration_CommentDataTable tbl = new DsConfigurationComment.Configuration_CommentDataTable();
db.FillByCompanyDomain(tbl, domain);
_configuration = tbl.Rows[0] as DsConfigurationComment.Configuration_CommentRow;
}
}
//public WorkerDownload()
// : base(RabbitMQManager.GetRabbitMQServer("rabbitMqCrlProperties"), "Product.Ads.Img.Wait.Download", false)
//{
// _connectionString = ConfigurationManager.AppSettings["ConnectionString"];
// //imbo
// _publicKeyImbo = ConfigurationManager.AppSettings["PublicKeyImboImageProduct"];
// _privateKeyImbo = ConfigurationManager.AppSettings["PrivateKeyImboImageProduct"];
// _userNameImbo = ConfigurationManager.AppSettings["UserNameImboImageProduct"];
// _hostImbo = ConfigurationManager.AppSettings["HostImboImageProduct"];
// _portImbo = Common.Obj2Int(ConfigurationManager.AppSettings["PortImboImageProduct"]);
// producerCountDownloaded = new ProducerBasic(RabbitMQManager.GetRabbitMQServer("rabbitMqCrlProperties"), "Product.Ads.Img.Downloaded");
// producerCountDownloadError = new ProducerBasic(RabbitMQManager.GetRabbitMQServer("rabbitMqCrlProperties"), "Product.Ads.Img.Download.Error");
//}
public WorkerDownload()
: base(RabbitMQManager.GetRabbitMQServer("rabbitMQ177"), "Product.All.Img.Wait.Download", false)
{
_connectionString = ConfigurationManager.AppSettings["ConnectionString"];
//imbo
_publicKeyImbo = ConfigurationManager.AppSettings["PublicKeyImboImageProduct"];
_privateKeyImbo = ConfigurationManager.AppSettings["PrivateKeyImboImageProduct"];
_userNameImbo = ConfigurationManager.AppSettings["UserNameImboImageProduct"];
_hostImbo = ConfigurationManager.AppSettings["HostImboImageProduct"];
_portImbo = Common.Obj2Int(ConfigurationManager.AppSettings["PortImboImageProduct"]);
producerCountDownloaded = new ProducerBasic(RabbitMQManager.GetRabbitMQServer("rabbitMQ177"), "Product.All.Img.Downloaded");
producerCountDownloadError = new ProducerBasic(RabbitMQManager.GetRabbitMQServer("rabbitMQ177"), "Product.All.Img.Download.Error");
}
public static void PushCompanyUpdateImgImbo(List <string> domain)
{
int countPush = 0;
ImageAdapterSql imageAdapterSql = new ImageAdapterSql();
ProducerBasic producerCompanyImage = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo), ConfigImbo.QueueCompanyWaitPushProductTransferImage);
List <long> companyIds = imageAdapterSql.GetCompanyIdByDomain(domain);
foreach (var companyId in companyIds)
{
countPush++;
producerCompanyImage.PublishString(new JobCmpWaitTransf()
{
CompanyId = companyId
}.GetJson());
}
}
public void Start()
{
ProducerBasic producerDelImgImbo = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo),
ConfigImbo.QueueDelImgImbo);
_sqlDb.ProcessDataTableLarge(
@"
select replace(Domain, '.', '_') as Domain, id, LogoImageId
from company
where TotalProduct>0
order by id
", 10000, (row, iRow) =>
{
string imgIdOld = Common.Obj2String(row["LogoImageId"]);
long id = Common.Obj2Int64(row["Id"]);
string domain = Common.Obj2String(row["Domain"]);
string fullLinkLog = string.Format(@"http://img.websosanh.vn/merchant/{0}.png", domain);
try
{
string imgIdNew = ImboImageService.PushFromUrl(ConfigImbo.PublicKey, ConfigImbo.PrivateKey, fullLinkLog, "logo", ConfigImbo.Host, ConfigImbo.Port);
if (!string.IsNullOrEmpty(imgIdNew))
{
if (!string.IsNullOrEmpty(imgIdOld))
{
producerDelImgImbo.PublishString(imgIdOld);
}
bool bOK = this._sqlDb.RunQuery("Update Company Set LogoImageId = @LogoImageId Where Id = @Id", CommandType.Text, new SqlParameter[]
{
SqlDb.CreateParamteterSQL("Id", id, SqlDbType.BigInt),
SqlDb.CreateParamteterSQL("LogoImageId", imgIdNew, SqlDbType.NVarChar)
});
}
}
catch (Exception ex)
{
_log.Error(ex);
}
_log.Info(string.Format("Uploaded {0} {1}", iRow, id));
return(true);
});
}
internal void PushQueueAsDocInfo()
{
ProducerBasic producerBasic =
new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigDocMan.KeyRabbitMqWaitDl),
ConfigDocMan.QueueDocInfo);
DocManAdapter docManAdapter = new DocManAdapter();
DataTable tbl = docManAdapter.GetUrls();
foreach (DataRow row in tbl.Rows)
{
producerBasic.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(new JobResetDocInfo()
{
Url = Convert.ToString(row["Url"]),
Id = Convert.ToInt64(row["Id"])
}));
}
}
private void pushReAnalysicToolStripMenuItem_Click(object sender, EventArgs e)
{
long companyId = Convert.ToInt64(gridView1.GetRowCellValue(gridView1.FocusedRowHandle, "CompanyId"));
NoSqlCommentSystem noSqlCommentSystem = NoSqlCommentSystem.Instance();
var lstJOb = noSqlCommentSystem.GetAllJobBySite(companyId);
ProducerBasic producerBasicPushAS = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(Config.RabbitMQServerComment), Config.QueueWaitAsComment);
foreach (var VARIABLE in lstJOb)
{
producerBasicPushAS.Publish(new JobWaitAS()
{
CompanyId = companyId,
Id = VARIABLE.Item1,
Url = VARIABLE.Item2,
}.ToObjMQ());
}
}
public static void UpdateError()
{
_rabbitMqServer = RabbitMQManager.GetRabbitMQServer(ConfigImages.RabbitMqServerName);
var producerUpdateImageIdSql = new ProducerBasic(_rabbitMqServer, ConfigImages.ImboExchangeImages, ConfigImages.ImboRoutingKeyUploadImageIdSql);
long CompanyID = 7501950358591465227;
var tblProduct = sqldb.GetTblData(string.Format("Select ID,ImageUrls from Product where Company = {0} and Valid = 1 and ImageUrls like '%.png' order by ID", CompanyID));
foreach (DataRow row in tblProduct.Rows)
{
try
{
long ProductID = Common.Obj2Int64(row["ID"]);
string ImageUrls = row["ImageUrls"].ToString().Trim();
DownloadImageProduct(ImageUrls, ProductID, producerUpdateImageIdSql);
Log.InfoFormat("Update image error product: {0}", ProductID);
}
catch (Exception)
{
}
}
//var tblCompany = sqldb.GetTblData("select ID from Company where status = 1 or Status = 18 or Status = 19");
//foreach (DataRow rowCompany in tblCompany.Rows)
//{
// int iCount = 1;
// long CompanyID = Common.Obj2Int64(rowCompany["ID"]);
// var tblProduct = sqldb.GetTblData(string.Format("Select ID,ImageUrls from Product where Company = {0} and Valid = 1 and ImageUrls like '%.png' order by ID", CompanyID));
// foreach (DataRow row in tblProduct.Rows)
// {
// try
// {
// long ProductID = Common.Obj2Int64(row["ID"]);
// string ImageUrls = row["ImageUrls"].ToString().Trim();
// DownloadImageProduct(ImageUrls, ProductID, producerUpdateImageIdSql);
// Log.InfoFormat("Update image error product: {0}", ProductID);
// }
// catch (Exception)
// {
// }
// }
// Log.InfoFormat("{0} Company Success:{1}", iCount, CompanyID);
// iCount++;
//}
Log.InfoFormat("Download all image Compant: {0}", CompanyID);
}
public static void PushCmp()
{
var server = RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler);
var sqldb = new SqlDb(ConfigCrawler.ConnectProduct);
var tupleSetup = new List <Tuple <string, string> >()
{
new Tuple <string, string>("00", "Vip.Cmp.Crl.Fn"),
new Tuple <string, string>("10", "Vip.Cmp.Crl.Rl"),
new Tuple <string, string>("01", "Normal.Cmp.Crl.Fn"),
new Tuple <string, string>("11", "Normal.Cmp.Crl.Rl"),
};
foreach (var tuple in tupleSetup)
{
Log.Info(string.Format("Push to {0} {1}", tuple.Item1, tuple.Item2));
var producer = new ProducerBasic(server, tuple.Item2);
var chanl = server.CreateChannel();
QueueDeclareOk dcl = chanl.QueueDeclare(tuple.Item2, true, false, false, null);
if (dcl.MessageCount < 1)
{
var tblCmpFn = sqldb.GetTblData("[prc_Company_GetCmpToPushCrl]", CommandType.StoredProcedure, new[]
{
SqlDb.CreateParamteterSQL("@TypeCrl", tuple.Item1, SqlDbType.VarChar)
});
foreach (DataRow variable in tblCmpFn.Rows)
{
long companyId = Common.Obj2Int64(variable["ID"]);
if (!RedisCompanyWaitCrawler.CheckRunningCrawler(companyId))
{
producer.PublishString(new JobCompanyCrawler()
{
CheckRunning = true,
CompanyId = companyId
}.GetJSon());
Log.Info(string.Format("Pushed for cmp: {0}", companyId));
}
else
{
Log.Info("Not push because running");
}
}
}
}
return;
}
private static void PushJobDownload(string domain)
{
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
IStorageProduct storageProduct = new StorageProduct();
int i = 0;
string DetailUrl = "";
Regex regex = new Regex(@"http://www.+html");
storageProduct.ProcessProduct(domain, (sender, product) =>
{
if (domain == "lazada.vn")
{
string urlencode = product.DetailUrl;
string urldecode = HttpUtility.UrlDecode(HttpUtility.UrlDecode(urlencode));
MatchCollection matches = regex.Matches(urldecode);
DetailUrl = matches[0].Value.ToString();
////string urlencode = product.DetailUrl;
////string urldecode = HttpUtility.UrlDecode(product.DetailUrl);
////char charRange = '?';
////int startIndex = urldecode.IndexOf(charRange) + 1;
////int endIndex = urldecode.LastIndexOf(charRange) - 1;
////int length = endIndex - startIndex + 1;
////DetailUrl = urldecode.Substring(startIndex, length).Replace("url=", "");
}
else
{
DetailUrl = product.DetailUrl;
}
producer.PublishString(new JobCrlProperties()
{
ProductId = product.Id,
DetailUrl = UtilCrl.GetUrl(DetailUrl, domain),
Domain = domain,
ClassificationId = product.ClassificationId,
Classification = product.Classification
}.GetJson());
i++;
//log.Info(string.Format("{0} {1}", i, product.Id));
log.InfoFormat("{0}: {1}", i, product.Id);
});
}
private void RunReportRunning()
{
var tokenReport = _tokenSource.Token;
Task.Factory.StartNew(() =>
{
ProducerBasic producerReportSessionRunning = null;
try
{
producerReportSessionRunning = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeSessionRunning, ConfigCrawler.RoutingkeySessionRunning);
while (true)
{
string mss =
Newtonsoft.Json.JsonConvert.SerializeObject(new ReportSessionRunning()
{
Thread = _nameThread,
CompanyId = _companyId,
Ip = Dns.GetHostName(),
Session = _session,
StartAt = _timeStart,
Type = "FindNew",
MachineCode = Server.MachineCode
});
_log.Info(string.Format("Running Findnew {0} {1} {2}'", this._companyId, (this._company != null) ? this._company.Domain : "", (DateTime.Now - this._timeStart).Minutes));
tokenReport.ThrowIfCancellationRequested();
producerReportSessionRunning.PublishString(mss, true, 300);
Thread.Sleep(20000);
}
}
catch (OperationCanceledException ex)
{
_log.Info("End thread report running");
if (producerReportSessionRunning != null)
{
producerReportSessionRunning.Dispose();
}
return;
}
catch (Exception ex)
{
// ignored
}
}, tokenReport);
}
