private void NotifyValidatedProduct(long companyID)
{
string query = @"SELECT a.ID, a.Domain, a.TotalProduct, a.MaxValid, b.MinProductToWarning, b.MaxProductToWarning
FROM Company a INNER JOIN Configuration b on a.ID = b.CompanyID
WHERE a.ID = @CompanyID";
DataTable tbl = _productAdapter.GetSqlDb().GetTblData(query, CommandType.Text, new SqlParameter[] {
SqlDb.CreateParamteterSQL("CompanyID", companyID, SqlDbType.BigInt)
});
if (tbl != null && tbl.Rows.Count > 0)
{
DataRow row = tbl.Rows[0];
int totalProduct = Common.CellToInt(row, "TotalProduct", 0);
int maxValid = Common.CellToInt(row, "MaxValid", 0);
string domain = Common.CellToString(row, "Domain", "");
int minProductToWarning = Common.CellToInt(row, "MinProductToWarning", 0);
int maxProductToWarning = Common.CellToInt(row, "MaxProductToWarning", 0);
var objData = new MssNoValidTotalCompany()
{
CompanyId = companyID,
Domain = domain,
MaxValid = maxValid,
MaxValidWarning = maxProductToWarning,
MinValidWarning = minProductToWarning,
TotalProduct = totalProduct,
MssNoValid = "Valid total"
};
if (totalProduct == 0)
{
objData.MssNoValid = "Total product = 0";
_producerNoValidTotalProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(objData));
}
else if (maxValid > 0 && Math.Abs((maxValid - totalProduct) / maxValid) > 0.2)
{
objData.MssNoValid = "Over 20%";
_producerNoValidTotalProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(objData));
}
else if (minProductToWarning > 0 && minProductToWarning > totalProduct)
{
objData.MssNoValid = "MinProduct Warning";
_producerNoValidTotalProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(objData));
}
else if (maxProductToWarning > 0 && maxProductToWarning < totalProduct)
{
objData.MssNoValid = "MaxProduct Warning";
_producerNoValidTotalProduct.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(objData));
}
_log.Info(string.Format("Company: {0} {1} TotalProduct: {2} MaxValid: {3} MaxProductToWarning: {4} MinProductToWarning: {5} Mss: {6}",
companyID, domain, totalProduct, maxValid, maxProductToWarning, minProductToWarning, objData.MssNoValid));
}
}
public static void RePushThumb()
{
SqlDb sqlDb = new SqlDb(ConfigImbo.ConnectionProduct);
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo),
"Img.Product.Thumb");
sqlDb.ProcessDataTableLarge(
@"select ImageId, Id
from product
where valid = 1
order by Id", 10000, (rowData, iRow) =>
{
string imgId = Common.Obj2String(rowData["ImageId"]);
long Id = Common.Obj2Int64(rowData["Id"]);
if (!string.IsNullOrEmpty(imgId))
{
producerBasic.PublishString(new JobUploadedImg()
{
ImageId = imgId,
ProductId = Id
}.ToJson());
}
return(true);
});
}
public void RePushJob()
{
sqlDb.ProcessDataTableLarge(@"
SELECT News_Image, News_ID
FROM [ReviewsCMS].[dbo].[NewsPublished] a
WHERE ISNULL(ImageId, '') = ''
AND ISNULL(News_Image, '') <> ''
and ISNULL(ChangeImage, 0) = 0
ORDER BY News_ID
", 10000, (row, iRow) =>
{
string newsImage = Common.Obj2String(row["News_Image"]);
long newsId = Common.Obj2Int64(row["News_ID"]);
if (!newsImage.Contains("http"))
{
newsImage = @"http://review.websosanh.net/" + newsImage;
}
pb.PublishString(new JobPublishedWaitTrans()
{
Id = newsId,
Url = newsImage
}.ToJson());
return(true);
});
}
public void ProcessJob(string str)
{
try
{
JobUploadedImg job = JobUploadedImg.FromJson(str);
string imageIdOld = _imgAdapterSql.GetImageId(job.ProductId);
if (!string.IsNullOrEmpty(imageIdOld))
{
_producerDelImbo.PublishString(imageIdOld);
}
MongoDB.Driver.IMongoClient mongoClient = new MongoClient("mongodb://172.22.1.226:27017");
var collection = mongoClient.GetDatabase("imbo").GetCollection <BsonDocument>("image");
var filter = Builders <BsonDocument> .Filter.Eq("imageIdentifier", job.ImageId);
var cursor = collection.FindSync(filter);
while (cursor.MoveNext())
{
int x = Convert.ToInt32(cursor.Current.ElementAt(0).GetValue("width"));
int y = Convert.ToInt32(cursor.Current.ElementAt(0).GetValue("height"));
_imgAdapterSql.UpdateImboProcess(job.ProductId, job.ImageId, x, y);
break;
}
//PushChangeMainInfo
RabbitMQAdapter.Instance.PushProductToQueueChangeMainInfo(new List <long>()
{
job.ProductId
});
}
catch (Exception ex)
{
log.Error(ex);
}
}
internal void PushQueueUpdateAlexaSeoSystem()
{
SqlDb _sqlDb =
new SqlDb("Data Source=192.168.100.178;Initial Catalog=QA_System;User ID=sa;Password=123456a@");
string queryCompany = @"
select ID, Name as Domain
From Web
where LastUpdateAlexa is null
--where IsCrawler = 1
";
ProducerBasic producerBasic =
new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct),
"UpdateAlexaCompanySeo");
DataTable tblCompanyTable = _sqlDb.GetTblData(queryCompany, CommandType.Text, null);
foreach (DataRow variable in tblCompanyTable.Rows)
{
producerBasic.PublishString(
Newtonsoft.Json.JsonConvert.SerializeObject(new MssUpdateAlexaCompany()
{
CompanyId = Convert.ToInt64(variable["ID"]),
Domain = Convert.ToString(variable["Domain"])
}));
}
_log.Info("Pushed all job");
}
public static void PushParseFromNoSql(IEnumerable <string> domains)
{
NoSqlAdapter noSqlAdapter = NoSqlAdapter.GetInstance();
foreach (var domain in domains)
{
Task.Factory.StartNew(() =>
{
int iCount = 0;
var producerBasicWaitPs = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(domain));
noSqlAdapter.ProcessAllIdProductByCompany(domain, (obj, productId) =>
{
iCount++;
producerBasicWaitPs.PublishString(new JobParse()
{
Id = productId
}.GetJson());
if (iCount % 10 == 0)
{
log.Info(string.Format("Pushed {0} mss of {1}", iCount, domain));
}
});
log.Info(string.Format("Pushed all data for company {0} {1}", domain, iCount));
});
}
Thread.Sleep(10000000);
}
public void SendImageIdToDelImageService(long productId, string idImageImbo, ProducerBasic producerDelByImageId)
{
int index = 0;
while (true)
{
try
{
producerDelByImageId.PublishString(idImageImbo);
break;
}
catch (Exception exception)
{
Thread.Sleep(1000);
Log.Error(
string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
productId), exception);
if (index == 5)
{
break;
}
else
{
index++;
}
}
}
}
public override void ProcessMessage(BasicDeliverEventArgs message)
{
try
{
var pt = ProductEntity.GetFromJson(message.Body);
if (!string.IsNullOrEmpty(pt.Session))
{
_noSqlHistoryCrawler.InsertProduct(pt);
_producerHistoryProduct.PublishString(
Newtonsoft.Json.JsonConvert.SerializeObject(new HistoryProductSorl()
{
DateUpdate = Convert.ToInt32(pt.LastUpdate.ToString("yyyyMMdd")),
CompanyId = pt.CompanyId,
Id = pt.ID + ":" + pt.Session,
ProductId = pt.ID,
LastUpdate = pt.LastUpdate.Ticks,
Session = pt.Session,
DetailUrl = pt.DetailUrl,
TypeRun = 1,
}));
}
}
catch (Exception ex)
{
_log.Error(ex);
}
this.GetChannel().BasicAck(message.DeliveryTag, true);
}
//public void ThumbImageService(long productId, string idImbo, ProducerBasic producerThumbImage)
//{
// int index = 0;
// while (_isRunning)
// {
// try
// {
// producerThumbImage.PublishString(new JobWaitThumb()
// {
// ImageId = idImbo,
// Sizes = _widthHeightImages
// }.ToJson());
// break;
// }
// catch (Exception exception)
// {
// Thread.Sleep(600000);
// Log.Error(
// string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
// productId), exception);
// if (index == 5)
// break;
// else
// index++;
// }
// }
//}
public void UpdateImageIdSqlService(long productId, string idImageImbo, ProducerBasic producerUpdateImageIdSql)
{
int index = 0;
while (_isRunning)
{
try
{
producerUpdateImageIdSql.PublishString(new JobUploadedImg()
{
ImageId = idImageImbo,
ProductId = productId
}.ToJson());
break;
}
catch (Exception exception)
{
Thread.Sleep(600000);
Log.Error(
string.Format("Product: ID = {0} Send message to service check error download image. Thread Sleep 10p",
productId), exception);
if (index == 5)
{
break;
}
else
{
index++;
}
}
}
}
public override void ProcessMessage(RabbitMQ.Client.Events.BasicDeliverEventArgs message)
{
var jobDownloadHtml = JobDownloadHtml.FromArByte(message.Body);
if (jobDownloadHtml != null)
{
var url = jobDownloadHtml.Url;
var html = HttpUtility.HtmlDecode(GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(jobDownloadHtml.Url, 45, 2));
if (!string.IsNullOrEmpty(html))
{
_noSqlHtml.SaveHtm(jobDownloadHtml.Id, jobDownloadHtml.CompanyId, html, jobDownloadHtml.Url);
_producerAfterDownload.PublishString(new JobWaitAS()
{
Id = jobDownloadHtml.Id, CompanyId = jobDownloadHtml.CompanyId, Url = url
}.ToJson());
_log.Info(string.Format("Download html for job:{0}", jobDownloadHtml.ToJSON()));
}
else
{
_log.Info(string.Format("Can't download html of job:{0}", jobDownloadHtml.ToJSON()));
}
}
else
{
_log.Info("JobErrorParse");
}
GetChannel().BasicAck(message.DeliveryTag, true);
}
public void PushBack(string imgId)
{
pb.PublishString(new JobDelImgImbo()
{
ImageId = imgId
}.ToJson());
}
public static void PushDownloadHtml(IEnumerable <string> domains)
{
SqlDb sql = new SqlDb(ConfigStatic.ProductConnection);
ProductAdapter productAdapter = new ProductAdapter(sql);
foreach (var domain in domains)
{
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
string queryData = string.Format(@"
Select Id, DetailUrl
From Product
Where Company = {0}
Order by Id
"
, productAdapter.GetCompanyIdByDomain(domain));
sql.ProcessDataTableLarge(queryData,
10000, (Row, iRow) =>
{
producerBasic.PublishString(new JobDownloadHtml()
{
ProductId = Convert.ToInt64(Row["Id"]),
DetailUrl = Convert.ToString(Row["DetailUrl"]),
Domain = domain
}.GetJson());
});
}
}
private void btnPushCompanyInfo_Reset_Click(object sender, EventArgs e)
{
long companyInfo = QT.Entities.Common.Obj2Int64((this.gridView1.GetRow(this.gridView1.FocusedRowHandle) as DataRowView)["ID"]);
string Domain = QT.Entities.Common.Obj2String((this.gridView1.GetRow(this.gridView1.FocusedRowHandle) as DataRowView)["Domain"]);
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct), ConfigRun.QueueUpdateCompanyInfoToWeb);
producerBasic.PublishString(companyInfo.ToString());
MessageBox.Show(string.Format("Pushed company: {0}", Domain));
}
public void PushJobToTest()
{
var producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueVipCompanyFindNew);
producerBasic.PublishString(new JobCompanyCrawler()
{
CheckRunning = true,
CompanyId = 2102811518945449457
}.GetJSon());
}
private void PushResetCacheCompany(object sender, EventArgs e)
{
var lstCompany = eventGetCompanys();
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.QueueResetCacheProduct);
foreach (var item in lstCompany)
{
producerBasic.PublishString(item.ToString());
}
MessageBox.Show(string.Format("Pushed {0} companys", lstCompany.Count));
}
private void ClickUpdateCompanyInfoToWeb(object sender, EventArgs e)
{
var companyIds = eventGetCompanys();
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqProduct), "UpdateCompany.ToWeb");
foreach (var companyId in companyIds)
{
producer.PublishString(companyId.ToString());
}
MessageBox.Show(string.Format("Pushed {0}!", companyIds.Count));
}
private void PushQueueCrawlerReload(object sender, EventArgs e)
{
ProducerBasic producerReloadCompany = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeCompanyReload, ConfigCrawler.RoutingkeyCompanyReload);
foreach (var item in eventGetCompanys())
{
producerReloadCompany.PublishString(new JobCompanyCrawler()
{
CompanyId = item, CheckRunning = false
}.GetJSon());
}
}
private void AnalysicHtmlForCompany(JobWaitAS job, HtmlDocument document)
{
List <Comment> lstComment = _pareseComment.ParseComment(_configuration, document, job);
if (lstComment.Count > 0)
{
_producerComment.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(lstComment));
//_noSqlHtml.SaveComment(job.Id, job.CompanyId, lstComment);
string url = (job.Url == null) ? "" : job.Url;
_log.Info(string.Format("Extract {0} comment from id {1} {2}", lstComment.Count, job.Id, url));
}
}
public void PushResetCompany()
{
string arCOmpanyId = GetStrCompany;
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigRun.KeyRabbitMqCrawler), ConfigCrawler.QueueResetCacheProduct);
foreach (var VARIABLE in arCOmpanyId.Split(Common.arSplitToList, StringSplitOptions.RemoveEmptyEntries))
{
producerBasic.PublishString(new JobCompanyCrawler()
{
CheckRunning = false, CompanyId = Convert.ToInt64(VARIABLE)
}.GetJSon());
}
}
private void Crawl()
{
try
{
LoadQueue();
while (true)
{
Token.ThrowIfCancellationRequested();
if (_linksQueue.Count == 0)
{
_typeEnd = TypeEnd.Success;
break;
}
else if ((DateTime.Now - _timeStart).TotalHours > MaxHourReload)
{
_typeEnd = TypeEnd.OverTime;
break;
}
else
{
var job = _linksQueue.Dequeue();
int statusProcess = ProcessJob(job);
string strLog = string.Format("ss: {0} cQ: {1} tP: {2} cV: {3} pt: {4} {5} sst: {6} cmp: {7} {8}", _session, _linksQueue.Count, _company.TotalProduct, _countVisited, job.ProductId,
job.url, statusProcess, _companyId,
(this._company == null) ? "" : this._company.Domain);
if (EventReportRun != null)
{
EventReportRun(strLog);
}
_log.Info(strLog);
if (_linksQueue.Count == 0)
{
LoadQueue();
}
}
}
}
catch (OperationCanceledException)
{
_typeEnd = TypeEnd.Immediate;
_producerPushCompanyReload.PublishString(_companyId.ToString(), true, 0);
End();
throw;
}
catch (Exception ex)
{
_typeEnd = TypeEnd.Error;
_log.Error(ex);
}
End();
}
public void ProcessJob(string str)
{
JobUploadedImg job = JobUploadedImg.FromJson(str);
string imageIdOld = _imgAdapterSql.GetImageId(job.ProductId);
if (!string.IsNullOrEmpty(imageIdOld))
{
_producerDelImbo.PublishString(new JobDelImgImbo()
{
ImageId = imageIdOld
}.ToJson());
}
_imgAdapterSql.UpdateImboProcess(job.ProductId, job.ImageId, 0, 0);
}
public static void PushCompanyUpdateImgImbo(List <string> domain)
{
int countPush = 0;
ImageAdapterSql imageAdapterSql = new ImageAdapterSql();
ProducerBasic producerCompanyImage = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo), ConfigImbo.QueueCompanyWaitPushProductTransferImage);
List <long> companyIds = imageAdapterSql.GetCompanyIdByDomain(domain);
foreach (var companyId in companyIds)
{
countPush++;
producerCompanyImage.PublishString(new JobCmpWaitTransf()
{
CompanyId = companyId
}.GetJson());
}
}
public void Start()
{
ProducerBasic producerDelImgImbo = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigImbo.KeyRabbitMqTransferImbo),
ConfigImbo.QueueDelImgImbo);
_sqlDb.ProcessDataTableLarge(
@"
select replace(Domain, '.', '_') as Domain, id, LogoImageId
from company
where TotalProduct>0
order by id
", 10000, (row, iRow) =>
{
string imgIdOld = Common.Obj2String(row["LogoImageId"]);
long id = Common.Obj2Int64(row["Id"]);
string domain = Common.Obj2String(row["Domain"]);
string fullLinkLog = string.Format(@"http://img.websosanh.vn/merchant/{0}.png", domain);
try
{
string imgIdNew = ImboImageService.PushFromUrl(ConfigImbo.PublicKey, ConfigImbo.PrivateKey, fullLinkLog, "logo", ConfigImbo.Host, ConfigImbo.Port);
if (!string.IsNullOrEmpty(imgIdNew))
{
if (!string.IsNullOrEmpty(imgIdOld))
{
producerDelImgImbo.PublishString(imgIdOld);
}
bool bOK = this._sqlDb.RunQuery("Update Company Set LogoImageId = @LogoImageId Where Id = @Id", CommandType.Text, new SqlParameter[]
{
SqlDb.CreateParamteterSQL("Id", id, SqlDbType.BigInt),
SqlDb.CreateParamteterSQL("LogoImageId", imgIdNew, SqlDbType.NVarChar)
});
}
}
catch (Exception ex)
{
_log.Error(ex);
}
_log.Info(string.Format("Uploaded {0} {1}", iRow, id));
return(true);
});
}
internal void PushQueueAsDocInfo()
{
ProducerBasic producerBasic =
new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigDocMan.KeyRabbitMqWaitDl),
ConfigDocMan.QueueDocInfo);
DocManAdapter docManAdapter = new DocManAdapter();
DataTable tbl = docManAdapter.GetUrls();
foreach (DataRow row in tbl.Rows)
{
producerBasic.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(new JobResetDocInfo()
{
Url = Convert.ToString(row["Url"]),
Id = Convert.ToInt64(row["Id"])
}));
}
}
private void CheckDuplicate(ProductEntity product)
{
if (_dicDuplicate.ContainsKey(product.GetHashDuplicate()) &&
_dicDuplicate[product.GetHashDuplicate()] != product.ID)
{
product.StatusChange.IsDuplicate = true;
_producerDuplicateProduct.PublishString(
Newtonsoft.Json.JsonConvert.SerializeObject(new ProductDuplicate()
{
CId = _companyId,
Id = product.ID,
Hash = product.GetHashDuplicate(),
IdDup = _dicDuplicate[product.GetHashDuplicate()],
Url = product.DetailUrl
}), true, 0);
}
}
public static void PushCmp()
{
var server = RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler);
var sqldb = new SqlDb(ConfigCrawler.ConnectProduct);
var tupleSetup = new List <Tuple <string, string> >()
{
new Tuple <string, string>("00", "Vip.Cmp.Crl.Fn"),
new Tuple <string, string>("10", "Vip.Cmp.Crl.Rl"),
new Tuple <string, string>("01", "Normal.Cmp.Crl.Fn"),
new Tuple <string, string>("11", "Normal.Cmp.Crl.Rl"),
};
foreach (var tuple in tupleSetup)
{
Log.Info(string.Format("Push to {0} {1}", tuple.Item1, tuple.Item2));
var producer = new ProducerBasic(server, tuple.Item2);
var chanl = server.CreateChannel();
QueueDeclareOk dcl = chanl.QueueDeclare(tuple.Item2, true, false, false, null);
if (dcl.MessageCount < 1)
{
var tblCmpFn = sqldb.GetTblData("[prc_Company_GetCmpToPushCrl]", CommandType.StoredProcedure, new[]
{
SqlDb.CreateParamteterSQL("@TypeCrl", tuple.Item1, SqlDbType.VarChar)
});
foreach (DataRow variable in tblCmpFn.Rows)
{
long companyId = Common.Obj2Int64(variable["ID"]);
if (!RedisCompanyWaitCrawler.CheckRunningCrawler(companyId))
{
producer.PublishString(new JobCompanyCrawler()
{
CheckRunning = true,
CompanyId = companyId
}.GetJSon());
Log.Info(string.Format("Pushed for cmp: {0}", companyId));
}
else
{
Log.Info("Not push because running");
}
}
}
}
return;
}
private void RunReportRunning()
{
var tokenReport = _tokenSource.Token;
Task.Factory.StartNew(() =>
{
ProducerBasic producerReportSessionRunning = null;
try
{
producerReportSessionRunning = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), ConfigCrawler.ExchangeSessionRunning, ConfigCrawler.RoutingkeySessionRunning);
while (true)
{
string mss =
Newtonsoft.Json.JsonConvert.SerializeObject(new ReportSessionRunning()
{
Thread = _nameThread,
CompanyId = _companyId,
Ip = Dns.GetHostName(),
Session = _session,
StartAt = _timeStart,
Type = "FindNew",
MachineCode = Server.MachineCode
});
_log.Info(string.Format("Running Findnew {0} {1} {2}'", this._companyId, (this._company != null) ? this._company.Domain : "", (DateTime.Now - this._timeStart).Minutes));
tokenReport.ThrowIfCancellationRequested();
producerReportSessionRunning.PublishString(mss, true, 300);
Thread.Sleep(20000);
}
}
catch (OperationCanceledException ex)
{
_log.Info("End thread report running");
if (producerReportSessionRunning != null)
{
producerReportSessionRunning.Dispose();
}
return;
}
catch (Exception ex)
{
// ignored
}
}, tokenReport);
}
private static void PushJobDownload(string domain)
{
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
IStorageProduct storageProduct = new StorageProduct();
int i = 0;
string DetailUrl = "";
Regex regex = new Regex(@"http://www.+html");
storageProduct.ProcessProduct(domain, (sender, product) =>
{
if (domain == "lazada.vn")
{
string urlencode = product.DetailUrl;
string urldecode = HttpUtility.UrlDecode(HttpUtility.UrlDecode(urlencode));
MatchCollection matches = regex.Matches(urldecode);
DetailUrl = matches[0].Value.ToString();
////string urlencode = product.DetailUrl;
////string urldecode = HttpUtility.UrlDecode(product.DetailUrl);
////char charRange = '?';
////int startIndex = urldecode.IndexOf(charRange) + 1;
////int endIndex = urldecode.LastIndexOf(charRange) - 1;
////int length = endIndex - startIndex + 1;
////DetailUrl = urldecode.Substring(startIndex, length).Replace("url=", "");
}
else
{
DetailUrl = product.DetailUrl;
}
producer.PublishString(new JobCrlProperties()
{
ProductId = product.Id,
DetailUrl = UtilCrl.GetUrl(DetailUrl, domain),
Domain = domain,
ClassificationId = product.ClassificationId,
Classification = product.Classification
}.GetJson());
i++;
//log.Info(string.Format("{0} {1}", i, product.Id));
log.InfoFormat("{0}: {1}", i, product.Id);
});
}
private void Analysic(JobFindNew jobCrawl, HtmlDocument doc)
{
if (_company.Status == Common.CompanyStatus.TIN)
{
var product = new Product();
product.Analytics(doc, jobCrawl.Url, _config, false, _company.Domain);
}
else
{
var product = new ProductEntity();
var productParse = new ProductParse();
productParse.Analytics(product, doc, jobCrawl.Url, _config, _company.Domain);
if (product.IsSuccessData(_config.CheckPrice))
{
product.Valid = false;
if (!IsExistsProduct(product.ID))
{
if (!_dicDuplicate.ContainsKey(product.GetHashDuplicate()))
{
product.StatusChange.IsNew = true;
PushChangeProduct(product);
_dicDuplicate.Add(product.GetHashDuplicate(), product.ID);
_crcProductOldGroup.Add(product.ID);
_countNewProduct++;
}
else
{
_producerDuplicateProduct.PublishString(
Newtonsoft.Json.JsonConvert.SerializeObject(new ProductDuplicate()
{
CId = _companyId,
Id = product.ID,
Hash = product.GetHashDuplicate(),
IdDup = _dicDuplicate[product.GetHashDuplicate()],
Url = product.DetailUrl
}), true);
}
}
}
}
}
public void PushQueueAs()
{
ProducerBasic producerBasic =
new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigDocMan.KeyRabbitMqWaitDl),
ConfigDocMan.QueueDl);
for (int i = 1; i < 100000; i++)
{
producerBasic.PublishString(Newtonsoft.Json.JsonConvert.SerializeObject(new JobQueue()
{
Deep = 0,
Url =
string.Format("http://moj.gov.vn/vbpq/Lists/Vn%20bn%20php%20lut/View_Detail.aspx?ItemID={0}", i)
}));
if (i % 10 == 0)
{
log.Info(i);
}
}
}