public static void Main()
{
ILog log = log4net.LogManager.GetLogger(typeof(Program));
var rabbitMQServer = RabbitMQManager.GetRabbitMQServer("rabbitMQ177");
var worker = new Worker("ChangePriceQueue.LogToSql", false, rabbitMQServer);
Task workerTask = new Task(() =>
{
string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200";
CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL));
log.Info("Start consumer!");
worker.JobHandler = (downloadImageJob) =>
{
log.Info("Get job from MQ");
try
{
Encoding enc = new UTF8Encoding(true, true);
string strData = enc.GetString(downloadImageJob.Data);
JobRabbitChangePrice job = JsonConvert.DeserializeObject <JobRabbitChangePrice>(strData);
crawlerProductAdapter.SaveLog(job.ProductID, job.NewPrice, job.OldPrice);
log.Info(string.Format("Log for {0}", strData));
return(true);
}
catch (Exception ex01)
{
log.Error("Exception:", ex01);
return(true);
}
};
worker.Start();
});
workerTask.Start();
Console.ReadLine();
}
protected override void OnStart(string[] args)
{
log.Info("Start service");
try
{
InitializeComponent();
cancelTokenSource = new CancellationTokenSource();
string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"];
workers = new Worker[workerCount];
rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName);
string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200";
string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200";
CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL));
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection));
Encoding enc = new UTF8Encoding(true, true);
for (int i = 0; i < workerCount; i++)
{
log.InfoFormat("Start worker {i}", i.ToString());
var worker = new Worker(AddProductToSqlJobName, false, rabbitMQServer);
workers[i] = worker;
var token = this.cancelTokenSource.Token;
Task workerTask = new Task(() =>
{
worker.JobHandler = (downloadImageJob) =>
{
try
{
token.ThrowIfCancellationRequested();
string strData = enc.GetString(downloadImageJob.Data);
JobRabbitAddProduct job = JsonConvert.DeserializeObject <JobRabbitAddProduct>(strData);
if (job.DateAdd == DateTime.MinValue)
{
job.DateAdd = productAdapter.GetLastChangeOfProduct(job.ProductID);
}
crawlerProductAdapter.SaveLogAddProduct(job.ProductID, job.DetailUrl, job.IDCompnay, job.Name, job.DateAdd);
log.Info(string.Format("Log for {0}", strData));
return(true);
}
catch (OperationCanceledException opc)
{
log.Info("End worker");
return(false);
}
};
worker.Start();
}, token);
workerTask.Start();
log.InfoFormat("Worker {0} started", i);
}
}
catch (Exception ex)
{
log.Error("Start error", ex);
throw;
}
}
public void Start()
{
log.Info("Start LogAddProductToSqlServercs");
var rabbitMQServer = RabbitMQManager.GetRabbitMQServer("rabbitMQ177");
worker = new Worker("AddProductQueue.Product", false, rabbitMQServer);
string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200";
string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200";
CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL));
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection));
log.Info("Start consumer!");
worker.JobHandler = (downloadImageJob) =>
{
log.Info("Get job from MQ");
try
{
Encoding enc = new UTF8Encoding(true, true);
string strData = enc.GetString(downloadImageJob.Data);
JobRabbitAddProduct job = JsonConvert.DeserializeObject <JobRabbitAddProduct>(strData);
if (job.DateAdd == DateTime.MinValue)
{
job.DateAdd = productAdapter.GetLastChangeOfProduct(job.ProductID);
}
crawlerProductAdapter.SaveLogAddProduct(job.ProductID, job.DetailUrl, job.IDCompnay, job.Name, job.DateAdd);
log.Info(string.Format("Log for {0}", strData));
return(true);
}
catch (Exception ex01)
{
log.Error("Exception:", ex01);
return(true);
}
};
worker.Start();
}
protected override void OnStart(string[] args)
{
log.Info("Start service");
try
{
InitializeComponent();
cancelTokenSource = new CancellationTokenSource();
string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"];
workers = new Worker[workerCount];
rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName);
string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200";
string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200";
CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL));
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection));
for (int i = 0; i < workerCount; i++)
{
log.InfoFormat("Start worker {i}", i.ToString());
var worker = new Worker(AddProductToSqlJobName, false, rabbitMQServer);
workers[i] = worker;
var token = this.cancelTokenSource.Token;
Task workerTask = new Task(() =>
{
worker.JobHandler = (downloadImageJob) =>
{
try
{
token.ThrowIfCancellationRequested();
QT.Entities.CrawlerProduct.RabbitMQ.MsSaveProduct Mss = QT.Entities.CrawlerProduct.RabbitMQ.MsSaveProduct.GetDataFromMessage(downloadImageJob.Data);
string Url = Mss.Url;
string Domain = QT.Entities.Common.GetDomainFromUrl(Url);
long CompanyID = QT.Entities.Common.GetIDCompany(Domain);
QT.Entities.Configuration config = new QT.Entities.Configuration(CompanyID);
if (_company.Status == Common.CompanyStatus.WEB_CRAWLERDOMAIN)
{
List <QT.Entities.Company> ls = new List <QT.Entities.Company>();
QT.Entities.CrawlerDomain obj = new CrawlerDomain();
string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(Url.Trim(), 15, 1);
GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument();
html = html.Replace("<form", "<div");
html = html.Replace("</form", "</div");
doc.LoadHtml(html);
}
else
{
int numberItemSaved = 0;
string[] arLink = Url.Trim().Split(SqlDb.arSplit, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in arLink)
{
QT.Entities.Product _product = new Product();
string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(item, 45, 2);
if (config.ContentAnanyticXPath.Count >= 1)
{
int i1 = 0, i2 = 0;
i1 = html.IndexOf(config.ContentAnanyticXPath[0]);
if (i1 >= 0)
{
html = html.Substring(i1);
if (config.ContentAnanyticXPath.Count >= 2)
{
i2 = html.IndexOf(config.ContentAnanyticXPath[1]);
if (i2 >= 0)
{
html = html.Substring(0, i2 + config.ContentAnanyticXPath[1].Length);
}
}
}
html = html.Replace("<form", "<div");
html = html.Replace("</form", "</div");
html = Common.TidyCleanR(html);
}
_htmlSource = html;
GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument();
html = html.Replace("<form", "<div");
html = html.Replace("</form", "</div");
doc.LoadHtml(html);
List <Product> lstUpdateProduct = new List <Product>();
List <Product> lstInsertProduct = new List <Product>();
_product.Analytics(doc, item, config, true, _company.Domain);
if (_product != null && _product.IsSuccessData(config.CheckPrice))
{
numberItemSaved++;
if (productAdapter.CheckExistInDb(_product.ID))
{
lstUpdateProduct.Add(_product);
}
else
{
lstInsertProduct.Add(_product);
}
productAdapter.UpdateProductsChangeToDb(lstUpdateProduct);
productAdapter.InsertListProduct(lstInsertProduct);
productAdapter.PushQueueIndexCompany(config.CompanyID);
productAdapter.PushQueueChangeChangeImage(new MQChangeImage()
{
ProductID = _product.ID,
Type = 1
});
log.InfoFormat("Saved {0} item product!", _product.Name);
}
}
}
return(true);
}
catch (OperationCanceledException opc)
{
log.Info("End worker");
return(false);
}
};
worker.Start();
}, token);
workerTask.Start();
log.InfoFormat("Worker {0} started", i);
}
}
catch (Exception ex)
{
log.Error("Start error", ex);
throw;
}
}
