public static void Main() { ILog log = log4net.LogManager.GetLogger(typeof(Program)); var rabbitMQServer = RabbitMQManager.GetRabbitMQServer("rabbitMQ177"); var worker = new Worker("ChangePriceQueue.LogToSql", false, rabbitMQServer); Task workerTask = new Task(() => { string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200"; CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL)); log.Info("Start consumer!"); worker.JobHandler = (downloadImageJob) => { log.Info("Get job from MQ"); try { Encoding enc = new UTF8Encoding(true, true); string strData = enc.GetString(downloadImageJob.Data); JobRabbitChangePrice job = JsonConvert.DeserializeObject <JobRabbitChangePrice>(strData); crawlerProductAdapter.SaveLog(job.ProductID, job.NewPrice, job.OldPrice); log.Info(string.Format("Log for {0}", strData)); return(true); } catch (Exception ex01) { log.Error("Exception:", ex01); return(true); } }; worker.Start(); }); workerTask.Start(); Console.ReadLine(); }
protected override void OnStart(string[] args) { log.Info("Start service"); try { InitializeComponent(); cancelTokenSource = new CancellationTokenSource(); string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; workers = new Worker[workerCount]; rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200"; string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"; CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL)); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection)); Encoding enc = new UTF8Encoding(true, true); for (int i = 0; i < workerCount; i++) { log.InfoFormat("Start worker {i}", i.ToString()); var worker = new Worker(AddProductToSqlJobName, false, rabbitMQServer); workers[i] = worker; var token = this.cancelTokenSource.Token; Task workerTask = new Task(() => { worker.JobHandler = (downloadImageJob) => { try { token.ThrowIfCancellationRequested(); string strData = enc.GetString(downloadImageJob.Data); JobRabbitAddProduct job = JsonConvert.DeserializeObject <JobRabbitAddProduct>(strData); if (job.DateAdd == DateTime.MinValue) { job.DateAdd = productAdapter.GetLastChangeOfProduct(job.ProductID); } crawlerProductAdapter.SaveLogAddProduct(job.ProductID, job.DetailUrl, job.IDCompnay, job.Name, job.DateAdd); log.Info(string.Format("Log for {0}", strData)); return(true); } catch (OperationCanceledException opc) { log.Info("End worker"); return(false); } }; worker.Start(); }, token); workerTask.Start(); log.InfoFormat("Worker {0} started", i); } } catch (Exception ex) { log.Error("Start error", ex); throw; } }
public void Start() { log.Info("Start LogAddProductToSqlServercs"); var rabbitMQServer = RabbitMQManager.GetRabbitMQServer("rabbitMQ177"); worker = new Worker("AddProductQueue.Product", false, rabbitMQServer); string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200"; string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"; CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL)); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection)); log.Info("Start consumer!"); worker.JobHandler = (downloadImageJob) => { log.Info("Get job from MQ"); try { Encoding enc = new UTF8Encoding(true, true); string strData = enc.GetString(downloadImageJob.Data); JobRabbitAddProduct job = JsonConvert.DeserializeObject <JobRabbitAddProduct>(strData); if (job.DateAdd == DateTime.MinValue) { job.DateAdd = productAdapter.GetLastChangeOfProduct(job.ProductID); } crawlerProductAdapter.SaveLogAddProduct(job.ProductID, job.DetailUrl, job.IDCompnay, job.Name, job.DateAdd); log.Info(string.Format("Log for {0}", strData)); return(true); } catch (Exception ex01) { log.Error("Exception:", ex01); return(true); } }; worker.Start(); }
protected override void OnStart(string[] args) { log.Info("Start service"); try { InitializeComponent(); cancelTokenSource = new CancellationTokenSource(); string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; workers = new Worker[workerCount]; rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); string connectToSQL = @"Data Source=172.22.30.86,1455;Initial Catalog=QT_2;Persist Security Info=True;User ID=qt_vn;Password=@F4sJ=l9/ryJt9MT;connection timeout=200"; string connectToConnection = @"Data Source=42.112.28.93;Initial Catalog=QT_2;Persist Security Info=True;User ID=wss_price;Password=HzlRt4$$axzG-*UlpuL2gYDu;connection timeout=200"; CrawlerProductAdapter crawlerProductAdapter = new CrawlerProductAdapter(new SqlDb(connectToSQL)); ProductAdapter productAdapter = new ProductAdapter(new SqlDb(connectToConnection)); for (int i = 0; i < workerCount; i++) { log.InfoFormat("Start worker {i}", i.ToString()); var worker = new Worker(AddProductToSqlJobName, false, rabbitMQServer); workers[i] = worker; var token = this.cancelTokenSource.Token; Task workerTask = new Task(() => { worker.JobHandler = (downloadImageJob) => { try { token.ThrowIfCancellationRequested(); QT.Entities.CrawlerProduct.RabbitMQ.MsSaveProduct Mss = QT.Entities.CrawlerProduct.RabbitMQ.MsSaveProduct.GetDataFromMessage(downloadImageJob.Data); string Url = Mss.Url; string Domain = QT.Entities.Common.GetDomainFromUrl(Url); long CompanyID = QT.Entities.Common.GetIDCompany(Domain); QT.Entities.Configuration config = new QT.Entities.Configuration(CompanyID); if (_company.Status == Common.CompanyStatus.WEB_CRAWLERDOMAIN) { List <QT.Entities.Company> ls = new List <QT.Entities.Company>(); QT.Entities.CrawlerDomain obj = new CrawlerDomain(); string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(Url.Trim(), 15, 1); GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument(); html = html.Replace("<form", "<div"); html = html.Replace("</form", "</div"); doc.LoadHtml(html); } else { int numberItemSaved = 0; string[] arLink = Url.Trim().Split(SqlDb.arSplit, StringSplitOptions.RemoveEmptyEntries); foreach (var item in arLink) { QT.Entities.Product _product = new Product(); string html = GABIZ.Base.HtmlUrl.HTMLTransmitter.getHTML(item, 45, 2); if (config.ContentAnanyticXPath.Count >= 1) { int i1 = 0, i2 = 0; i1 = html.IndexOf(config.ContentAnanyticXPath[0]); if (i1 >= 0) { html = html.Substring(i1); if (config.ContentAnanyticXPath.Count >= 2) { i2 = html.IndexOf(config.ContentAnanyticXPath[1]); if (i2 >= 0) { html = html.Substring(0, i2 + config.ContentAnanyticXPath[1].Length); } } } html = html.Replace("<form", "<div"); html = html.Replace("</form", "</div"); html = Common.TidyCleanR(html); } _htmlSource = html; GABIZ.Base.HtmlAgilityPack.HtmlDocument doc = new GABIZ.Base.HtmlAgilityPack.HtmlDocument(); html = html.Replace("<form", "<div"); html = html.Replace("</form", "</div"); doc.LoadHtml(html); List <Product> lstUpdateProduct = new List <Product>(); List <Product> lstInsertProduct = new List <Product>(); _product.Analytics(doc, item, config, true, _company.Domain); if (_product != null && _product.IsSuccessData(config.CheckPrice)) { numberItemSaved++; if (productAdapter.CheckExistInDb(_product.ID)) { lstUpdateProduct.Add(_product); } else { lstInsertProduct.Add(_product); } productAdapter.UpdateProductsChangeToDb(lstUpdateProduct); productAdapter.InsertListProduct(lstInsertProduct); productAdapter.PushQueueIndexCompany(config.CompanyID); productAdapter.PushQueueChangeChangeImage(new MQChangeImage() { ProductID = _product.ID, Type = 1 }); log.InfoFormat("Saved {0} item product!", _product.Name); } } } return(true); } catch (OperationCanceledException opc) { log.Info("End worker"); return(false); } }; worker.Start(); }, token); workerTask.Start(); log.InfoFormat("Worker {0} started", i); } } catch (Exception ex) { log.Error("Start error", ex); throw; } }