public void Start() { _log.InfoFormat("Start consumer: {0} at queue {1}", "", _queueName); while (true) { try { _cancellationToken.ThrowIfCancellationRequested(); JobCompanyCrawler jobCompany = GetJob(); if (jobCompany != null) { if (jobCompany.CheckRunning && CheckOtherRunning(jobCompany.CompanyId)) { _log.Info(string.Format("Other running company: {0}", jobCompany)); } else if (!_productAdapter.AllowCrawlReload(jobCompany.CompanyId)) { _log.Info("Not allow crawler"); _redisWaitCrawler.DeleteWaitReload(new List <long> { jobCompany.CompanyId }); } else { using (var worker = new WorkerReload(jobCompany.CompanyId, _nameThread)) { worker.StartCrawler(); } } } else { _log.Info("No company to crawler"); Thread.Sleep(60000); } } catch (OperationCanceledException task) { break; } catch (Exception ex1) { _log.Error(ex1); Thread.Sleep(10000); } } }
private static void Main(string[] args) { Server.ConnectionString = ConfigCrawler.ConnectProduct; Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler; Server.LogConnectionString = ConfigCrawler.ConnectLog; ProductAdapter productAdapter = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct)); string strParaInput = (args.Length == 0) ? Console.ReadLine() : string.Join(" ", args); strParaInput = @"-c svptccache -u http://maxmobile.vn/dien-thoai/lg-g5-cu.html"; ParameterManager p = ParameterManager.Parse(strParaInput); //p.SubCmd = "svptccache"; if (p.SubCmd == "crlrl") { string domain = p.Parameters["dm"][0]; long idCOmpany = productAdapter.GetCompanyIDFromDomain(domain); using (var worker = new WorkerReload(idCOmpany, new CancellationToken(), "", true)) { worker.StartCrawler(); } } else if (p.SubCmd == "crlfn") { } else if (p.SubCmd == "svcudclss") { var c = new ConsumerClassificationToSql(); c.StartConsume(); } else if (p.SubCmd == "svptccache") { var c = new ConsumerProductChangeToCache(); c.StartConsume(); } else if (p.SubCmd == "svudprsql") { var c = new ConsumerProductChangeToSql(); c.StartConsume(); } else if (p.SubCmd == "prpt") { ProductAdapter pta = new ProductAdapter(new SqlDb(Server.ConnectionString)); string url = p.Parameters["u"][0].ToString(); url = "http://maxmobile.vn/dien-thoai/lg-g5-cu.html"; Uri uri = new Uri(url); string domain = Common.GetDomainFromUrl(uri); long idCompanyId = pta.GetCompanyIdByDomain(domain); IDownloadHtml downloader = new DownloadHtmlCrawler(); Configuration config = new Configuration(); HtmlDocument htmlDocument = new HtmlDocument(); var ext = new WebExceptionStatus(); string html = downloader.GetHTML(url, 45, 2, out ext); htmlDocument.LoadHtml(html); ProductEntity pte = new ProductEntity(); ProductParse ppr = new ProductParse(); ppr.Analytics(pte, htmlDocument, url, config, domain); } //ConsumerSaveEndSession consumerSaveEndSession = new ConsumerSaveEndSession(); //consumerSaveEndSession.StartConsume(); }
private static void Main(string[] args) { try { Server.ConnectionString = ConfigCrawler.ConnectProduct; Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler; Server.LogConnectionString = ConfigCrawler.ConnectLog; InitQueue isq = new InitQueue(); isq.Start(); if (args == null || args.Length == 0) { Console.WriteLine(@"Input para:"); var readLine = Console.ReadLine(); if (readLine != null) { args = readLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); } } else { Console.WriteLine(string.Join(" ", args)); } var strPara = string.Join(" ", args); var pr = new Parameter(); pr.ParseData(strPara); Console.Title = strPara; Console.CancelKeyPress += EndApp; if (pr.TypeRun == 1) { if (string.IsNullOrEmpty(pr.Domain)) { for (var i = 0; i < pr.NumberThread; i++) { Task.Factory.StartNew(() => { var w = new WorkerMqRl(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ, pr.AckIm); w.StartConsume(); }); Thread.Sleep(20000); } } else { var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString)); var token = new CancellationToken(); var wokerFn = new WSS.Core.Crawler.WorkerReload(pt.GetCompanyIDFromDomain(pr.Domain), ""); wokerFn.StartCrawler(); } } else if (pr.TypeRun == 0) { if (string.IsNullOrEmpty(pr.Domain)) { for (var i = 0; i < pr.NumberThread; i++) { var token = Source.Token; var j = i; Task.Factory.StartNew(() => { var w = new WorkerMqFn(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ); w.StartConsume(); }, token); Thread.Sleep(20000); } } else { var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString)); var token = new CancellationToken(); var wokerFn = new WorkerFindNew(pt.GetCompanyIDFromDomain(pr.Domain), ""); wokerFn.StartCrawler(); } } while (true) { if (Source.IsCancellationRequested) { Thread.Sleep(10000); return; } else { Thread.Sleep(1000); } } } catch (Exception ex0) { log.Error(ex0); Console.ReadLine(); } }