public void Start() { _log.InfoFormat("Start consumer FindNew"); while (true) { try { _token.ThrowIfCancellationRequested(); JobCompanyCrawler jobCompanyCrawler = GetJob(); if (jobCompanyCrawler != null) { if (jobCompanyCrawler.CheckRunning && CheckOtherRunning(jobCompanyCrawler.CompanyId)) { _log.Info(string.Format("Other running company: {0}", jobCompanyCrawler)); } else if (!_productAdapter.AllowCrawlFindNew(jobCompanyCrawler.CompanyId)) { _log.Info("Not allow crawler"); _redisWaitCrawler.DeleteWaitFindNew(new List <long> { jobCompanyCrawler.CompanyId }); } else { using (var worker = new WorkerFindNew(jobCompanyCrawler.CompanyId, _nameThread)) { worker.StartCrawler(); } } } else { Thread.Sleep(60000); _log.Info("No company to crawler"); } } catch (OperationCanceledException task) { break; } catch (Exception ex1) { _log.Error(ex1); } } }
private static void Main(string[] args) { try { Server.ConnectionString = ConfigCrawler.ConnectProduct; Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler; Server.LogConnectionString = ConfigCrawler.ConnectLog; InitQueue isq = new InitQueue(); isq.Start(); if (args == null || args.Length == 0) { Console.WriteLine(@"Input para:"); var readLine = Console.ReadLine(); if (readLine != null) { args = readLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); } } else { Console.WriteLine(string.Join(" ", args)); } var strPara = string.Join(" ", args); var pr = new Parameter(); pr.ParseData(strPara); Console.Title = strPara; Console.CancelKeyPress += EndApp; if (pr.TypeRun == 1) { if (string.IsNullOrEmpty(pr.Domain)) { for (var i = 0; i < pr.NumberThread; i++) { Task.Factory.StartNew(() => { var w = new WorkerMqRl(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ, pr.AckIm); w.StartConsume(); }); Thread.Sleep(20000); } } else { var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString)); var token = new CancellationToken(); var wokerFn = new WSS.Core.Crawler.WorkerReload(pt.GetCompanyIDFromDomain(pr.Domain), ""); wokerFn.StartCrawler(); } } else if (pr.TypeRun == 0) { if (string.IsNullOrEmpty(pr.Domain)) { for (var i = 0; i < pr.NumberThread; i++) { var token = Source.Token; var j = i; Task.Factory.StartNew(() => { var w = new WorkerMqFn(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ); w.StartConsume(); }, token); Thread.Sleep(20000); } } else { var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString)); var token = new CancellationToken(); var wokerFn = new WorkerFindNew(pt.GetCompanyIDFromDomain(pr.Domain), ""); wokerFn.StartCrawler(); } } while (true) { if (Source.IsCancellationRequested) { Thread.Sleep(10000); return; } else { Thread.Sleep(1000); } } } catch (Exception ex0) { log.Error(ex0); Console.ReadLine(); } }