public static void PushDownloadHtml(IEnumerable <string> domains) { SqlDb sql = new SqlDb(ConfigStatic.ProductConnection); ProductAdapter productAdapter = new ProductAdapter(sql); foreach (var domain in domains) { ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueWaitDownloadHtml(domain)); string queryData = string.Format(@" Select Id, DetailUrl From Product Where Company = {0} Order by Id " , productAdapter.GetCompanyIdByDomain(domain)); sql.ProcessDataTableLarge(queryData, 10000, (Row, iRow) => { producerBasic.PublishString(new JobDownloadHtml() { ProductId = Convert.ToInt64(Row["Id"]), DetailUrl = Convert.ToString(Row["DetailUrl"]), Domain = domain }.GetJson()); }); } }
private static void ResesCacheProduct() { ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString)); Console.WriteLine("1- Reset all Companies)"); Console.WriteLine("2- Reset for Company"); Console.Write("Input choose:"); int i = Convert.ToInt32(Console.ReadLine()); switch (i) { case 1: { int countProduct = 0; var lstCompany = productAdapter.GetAllCompanyIdCrawler(); foreach (var companyID in lstCompany) { ResetCacheProduct(companyID); log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count)); } log.Info("Complete all company"); } break; case 2: { Console.Write("Input domain:"); string strDomain = Console.ReadLine().Trim(); long companyID = productAdapter.GetCompanyIdByDomain(strDomain); ResetCacheProduct(companyID); log.Info(string.Format("Complete Company: {0}", companyID)); } break; } }
private static void Main(string[] args) { Server.ConnectionString = ConfigCrawler.ConnectProduct; Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler; Server.LogConnectionString = ConfigCrawler.ConnectLog; ProductAdapter productAdapter = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct)); string strParaInput = (args.Length == 0) ? Console.ReadLine() : string.Join(" ", args); strParaInput = @"-c svptccache -u http://maxmobile.vn/dien-thoai/lg-g5-cu.html"; ParameterManager p = ParameterManager.Parse(strParaInput); //p.SubCmd = "svptccache"; if (p.SubCmd == "crlrl") { string domain = p.Parameters["dm"][0]; long idCOmpany = productAdapter.GetCompanyIDFromDomain(domain); using (var worker = new WorkerReload(idCOmpany, new CancellationToken(), "", true)) { worker.StartCrawler(); } } else if (p.SubCmd == "crlfn") { } else if (p.SubCmd == "svcudclss") { var c = new ConsumerClassificationToSql(); c.StartConsume(); } else if (p.SubCmd == "svptccache") { var c = new ConsumerProductChangeToCache(); c.StartConsume(); } else if (p.SubCmd == "svudprsql") { var c = new ConsumerProductChangeToSql(); c.StartConsume(); } else if (p.SubCmd == "prpt") { ProductAdapter pta = new ProductAdapter(new SqlDb(Server.ConnectionString)); string url = p.Parameters["u"][0].ToString(); url = "http://maxmobile.vn/dien-thoai/lg-g5-cu.html"; Uri uri = new Uri(url); string domain = Common.GetDomainFromUrl(uri); long idCompanyId = pta.GetCompanyIdByDomain(domain); IDownloadHtml downloader = new DownloadHtmlCrawler(); Configuration config = new Configuration(); HtmlDocument htmlDocument = new HtmlDocument(); var ext = new WebExceptionStatus(); string html = downloader.GetHTML(url, 45, 2, out ext); htmlDocument.LoadHtml(html); ProductEntity pte = new ProductEntity(); ProductParse ppr = new ProductParse(); ppr.Analytics(pte, htmlDocument, url, config, domain); } //ConsumerSaveEndSession consumerSaveEndSession = new ConsumerSaveEndSession(); //consumerSaveEndSession.StartConsume(); }