private void frmGetProductFromSolr_Load(object sender, EventArgs e) { this.logKeywordsWebsiteTableAdapter.Connection.ConnectionString = WssConnection.ConnectionIndividual; _productTableAdapter.Connection.ConnectionString = WssConnection.ConnectionProduct; solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts")); LoadLogKeywordsWebsite(); }
private void Init() { InitializeComponent(); _productConnectionString = ConfigurationManager.ConnectionStrings["ProductConnectionString"] .ToString(); _userConnectionString = ConfigurationManager.ConnectionStrings["UserConnectionString"] .ToString(); _searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"]; solrIndexer = new SolrIndexer(); solrIndexer.ConnectionStringProducts = _productConnectionString; solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); var taskGetData = new Task(InitData); taskGetData.Start(); }
protected override void OnStart(string[] args) { try { var productConnectionString = ConfigurationManager.ConnectionStrings["productConnectionString"].ConnectionString; var userConnectionString = ConfigurationManager.ConnectionStrings["userConnectionString"].ConnectionString; _productConnectionString = productConnectionString; _userConnectionString = userConnectionString; string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); string indexAllProductOfMerchantJobName = ConfigurationManager.AppSettings["indexAllProductOfMerchantJobName"]; int indexAllProductOfMerchantWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexAllProductOfMerchantWorkerCount"], 1); indexAllProductOfMerchantWorkers = new Worker[indexAllProductOfMerchantWorkerCount]; string indexProductJobName = ConfigurationManager.AppSettings["indexProductJobName"]; int indexProductWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexProductWorkerCount"], 1); _searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"]; indexProductWorkers = new BatchWorker[indexProductWorkerCount]; solrIndexer = new SolrIndexer(); solrIndexer.ConnectionStringProducts = productConnectionString; solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); _isRunning = true; var updateDataTask = new Task(UpdateData); updateDataTask.Start(); var updateListMerchantTask = new Task(UpdateListMerchant); updateListMerchantTask.Start(); var updatePropertyTask = new Task(UpdateProperties); updatePropertyTask.Start(); for (int i = 0; i < indexAllProductOfMerchantWorkerCount; i++) { var worker = new Worker(indexAllProductOfMerchantJobName, false, rabbitMQServer); indexAllProductOfMerchantWorkers[i] = worker; var workerTask = new Task(() => { Thread.Sleep(60000); //Sleep 90 seconds to wait for get Data Tasks; worker.JobHandler = (indexAllProductOfMerchantJob) => { var companyID = BitConverter.ToInt64(indexAllProductOfMerchantJob.Data, 0); if (companyID == SolrIndexer.IDWebsosanh) { string message; var numProducts = solrIndexer.UpdateAllRootProducts(out message, productConnectionString, userConnectionString, _searchEnginesServiceUrl); if (numProducts < 0) { return(false); } Logger.Debug("Update root products success!"); return(true); } else { var merchantInfo = MerchantBAL.GetMerchantShortInfoFromCache(companyID); if (merchantInfo == null) { merchantInfo = MerchantBAL.GetMerchantShortInfo(companyID, _productConnectionString, _userConnectionString); if (merchantInfo == null) { return(true); } MerchantBAL.InsertMerchantShortInfoToCache(merchantInfo); } var numProducts = solrIndexer.UpdateAllProductOfMerchant(companyID, merchantInfo.Domain); if (numProducts < 0) { return(false); } Logger.DebugFormat("Update company {0} success!", merchantInfo.Domain); return(true); } }; worker.Start(); }); workerTask.Start(); Logger.InfoFormat("Update All products of Merchant Worker {0} started", i); } for (int i = 0; i < indexProductWorkerCount; i++) { var worker = new BatchWorker(indexProductJobName, false, rabbitMQServer, 100, 300000); indexProductWorkers[i] = worker; var workerTask = new Task(() => { Thread.Sleep(90000); //Sleep 90 seconds to wait for get Data Tasks; worker.JobHandler = (indexProductJobs) => { var merchantProductIds = new List <long>(); var rootProductIds = new List <long>(); foreach (var indexProductJob in indexProductJobs) { var productID = BitConverter.ToInt64(indexProductJob.Data, 0); if (indexProductJob.Type == 1) { rootProductIds.Add(productID); } else { merchantProductIds.Add(productID); } } if (rootProductIds.Count > 0) { var startTime = DateTime.Now; solrIndexer.UpdateRootProducts(rootProductIds, productConnectionString, userConnectionString, _searchEnginesServiceUrl); Logger.DebugFormat("Update time : {0} ms - {1} rootProducts", (DateTime.Now - startTime).TotalMilliseconds, rootProductIds.Count); } else { var startTime = DateTime.Now; solrIndexer.UpdateMerchantProducts(merchantProductIds); Logger.DebugFormat("Update time : {0} ms - {1} merchantProducts", (DateTime.Now - startTime).TotalMilliseconds, merchantProductIds.Count); } return(true); }; worker.Start(); }); workerTask.Start(); Logger.InfoFormat("Update Product Worker {0} started", i); } } catch (Exception excepion) { Logger.Error("Error on start", excepion); throw; } }
private void FrmSuggestRootProduct_Load(object sender, EventArgs e) { WssConnection.ConnectionIndividual = ConfigurationSettings.AppSettings["ConnectionIndividual"]; WssConnection.ConnectionProduct = ConfigurationSettings.AppSettings["ConnectionProduct"]; solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts")); }
private static void Main(string[] args) { string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; string updateAllProductsOfMerchantGroupName = ConfigurationManager.AppSettings["updateAllProductsOfMerchantGroupName"]; string updateAllProductsOfMerchantJobName = ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobName"]; int updateAllProductsOfMerchantJobExpirationMS = CommonUtilities.Object2Int( ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobExpirationMS"], 0); string productConnectionString = ConfigurationManager.ConnectionStrings["productConnectionString"].ToString(); try { var rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); JobClient jobClient = new JobClient(updateAllProductsOfMerchantGroupName, GroupType.Topic, updateAllProductsOfMerchantJobName, true, rabbitMQServer); //Get List Company var companyTableAdapter = new CompanyTableAdapter { Connection = { ConnectionString = productConnectionString } }; var companyDataTable = companyTableAdapter.GetAllCompanies(); Logger.InfoFormat("Selected {0} merchantIDs from db.", companyDataTable.Rows.Count); var solrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); var currentCompanyList = solrClient.GetAllCompany().Where(x => x.Value > 0).Select(x => x.Key).Select(long.Parse).ToArray(); Logger.InfoFormat("Got {0} merchantIDs from solr.", currentCompanyList.Length); var remainCompanySet = new HashSet <long>(); foreach (var companyID in currentCompanyList) { remainCompanySet.Add(companyID); } var numPushed = 0; for (var rowIndex = 0; rowIndex < companyDataTable.Rows.Count; rowIndex++) { var companyRow = companyDataTable[rowIndex]; var companyID = companyRow.ID; string companyDomain = ""; if (companyRow["Domain"] != DBNull.Value) { companyDomain = companyRow.Domain; } try { var job = new Job { Data = BitConverter.GetBytes(companyID) }; jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS); if (remainCompanySet.Contains(companyID)) { remainCompanySet.Remove(companyID); } numPushed++; } catch (Exception ex) { Logger.Error(string.Format("Publish error! Company: {0} - Domain: {1}", companyID, companyDomain), ex); } } foreach (var companyID in remainCompanySet) { try { var job = new Job { Data = BitConverter.GetBytes(companyID) }; jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS); numPushed++; } catch (Exception ex) { Logger.Error(string.Format("Publish error! Company: {0}", companyID), ex); } } Logger.InfoFormat("Schedule success! Published [{0}/{1}] Companies", numPushed, companyDataTable.Rows.Count); Thread.Sleep(100); rabbitMQServer.Stop(); } catch (Exception ex) { Logger.Error("Schedule Error!", ex); } }