private void frmGetProductFromSolr_Load(object sender, EventArgs e) { this.logKeywordsWebsiteTableAdapter.Connection.ConnectionString = WssConnection.ConnectionIndividual; _productTableAdapter.Connection.ConnectionString = WssConnection.ConnectionProduct; solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts")); LoadLogKeywordsWebsite(); }
public static KeywordSolrDriver GetInstance() { var solrClient = SolrClientManager.GetSolrClient(nodeSolrRaoVat); return(new KeywordSolrDriver { AutoPhraseDictionary = solrClient.AutoPhraseDictionary, Container = solrClient.Container }); }
private void frmAddTagsWebsites_Load(object sender, EventArgs e) { this.tagsTableAdapter.Connection.ConnectionString = WssConnection.ConnectionIndividual; _solrRootProductClient = SolrRootProductClient.GetClient(SolrClientManager.GetSolrClient("solrRootProducts")); try { tagsTableAdapter.FillBy_WebsiteId(dBIndi.Tags, _idWebsite); } catch (Exception exception) { Log.Error(string.Format("Website: {0} . {1}", _domainCurrent, exception)); } }
private void Init() { InitializeComponent(); _productConnectionString = ConfigurationManager.ConnectionStrings["ProductConnectionString"] .ToString(); _userConnectionString = ConfigurationManager.ConnectionStrings["UserConnectionString"] .ToString(); _searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"]; solrIndexer = new SolrIndexer(); solrIndexer.ConnectionStringProducts = _productConnectionString; solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); var taskGetData = new Task(InitData); taskGetData.Start(); }
private void InitSolr() { _solrRootProductClient = SolrRootProductClient.GetClient(SolrClientManager.GetSolrClient("solrRootProducts")); }
protected override void OnStart(string[] args) { try { var productConnectionString = ConfigurationManager.ConnectionStrings["productConnectionString"].ConnectionString; var userConnectionString = ConfigurationManager.ConnectionStrings["userConnectionString"].ConnectionString; _productConnectionString = productConnectionString; _userConnectionString = userConnectionString; string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); string indexAllProductOfMerchantJobName = ConfigurationManager.AppSettings["indexAllProductOfMerchantJobName"]; int indexAllProductOfMerchantWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexAllProductOfMerchantWorkerCount"], 1); indexAllProductOfMerchantWorkers = new Worker[indexAllProductOfMerchantWorkerCount]; string indexProductJobName = ConfigurationManager.AppSettings["indexProductJobName"]; int indexProductWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexProductWorkerCount"], 1); _searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"]; indexProductWorkers = new BatchWorker[indexProductWorkerCount]; solrIndexer = new SolrIndexer(); solrIndexer.ConnectionStringProducts = productConnectionString; solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); _isRunning = true; var updateDataTask = new Task(UpdateData); updateDataTask.Start(); var updateListMerchantTask = new Task(UpdateListMerchant); updateListMerchantTask.Start(); var updatePropertyTask = new Task(UpdateProperties); updatePropertyTask.Start(); for (int i = 0; i < indexAllProductOfMerchantWorkerCount; i++) { var worker = new Worker(indexAllProductOfMerchantJobName, false, rabbitMQServer); indexAllProductOfMerchantWorkers[i] = worker; var workerTask = new Task(() => { Thread.Sleep(60000); //Sleep 90 seconds to wait for get Data Tasks; worker.JobHandler = (indexAllProductOfMerchantJob) => { var companyID = BitConverter.ToInt64(indexAllProductOfMerchantJob.Data, 0); if (companyID == SolrIndexer.IDWebsosanh) { string message; var numProducts = solrIndexer.UpdateAllRootProducts(out message, productConnectionString, userConnectionString, _searchEnginesServiceUrl); if (numProducts < 0) { return(false); } Logger.Debug("Update root products success!"); return(true); } else { var merchantInfo = MerchantBAL.GetMerchantShortInfoFromCache(companyID); if (merchantInfo == null) { merchantInfo = MerchantBAL.GetMerchantShortInfo(companyID, _productConnectionString, _userConnectionString); if (merchantInfo == null) { return(true); } MerchantBAL.InsertMerchantShortInfoToCache(merchantInfo); } var numProducts = solrIndexer.UpdateAllProductOfMerchant(companyID, merchantInfo.Domain); if (numProducts < 0) { return(false); } Logger.DebugFormat("Update company {0} success!", merchantInfo.Domain); return(true); } }; worker.Start(); }); workerTask.Start(); Logger.InfoFormat("Update All products of Merchant Worker {0} started", i); } for (int i = 0; i < indexProductWorkerCount; i++) { var worker = new BatchWorker(indexProductJobName, false, rabbitMQServer, 100, 300000); indexProductWorkers[i] = worker; var workerTask = new Task(() => { Thread.Sleep(90000); //Sleep 90 seconds to wait for get Data Tasks; worker.JobHandler = (indexProductJobs) => { var merchantProductIds = new List <long>(); var rootProductIds = new List <long>(); foreach (var indexProductJob in indexProductJobs) { var productID = BitConverter.ToInt64(indexProductJob.Data, 0); if (indexProductJob.Type == 1) { rootProductIds.Add(productID); } else { merchantProductIds.Add(productID); } } if (rootProductIds.Count > 0) { var startTime = DateTime.Now; solrIndexer.UpdateRootProducts(rootProductIds, productConnectionString, userConnectionString, _searchEnginesServiceUrl); Logger.DebugFormat("Update time : {0} ms - {1} rootProducts", (DateTime.Now - startTime).TotalMilliseconds, rootProductIds.Count); } else { var startTime = DateTime.Now; solrIndexer.UpdateMerchantProducts(merchantProductIds); Logger.DebugFormat("Update time : {0} ms - {1} merchantProducts", (DateTime.Now - startTime).TotalMilliseconds, merchantProductIds.Count); } return(true); }; worker.Start(); }); workerTask.Start(); Logger.InfoFormat("Update Product Worker {0} started", i); } } catch (Exception excepion) { Logger.Error("Error on start", excepion); throw; } }
private void FrmSuggestRootProduct_Load(object sender, EventArgs e) { WssConnection.ConnectionIndividual = ConfigurationSettings.AppSettings["ConnectionIndividual"]; WssConnection.ConnectionProduct = ConfigurationSettings.AppSettings["ConnectionProduct"]; solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts")); }
private void RootProductAnalyzed(byte[] rootProductByte) { var jsonListProduct = UtilZipFile.Unzip(rootProductByte); var result = JsonConvert.DeserializeObject <List <RootProductAnalyzedObject> >(jsonListProduct); var dataNameCode = new Dictionary <long, List <RootProductAnalyzedObject> >(); foreach (var rootItem in result) { if (dataNameCode.ContainsKey(rootItem.RootProductId)) { dataNameCode[rootItem.RootProductId].Add(rootItem); } else { dataNameCode.Add(rootItem.RootProductId, new List <RootProductAnalyzedObject>() { rootItem }); } } var rootAdapter = new RootProductsTableAdapter(); rootAdapter.Connection.ConnectionString = _connectionStringIndi; var solrRootProductClient = SolrRootProductClient.GetClient(SolrClientManager.GetSolrClient("solrRootProducts")); DBProductTableAdapters.ProductTableAdapter productAdapter = new ProductTableAdapter(); productAdapter.Connection.ConnectionString = _connectionProduct; foreach (var rootItem in dataNameCode) { try { var rootProductSql = new RootProductSql() { RootId = rootItem.Key, Name = rootItem.Value[0].Name, WebsiteId = rootItem.Value[0].WebsiteId, NumMerchant = rootItem.Value.Count, CategoryId = rootItem.Value[0].CategoryId }; rootProductSql.LocalPath = Websosanh.Core.Common.BAL.UrlUtilities.GetRootProductLocalPath(rootProductSql.RootId, rootProductSql.Name); long minprice = 0; rootProductSql.ProductIdList = new List <long>(); var imagePath = ""; foreach (var item in rootItem.Value) { rootProductSql.ProductIdList.Add(item.Id); if (!(minprice > 0 && minprice < item.Price)) { minprice = item.Price; } if (string.IsNullOrEmpty(imagePath)) { imagePath = GetImagePath(item.Id, productAdapter); } } rootProductSql.MinPrice = minprice; rootProductSql.Image = imagePath; rootProductSql.ProductIdListString = rootProductSql.GetProductIdListString(); if (InsertSQlRootProduct(ref rootProductSql, rootAdapter)) { InsertSolrRootProduct(rootProductSql, solrRootProductClient); } } catch (Exception exception) { Log.Error(exception); } } }
private static void Main(string[] args) { string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"]; string updateAllProductsOfMerchantGroupName = ConfigurationManager.AppSettings["updateAllProductsOfMerchantGroupName"]; string updateAllProductsOfMerchantJobName = ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobName"]; int updateAllProductsOfMerchantJobExpirationMS = CommonUtilities.Object2Int( ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobExpirationMS"], 0); string productConnectionString = ConfigurationManager.ConnectionStrings["productConnectionString"].ToString(); try { var rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName); JobClient jobClient = new JobClient(updateAllProductsOfMerchantGroupName, GroupType.Topic, updateAllProductsOfMerchantJobName, true, rabbitMQServer); //Get List Company var companyTableAdapter = new CompanyTableAdapter { Connection = { ConnectionString = productConnectionString } }; var companyDataTable = companyTableAdapter.GetAllCompanies(); Logger.InfoFormat("Selected {0} merchantIDs from db.", companyDataTable.Rows.Count); var solrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS)); var currentCompanyList = solrClient.GetAllCompany().Where(x => x.Value > 0).Select(x => x.Key).Select(long.Parse).ToArray(); Logger.InfoFormat("Got {0} merchantIDs from solr.", currentCompanyList.Length); var remainCompanySet = new HashSet <long>(); foreach (var companyID in currentCompanyList) { remainCompanySet.Add(companyID); } var numPushed = 0; for (var rowIndex = 0; rowIndex < companyDataTable.Rows.Count; rowIndex++) { var companyRow = companyDataTable[rowIndex]; var companyID = companyRow.ID; string companyDomain = ""; if (companyRow["Domain"] != DBNull.Value) { companyDomain = companyRow.Domain; } try { var job = new Job { Data = BitConverter.GetBytes(companyID) }; jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS); if (remainCompanySet.Contains(companyID)) { remainCompanySet.Remove(companyID); } numPushed++; } catch (Exception ex) { Logger.Error(string.Format("Publish error! Company: {0} - Domain: {1}", companyID, companyDomain), ex); } } foreach (var companyID in remainCompanySet) { try { var job = new Job { Data = BitConverter.GetBytes(companyID) }; jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS); numPushed++; } catch (Exception ex) { Logger.Error(string.Format("Publish error! Company: {0}", companyID), ex); } } Logger.InfoFormat("Schedule success! Published [{0}/{1}] Companies", numPushed, companyDataTable.Rows.Count); Thread.Sleep(100); rabbitMQServer.Stop(); } catch (Exception ex) { Logger.Error("Schedule Error!", ex); } }