private void frmGetProductFromSolr_Load(object sender, EventArgs e)
{
this.logKeywordsWebsiteTableAdapter.Connection.ConnectionString = WssConnection.ConnectionIndividual;
_productTableAdapter.Connection.ConnectionString = WssConnection.ConnectionProduct;
solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts"));
LoadLogKeywordsWebsite();
}
private void Init()
{
InitializeComponent();
_productConnectionString =
ConfigurationManager.ConnectionStrings["ProductConnectionString"]
.ToString();
_userConnectionString =
ConfigurationManager.ConnectionStrings["UserConnectionString"]
.ToString();
_searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"];
solrIndexer = new SolrIndexer();
solrIndexer.ConnectionStringProducts = _productConnectionString;
solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS));
var taskGetData = new Task(InitData);
taskGetData.Start();
}
protected override void OnStart(string[] args)
{
try
{
var productConnectionString = ConfigurationManager.ConnectionStrings["productConnectionString"].ConnectionString;
var userConnectionString = ConfigurationManager.ConnectionStrings["userConnectionString"].ConnectionString;
_productConnectionString = productConnectionString;
_userConnectionString = userConnectionString;
string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"];
rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName);
string indexAllProductOfMerchantJobName = ConfigurationManager.AppSettings["indexAllProductOfMerchantJobName"];
int indexAllProductOfMerchantWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexAllProductOfMerchantWorkerCount"], 1);
indexAllProductOfMerchantWorkers = new Worker[indexAllProductOfMerchantWorkerCount];
string indexProductJobName = ConfigurationManager.AppSettings["indexProductJobName"];
int indexProductWorkerCount = CommonUtilities.Object2Int(ConfigurationManager.AppSettings["indexProductWorkerCount"], 1);
_searchEnginesServiceUrl = ConfigurationManager.AppSettings["searchEnginesServiceUrl"];
indexProductWorkers = new BatchWorker[indexProductWorkerCount];
solrIndexer = new SolrIndexer();
solrIndexer.ConnectionStringProducts = productConnectionString;
solrIndexer.SolrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS));
_isRunning = true;
var updateDataTask = new Task(UpdateData);
updateDataTask.Start();
var updateListMerchantTask = new Task(UpdateListMerchant);
updateListMerchantTask.Start();
var updatePropertyTask = new Task(UpdateProperties);
updatePropertyTask.Start();
for (int i = 0; i < indexAllProductOfMerchantWorkerCount; i++)
{
var worker = new Worker(indexAllProductOfMerchantJobName, false, rabbitMQServer);
indexAllProductOfMerchantWorkers[i] = worker;
var workerTask = new Task(() =>
{
Thread.Sleep(60000); //Sleep 90 seconds to wait for get Data Tasks;
worker.JobHandler = (indexAllProductOfMerchantJob) =>
{
var companyID = BitConverter.ToInt64(indexAllProductOfMerchantJob.Data, 0);
if (companyID == SolrIndexer.IDWebsosanh)
{
string message;
var numProducts = solrIndexer.UpdateAllRootProducts(out message, productConnectionString, userConnectionString, _searchEnginesServiceUrl);
if (numProducts < 0)
{
return(false);
}
Logger.Debug("Update root products success!");
return(true);
}
else
{
var merchantInfo = MerchantBAL.GetMerchantShortInfoFromCache(companyID);
if (merchantInfo == null)
{
merchantInfo = MerchantBAL.GetMerchantShortInfo(companyID, _productConnectionString, _userConnectionString);
if (merchantInfo == null)
{
return(true);
}
MerchantBAL.InsertMerchantShortInfoToCache(merchantInfo);
}
var numProducts = solrIndexer.UpdateAllProductOfMerchant(companyID, merchantInfo.Domain);
if (numProducts < 0)
{
return(false);
}
Logger.DebugFormat("Update company {0} success!", merchantInfo.Domain);
return(true);
}
};
worker.Start();
});
workerTask.Start();
Logger.InfoFormat("Update All products of Merchant Worker {0} started", i);
}
for (int i = 0; i < indexProductWorkerCount; i++)
{
var worker = new BatchWorker(indexProductJobName, false, rabbitMQServer, 100, 300000);
indexProductWorkers[i] = worker;
var workerTask = new Task(() =>
{
Thread.Sleep(90000); //Sleep 90 seconds to wait for get Data Tasks;
worker.JobHandler = (indexProductJobs) =>
{
var merchantProductIds = new List <long>();
var rootProductIds = new List <long>();
foreach (var indexProductJob in indexProductJobs)
{
var productID = BitConverter.ToInt64(indexProductJob.Data, 0);
if (indexProductJob.Type == 1)
{
rootProductIds.Add(productID);
}
else
{
merchantProductIds.Add(productID);
}
}
if (rootProductIds.Count > 0)
{
var startTime = DateTime.Now;
solrIndexer.UpdateRootProducts(rootProductIds, productConnectionString, userConnectionString, _searchEnginesServiceUrl);
Logger.DebugFormat("Update time : {0} ms - {1} rootProducts", (DateTime.Now - startTime).TotalMilliseconds, rootProductIds.Count);
}
else
{
var startTime = DateTime.Now;
solrIndexer.UpdateMerchantProducts(merchantProductIds);
Logger.DebugFormat("Update time : {0} ms - {1} merchantProducts", (DateTime.Now - startTime).TotalMilliseconds, merchantProductIds.Count);
}
return(true);
};
worker.Start();
});
workerTask.Start();
Logger.InfoFormat("Update Product Worker {0} started", i);
}
}
catch (Exception excepion)
{
Logger.Error("Error on start", excepion);
throw;
}
}
private void FrmSuggestRootProduct_Load(object sender, EventArgs e)
{
WssConnection.ConnectionIndividual = ConfigurationSettings.AppSettings["ConnectionIndividual"];
WssConnection.ConnectionProduct = ConfigurationSettings.AppSettings["ConnectionProduct"];
solrProductClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient("solrProducts"));
}
private static void Main(string[] args)
{
string rabbitMQServerName = ConfigurationManager.AppSettings["rabbitMQServerName"];
string updateAllProductsOfMerchantGroupName =
ConfigurationManager.AppSettings["updateAllProductsOfMerchantGroupName"];
string updateAllProductsOfMerchantJobName =
ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobName"];
int updateAllProductsOfMerchantJobExpirationMS =
CommonUtilities.Object2Int(
ConfigurationManager.AppSettings["updateAllProductsOfMerchantJobExpirationMS"], 0);
string productConnectionString =
ConfigurationManager.ConnectionStrings["productConnectionString"].ToString();
try
{
var rabbitMQServer = RabbitMQManager.GetRabbitMQServer(rabbitMQServerName);
JobClient jobClient = new JobClient(updateAllProductsOfMerchantGroupName, GroupType.Topic,
updateAllProductsOfMerchantJobName, true, rabbitMQServer);
//Get List Company
var companyTableAdapter = new CompanyTableAdapter
{
Connection = { ConnectionString = productConnectionString }
};
var companyDataTable = companyTableAdapter.GetAllCompanies();
Logger.InfoFormat("Selected {0} merchantIDs from db.", companyDataTable.Rows.Count);
var solrClient = SolrProductClient.GetClient(SolrClientManager.GetSolrClient(SolrProductConstants.SOLR_NODE_PRODUCTS));
var currentCompanyList = solrClient.GetAllCompany().Where(x => x.Value > 0).Select(x => x.Key).Select(long.Parse).ToArray();
Logger.InfoFormat("Got {0} merchantIDs from solr.", currentCompanyList.Length);
var remainCompanySet = new HashSet <long>();
foreach (var companyID in currentCompanyList)
{
remainCompanySet.Add(companyID);
}
var numPushed = 0;
for (var rowIndex = 0; rowIndex < companyDataTable.Rows.Count; rowIndex++)
{
var companyRow = companyDataTable[rowIndex];
var companyID = companyRow.ID;
string companyDomain = "";
if (companyRow["Domain"] != DBNull.Value)
{
companyDomain = companyRow.Domain;
}
try
{
var job = new Job {
Data = BitConverter.GetBytes(companyID)
};
jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS);
if (remainCompanySet.Contains(companyID))
{
remainCompanySet.Remove(companyID);
}
numPushed++;
}
catch (Exception ex)
{
Logger.Error(string.Format("Publish error! Company: {0} - Domain: {1}", companyID, companyDomain),
ex);
}
}
foreach (var companyID in remainCompanySet)
{
try
{
var job = new Job {
Data = BitConverter.GetBytes(companyID)
};
jobClient.PublishJob(job, updateAllProductsOfMerchantJobExpirationMS);
numPushed++;
}
catch (Exception ex)
{
Logger.Error(string.Format("Publish error! Company: {0}", companyID),
ex);
}
}
Logger.InfoFormat("Schedule success! Published [{0}/{1}] Companies", numPushed,
companyDataTable.Rows.Count);
Thread.Sleep(100);
rabbitMQServer.Stop();
}
catch (Exception ex)
{
Logger.Error("Schedule Error!", ex);
}
}
