public static void PushDownloadHtml(IEnumerable <string> domains)
{
SqlDb sql = new SqlDb(ConfigStatic.ProductConnection);
ProductAdapter productAdapter = new ProductAdapter(sql);
foreach (var domain in domains)
{
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
string queryData = string.Format(@"
Select Id, DetailUrl
From Product
Where Company = {0}
Order by Id
"
, productAdapter.GetCompanyIdByDomain(domain));
sql.ProcessDataTableLarge(queryData,
10000, (Row, iRow) =>
{
producerBasic.PublishString(new JobDownloadHtml()
{
ProductId = Convert.ToInt64(Row["Id"]),
DetailUrl = Convert.ToString(Row["DetailUrl"]),
Domain = domain
}.GetJson());
});
}
}
public WorkerParseData(string domain) : base(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueParse(domain), false)
{
var domainModule = new DomainModule();
var kernel = new StandardKernel(domainModule);
_handlerParserProperties = kernel.Get <IHandlerParserProperties>();
_handlerParserProperties.Init(domain);
}
public WorkerDownloadHtml(string domain) : base(
RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain), false)
{
var domainModule = new DomainModule();
var kernel = new StandardKernel(domainModule);
_h1 = kernel.Get <HandlerDownloadHtml>();
_h1.Init(domain);
}
/// <summary>
/// Public Constructor for WindowsService.
/// - Put all of your Initialization code here.
/// </summary>
public WindowsService()
{
var allOk = false;
this.ServiceName = "StationMockService";
this.EventLog.Log = "Application";
// These Flags set whether or not to handle that specific
// type of event. Set to true if you need it, false otherwise.
this.CanHandlePowerEvent = true;
this.CanHandleSessionChangeEvent = true;
this.CanPauseAndContinue = true;
this.CanShutdown = true;
this.CanStop = true;
var setting = ConfigStatic.GetConfigSetting();
if (setting.UseConfigSetting)
{
socket = new ServerSocket(setting.HostIPAddress, setting.Port);
log.Debug(LogHelp.LogText($"Socket Loaded {setting.HostIPAddress}:{setting.Port}"));
allOk = true;
}
else if (setting.UseAutoIPconfig)
{
var ipAddresses = IPAddressResolver.GetList();
foreach (var item in ipAddresses)
{
ConfigStatic.HostIpAddress = item.Address;
}
ConfigStatic.HostPort = "3000";
ConfigStatic.UseConfigSetting = "true";
socket = new ServerSocket(setting.HostIPAddress, setting.Port);
log.Info(LogHelp.LogText($"Socket Loaded {setting.HostIPAddress}:{setting.Port}"));
allOk = true;
}
if (allOk)
{
log.Debug(LogHelp.LogText($"Start... Initialize"));
socket.Initialize();
log.Debug(LogHelp.LogText($"Start... Stories"));
log.Debug(LogHelp.LogText($"Start... Timer"));
aTimer.Elapsed += new ElapsedEventHandler(OnTimedEvent);
aTimer.Interval = 5000;
aTimer.Enabled = true;
log.Debug(LogHelp.LogText($"Start... StartListening"));
socket.StartListening();
log.Debug(LogHelp.LogText($"Start... OK!"));
}
}
private static void PushJobDownload(string domain)
{
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
IStorageProduct storageProduct = new StorageProduct();
int i = 0;
string DetailUrl = "";
Regex regex = new Regex(@"http://www.+html");
storageProduct.ProcessProduct(domain, (sender, product) =>
{
if (domain == "lazada.vn")
{
string urlencode = product.DetailUrl;
string urldecode = HttpUtility.UrlDecode(HttpUtility.UrlDecode(urlencode));
MatchCollection matches = regex.Matches(urldecode);
DetailUrl = matches[0].Value.ToString();
////string urlencode = product.DetailUrl;
////string urldecode = HttpUtility.UrlDecode(product.DetailUrl);
////char charRange = '?';
////int startIndex = urldecode.IndexOf(charRange) + 1;
////int endIndex = urldecode.LastIndexOf(charRange) - 1;
////int length = endIndex - startIndex + 1;
////DetailUrl = urldecode.Substring(startIndex, length).Replace("url=", "");
}
else
{
DetailUrl = product.DetailUrl;
}
producer.PublishString(new JobCrlProperties()
{
ProductId = product.Id,
DetailUrl = UtilCrl.GetUrl(DetailUrl, domain),
Domain = domain,
ClassificationId = product.ClassificationId,
Classification = product.Classification
}.GetJson());
i++;
//log.Info(string.Format("{0} {1}", i, product.Id));
log.InfoFormat("{0}: {1}", i, product.Id);
});
}
public static void PushParseFromNoSql(IEnumerable <string> domains)
{
NoSqlAdapter noSqlAdapter = NoSqlAdapter.GetInstance();
foreach (var domain in domains)
{
Task.Factory.StartNew(() =>
{
int iCount = 0;
var producerBasicWaitPs = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(domain));
noSqlAdapter.ProcessAllIdProductByCompany(domain, (obj, productId) =>
{
iCount++;
producerBasicWaitPs.PublishString(new JobParse()
{
Id = productId
}.GetJson());
if (iCount % 10 == 0)
{
log.Info(string.Format("Pushed {0} mss of {1}", iCount, domain));
}
});
log.Info(string.Format("Pushed all data for company {0} {1}", domain, iCount));
});
}
Thread.Sleep(10000000);
}
public void Init(string domain)
{
this._domain = domain;
this._producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(this._domain));
this._config = _storageConfigCrl.GetConfig(this._domain);
}