public static void PushDownloadHtml(IEnumerable <string> domains)
{
SqlDb sql = new SqlDb(ConfigStatic.ProductConnection);
ProductAdapter productAdapter = new ProductAdapter(sql);
foreach (var domain in domains)
{
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
string queryData = string.Format(@"
Select Id, DetailUrl
From Product
Where Company = {0}
Order by Id
"
, productAdapter.GetCompanyIdByDomain(domain));
sql.ProcessDataTableLarge(queryData,
10000, (Row, iRow) =>
{
producerBasic.PublishString(new JobDownloadHtml()
{
ProductId = Convert.ToInt64(Row["Id"]),
DetailUrl = Convert.ToString(Row["DetailUrl"]),
Domain = domain
}.GetJson());
});
}
}
public WorkerDownloadHtml(string domain) : base(
RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain), false)
{
var domainModule = new DomainModule();
var kernel = new StandardKernel(domainModule);
_h1 = kernel.Get <HandlerDownloadHtml>();
_h1.Init(domain);
}
private static void PushJobDownload(string domain)
{
ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
IStorageProduct storageProduct = new StorageProduct();
int i = 0;
string DetailUrl = "";
Regex regex = new Regex(@"http://www.+html");
storageProduct.ProcessProduct(domain, (sender, product) =>
{
if (domain == "lazada.vn")
{
string urlencode = product.DetailUrl;
string urldecode = HttpUtility.UrlDecode(HttpUtility.UrlDecode(urlencode));
MatchCollection matches = regex.Matches(urldecode);
DetailUrl = matches[0].Value.ToString();
////string urlencode = product.DetailUrl;
////string urldecode = HttpUtility.UrlDecode(product.DetailUrl);
////char charRange = '?';
////int startIndex = urldecode.IndexOf(charRange) + 1;
////int endIndex = urldecode.LastIndexOf(charRange) - 1;
////int length = endIndex - startIndex + 1;
////DetailUrl = urldecode.Substring(startIndex, length).Replace("url=", "");
}
else
{
DetailUrl = product.DetailUrl;
}
producer.PublishString(new JobCrlProperties()
{
ProductId = product.Id,
DetailUrl = UtilCrl.GetUrl(DetailUrl, domain),
Domain = domain,
ClassificationId = product.ClassificationId,
Classification = product.Classification
}.GetJson());
i++;
//log.Info(string.Format("{0} {1}", i, product.Id));
log.InfoFormat("{0}: {1}", i, product.Id);
});
}
