public static void PushDownloadHtml(IEnumerable <string> domains)
{
SqlDb sql = new SqlDb(ConfigStatic.ProductConnection);
ProductAdapter productAdapter = new ProductAdapter(sql);
foreach (var domain in domains)
{
ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
ConfigStatic.GetQueueWaitDownloadHtml(domain));
string queryData = string.Format(@"
Select Id, DetailUrl
From Product
Where Company = {0}
Order by Id
"
, productAdapter.GetCompanyIdByDomain(domain));
sql.ProcessDataTableLarge(queryData,
10000, (Row, iRow) =>
{
producerBasic.PublishString(new JobDownloadHtml()
{
ProductId = Convert.ToInt64(Row["Id"]),
DetailUrl = Convert.ToString(Row["DetailUrl"]),
Domain = domain
}.GetJson());
});
}
}
private static void ResesCacheProduct()
{
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(Server.ConnectionString));
Console.WriteLine("1- Reset all Companies)");
Console.WriteLine("2- Reset for Company");
Console.Write("Input choose:");
int i = Convert.ToInt32(Console.ReadLine());
switch (i)
{
case 1:
{
int countProduct = 0;
var lstCompany = productAdapter.GetAllCompanyIdCrawler();
foreach (var companyID in lstCompany)
{
ResetCacheProduct(companyID);
log.Info(string.Format("Complete Company: {0} {1}/{2}", companyID, countProduct++, lstCompany.Count));
}
log.Info("Complete all company");
} break;
case 2:
{
Console.Write("Input domain:");
string strDomain = Console.ReadLine().Trim();
long companyID = productAdapter.GetCompanyIdByDomain(strDomain);
ResetCacheProduct(companyID);
log.Info(string.Format("Complete Company: {0}", companyID));
} break;
}
}
private static void Main(string[] args)
{
Server.ConnectionString = ConfigCrawler.ConnectProduct;
Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler;
Server.LogConnectionString = ConfigCrawler.ConnectLog;
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct));
string strParaInput = (args.Length == 0) ? Console.ReadLine() : string.Join(" ", args);
strParaInput = @"-c svptccache -u http://maxmobile.vn/dien-thoai/lg-g5-cu.html";
ParameterManager p = ParameterManager.Parse(strParaInput);
//p.SubCmd = "svptccache";
if (p.SubCmd == "crlrl")
{
string domain = p.Parameters["dm"][0];
long idCOmpany = productAdapter.GetCompanyIDFromDomain(domain);
using (var worker = new WorkerReload(idCOmpany, new CancellationToken(), "", true))
{
worker.StartCrawler();
}
}
else if (p.SubCmd == "crlfn")
{
}
else if (p.SubCmd == "svcudclss")
{
var c = new ConsumerClassificationToSql();
c.StartConsume();
}
else if (p.SubCmd == "svptccache")
{
var c = new ConsumerProductChangeToCache();
c.StartConsume();
}
else if (p.SubCmd == "svudprsql")
{
var c = new ConsumerProductChangeToSql();
c.StartConsume();
}
else if (p.SubCmd == "prpt")
{
ProductAdapter pta = new ProductAdapter(new SqlDb(Server.ConnectionString));
string url = p.Parameters["u"][0].ToString();
url = "http://maxmobile.vn/dien-thoai/lg-g5-cu.html";
Uri uri = new Uri(url);
string domain = Common.GetDomainFromUrl(uri);
long idCompanyId = pta.GetCompanyIdByDomain(domain);
IDownloadHtml downloader = new DownloadHtmlCrawler();
Configuration config = new Configuration();
HtmlDocument htmlDocument = new HtmlDocument();
var ext = new WebExceptionStatus();
string html = downloader.GetHTML(url, 45, 2, out ext);
htmlDocument.LoadHtml(html);
ProductEntity pte = new ProductEntity();
ProductParse ppr = new ProductParse();
ppr.Analytics(pte, htmlDocument, url, config, domain);
}
//ConsumerSaveEndSession consumerSaveEndSession = new ConsumerSaveEndSession();
//consumerSaveEndSession.StartConsume();
}
