public void Start()
{
_log.InfoFormat("Start consumer: {0} at queue {1}", "", _queueName);
while (true)
{
try
{
_cancellationToken.ThrowIfCancellationRequested();
JobCompanyCrawler jobCompany = GetJob();
if (jobCompany != null)
{
if (jobCompany.CheckRunning && CheckOtherRunning(jobCompany.CompanyId))
{
_log.Info(string.Format("Other running company: {0}", jobCompany));
}
else if (!_productAdapter.AllowCrawlReload(jobCompany.CompanyId))
{
_log.Info("Not allow crawler");
_redisWaitCrawler.DeleteWaitReload(new List <long> {
jobCompany.CompanyId
});
}
else
{
using (var worker = new WorkerReload(jobCompany.CompanyId, _nameThread))
{
worker.StartCrawler();
}
}
}
else
{
_log.Info("No company to crawler");
Thread.Sleep(60000);
}
}
catch (OperationCanceledException task)
{
break;
}
catch (Exception ex1)
{
_log.Error(ex1);
Thread.Sleep(10000);
}
}
}
private static void Main(string[] args)
{
Server.ConnectionString = ConfigCrawler.ConnectProduct;
Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler;
Server.LogConnectionString = ConfigCrawler.ConnectLog;
ProductAdapter productAdapter = new ProductAdapter(new SqlDb(ConfigCrawler.ConnectProduct));
string strParaInput = (args.Length == 0) ? Console.ReadLine() : string.Join(" ", args);
strParaInput = @"-c svptccache -u http://maxmobile.vn/dien-thoai/lg-g5-cu.html";
ParameterManager p = ParameterManager.Parse(strParaInput);
//p.SubCmd = "svptccache";
if (p.SubCmd == "crlrl")
{
string domain = p.Parameters["dm"][0];
long idCOmpany = productAdapter.GetCompanyIDFromDomain(domain);
using (var worker = new WorkerReload(idCOmpany, new CancellationToken(), "", true))
{
worker.StartCrawler();
}
}
else if (p.SubCmd == "crlfn")
{
}
else if (p.SubCmd == "svcudclss")
{
var c = new ConsumerClassificationToSql();
c.StartConsume();
}
else if (p.SubCmd == "svptccache")
{
var c = new ConsumerProductChangeToCache();
c.StartConsume();
}
else if (p.SubCmd == "svudprsql")
{
var c = new ConsumerProductChangeToSql();
c.StartConsume();
}
else if (p.SubCmd == "prpt")
{
ProductAdapter pta = new ProductAdapter(new SqlDb(Server.ConnectionString));
string url = p.Parameters["u"][0].ToString();
url = "http://maxmobile.vn/dien-thoai/lg-g5-cu.html";
Uri uri = new Uri(url);
string domain = Common.GetDomainFromUrl(uri);
long idCompanyId = pta.GetCompanyIdByDomain(domain);
IDownloadHtml downloader = new DownloadHtmlCrawler();
Configuration config = new Configuration();
HtmlDocument htmlDocument = new HtmlDocument();
var ext = new WebExceptionStatus();
string html = downloader.GetHTML(url, 45, 2, out ext);
htmlDocument.LoadHtml(html);
ProductEntity pte = new ProductEntity();
ProductParse ppr = new ProductParse();
ppr.Analytics(pte, htmlDocument, url, config, domain);
}
//ConsumerSaveEndSession consumerSaveEndSession = new ConsumerSaveEndSession();
//consumerSaveEndSession.StartConsume();
}
private static void Main(string[] args)
{
try
{
Server.ConnectionString = ConfigCrawler.ConnectProduct;
Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler;
Server.LogConnectionString = ConfigCrawler.ConnectLog;
InitQueue isq = new InitQueue();
isq.Start();
if (args == null || args.Length == 0)
{
Console.WriteLine(@"Input para:");
var readLine = Console.ReadLine();
if (readLine != null)
{
args = readLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
}
}
else
{
Console.WriteLine(string.Join(" ", args));
}
var strPara = string.Join(" ", args);
var pr = new Parameter();
pr.ParseData(strPara);
Console.Title = strPara;
Console.CancelKeyPress += EndApp;
if (pr.TypeRun == 1)
{
if (string.IsNullOrEmpty(pr.Domain))
{
for (var i = 0; i < pr.NumberThread; i++)
{
Task.Factory.StartNew(() =>
{
var w = new WorkerMqRl(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ, pr.AckIm);
w.StartConsume();
});
Thread.Sleep(20000);
}
}
else
{
var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString));
var token = new CancellationToken();
var wokerFn = new WSS.Core.Crawler.WorkerReload(pt.GetCompanyIDFromDomain(pr.Domain), "");
wokerFn.StartCrawler();
}
}
else if (pr.TypeRun == 0)
{
if (string.IsNullOrEmpty(pr.Domain))
{
for (var i = 0; i < pr.NumberThread; i++)
{
var token = Source.Token;
var j = i;
Task.Factory.StartNew(() =>
{
var w = new WorkerMqFn(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ);
w.StartConsume();
}, token);
Thread.Sleep(20000);
}
}
else
{
var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString));
var token = new CancellationToken();
var wokerFn = new WorkerFindNew(pt.GetCompanyIDFromDomain(pr.Domain), "");
wokerFn.StartCrawler();
}
}
while (true)
{
if (Source.IsCancellationRequested)
{
Thread.Sleep(10000);
return;
}
else
{
Thread.Sleep(1000);
}
}
}
catch (Exception ex0)
{
log.Error(ex0);
Console.ReadLine();
}
}
