public void Start()
{
_log.InfoFormat("Start consumer FindNew");
while (true)
{
try
{
_token.ThrowIfCancellationRequested();
JobCompanyCrawler jobCompanyCrawler = GetJob();
if (jobCompanyCrawler != null)
{
if (jobCompanyCrawler.CheckRunning && CheckOtherRunning(jobCompanyCrawler.CompanyId))
{
_log.Info(string.Format("Other running company: {0}", jobCompanyCrawler));
}
else if (!_productAdapter.AllowCrawlFindNew(jobCompanyCrawler.CompanyId))
{
_log.Info("Not allow crawler");
_redisWaitCrawler.DeleteWaitFindNew(new List <long> {
jobCompanyCrawler.CompanyId
});
}
else
{
using (var worker = new WorkerFindNew(jobCompanyCrawler.CompanyId, _nameThread))
{
worker.StartCrawler();
}
}
}
else
{
Thread.Sleep(60000);
_log.Info("No company to crawler");
}
}
catch (OperationCanceledException task)
{
break;
}
catch (Exception ex1)
{
_log.Error(ex1);
}
}
}
private static void Main(string[] args)
{
try
{
Server.ConnectionString = ConfigCrawler.ConnectProduct;
Server.ConnectionStringCrawler = ConfigCrawler.ConnectionCrawler;
Server.LogConnectionString = ConfigCrawler.ConnectLog;
InitQueue isq = new InitQueue();
isq.Start();
if (args == null || args.Length == 0)
{
Console.WriteLine(@"Input para:");
var readLine = Console.ReadLine();
if (readLine != null)
{
args = readLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
}
}
else
{
Console.WriteLine(string.Join(" ", args));
}
var strPara = string.Join(" ", args);
var pr = new Parameter();
pr.ParseData(strPara);
Console.Title = strPara;
Console.CancelKeyPress += EndApp;
if (pr.TypeRun == 1)
{
if (string.IsNullOrEmpty(pr.Domain))
{
for (var i = 0; i < pr.NumberThread; i++)
{
Task.Factory.StartNew(() =>
{
var w = new WorkerMqRl(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ, pr.AckIm);
w.StartConsume();
});
Thread.Sleep(20000);
}
}
else
{
var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString));
var token = new CancellationToken();
var wokerFn = new WSS.Core.Crawler.WorkerReload(pt.GetCompanyIDFromDomain(pr.Domain), "");
wokerFn.StartCrawler();
}
}
else if (pr.TypeRun == 0)
{
if (string.IsNullOrEmpty(pr.Domain))
{
for (var i = 0; i < pr.NumberThread; i++)
{
var token = Source.Token;
var j = i;
Task.Factory.StartNew(() =>
{
var w = new WorkerMqFn(RabbitMQManager.GetRabbitMQServer(ConfigCrawler.KeyRabbitMqCrawler), pr.QueueMQ);
w.StartConsume();
}, token);
Thread.Sleep(20000);
}
}
else
{
var pt = new ProductAdapter(new SqlDb(QT.Entities.Server.ConnectionString));
var token = new CancellationToken();
var wokerFn = new WorkerFindNew(pt.GetCompanyIDFromDomain(pr.Domain), "");
wokerFn.StartCrawler();
}
}
while (true)
{
if (Source.IsCancellationRequested)
{
Thread.Sleep(10000);
return;
}
else
{
Thread.Sleep(1000);
}
}
}
catch (Exception ex0)
{
log.Error(ex0);
Console.ReadLine();
}
}