public SpiderStatus GetTaskStatus(string taskIdentify)
{
string json = Redis.HashGet(RedisScheduler.TaskStatus, taskIdentify);
if (!string.IsNullOrEmpty(json))
{
return(JsonConvert.DeserializeObject <SpiderStatus>(json));
}
return(new SpiderStatus());
}
private Core.Spider PrepareSpider(params string[] args)
{
RedisServer redis = GetManageRedisServer();
var schedulerType = SpiderContext.Scheduler.Type;
bool isTestSpider = args != null && args.Contains("test");
switch (schedulerType)
{
case Configuration.Scheduler.Types.Queue:
{
PrepareSite();
var spider = GenerateSpider(SpiderContext.Scheduler.GetScheduler());
if (isTestSpider && spider.Site.StartRequests.Count > 0)
{
spider.Site.StartRequests = new List <Request> {
spider.Site.StartRequests[0]
};
}
spider.InitComponent();
return(spider);
}
case Configuration.Scheduler.Types.Redis:
{
var scheduler = (Scheduler.RedisScheduler)(SpiderContext.Scheduler.GetScheduler());
string key = "locker-" + Name;
if (args != null && args.Length > 0)
{
if (args.Contains("rerun"))
{
if (redis != null)
{
redis.KeyDelete(key);
redis.HashDelete("init-status", Name);
redis.HashDelete("validate-status", Name);
redis.HashDelete(Scheduler.RedisScheduler.TaskStatus, Name);
redis.SortedSetRemove(Scheduler.RedisScheduler.TaskList, Name);
}
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetQueueKey(Name));
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetSetKey(Name));
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetItemKey(Name));
}
if (args.Contains("noconsole"))
{
Log.WriteLine("No console log info.");
Log.NoConsole = true;
}
}
try
{
if (redis != null)
{
while (!redis.LockTake(key, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
}
var lockerValue = redis?.HashGet(InitStatusSetName, Name);
bool needInitStartRequest = lockerValue != "init finished";
if (needInitStartRequest)
{
PrepareSite();
}
else
{
Logger.Info("Site 已经初始化");
SpiderContext.Site.ClearStartRequests();
}
Logger.Info("创建爬虫...");
var spider = GenerateSpider(scheduler);
spider.SaveStatus = true;
SpiderMonitor.Default.Register(spider);
Logger.Info("构建内部模块...");
if (isTestSpider && spider.Site.StartRequests.Count > 0)
{
spider.Site.StartRequests = new List <Request> {
spider.Site.StartRequests[0]
};
}
spider.InitComponent();
if (needInitStartRequest)
{
redis?.HashSet(InitStatusSetName, Name, "init finished");
}
return(spider);
}
catch (Exception e)
{
Logger.Error(e.Message, e);
return(null);
}
finally
{
try
{
redis?.LockRelease(key, 0);
}
catch
{
// ignored
}
}
}
}
throw new SpiderExceptoin("初始化失败.");
}
private void DoValidate()
{
RedisServer redis = new RedisServer(ConfigurationManager.Get("redisHost"), 6379, ConfigurationManager.Get("redisPassword"));
string key = "locker-validate-" + Name;
try
{
if (SpiderContext.Validations == null)
{
return;
}
var validations = SpiderContext.Validations.GetValidations();
if (validations != null && validations.Count > 0)
{
foreach (var validation in validations)
{
validation.CheckArguments();
}
}
if (redis != null)
{
while (!redis.LockTake(key, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
}
var lockerValue = redis?.HashGet(ValidateStatusName, Name);
bool needInitStartRequest = lockerValue != "validate finished";
if (needInitStartRequest)
{
Logger.Info("开始数据验证 ...");
if (validations != null && validations.Count > 0)
{
MailBodyBuilder builder = new MailBodyBuilder(Name, SpiderContext.Validations.Corporation);
foreach (var validation in validations)
{
builder.AddValidateResult(validation.Validate());
}
string mailBody = builder.Build();
using (EmailClient client = new EmailClient(SpiderContext.Validations.EmailSmtpServer, SpiderContext.Validations.EmailUser, SpiderContext.Validations.EmailPassword, SpiderContext.Validations.EmailSmtpPort))
{
client.SendMail(new EmaillMessage($"{Name} " + "validation report", mailBody, SpiderContext.Validations.EmailTo)
{
IsHtml = true
});
}
}
}
else
{
Logger.Info("有其他线程执行了数据验证.");
}
if (needInitStartRequest)
{
redis?.HashSet(ValidateStatusName, Name, "validate finished");
}
}
catch (Exception e)
{
Logger.Error(e.Message, e);
}
finally
{
redis?.LockRelease(key, 0);
}
}