public void RemoveTask(string taskIdentify)
{
Redis.KeyDelete(GetQueueKey(taskIdentify));
Redis.KeyDelete(GetSetKey(taskIdentify));
Redis.HashDelete(RedisScheduler.TaskStatus, taskIdentify);
Redis.KeyDelete(RedisScheduler.ItemPrefix + taskIdentify);
Redis.KeyDelete(taskIdentify);
Redis.KeyDelete("locker-" + taskIdentify);
Redis.SortedSetRemove(RedisScheduler.TaskList, taskIdentify);
Redis.HashDelete("init-status", taskIdentify);
Redis.HashDelete("validate-status", taskIdentify);
Redis.KeyDelete("set-" + Encrypt.Md5Encrypt(taskIdentify));
}
private Core.Spider PrepareSpider(params string[] args)
{
RedisServer redis = GetManageRedisServer();
var schedulerType = SpiderContext.Scheduler.Type;
bool isTestSpider = args != null && args.Contains("test");
switch (schedulerType)
{
case Configuration.Scheduler.Types.Queue:
{
PrepareSite();
var spider = GenerateSpider(SpiderContext.Scheduler.GetScheduler());
if (isTestSpider && spider.Site.StartRequests.Count > 0)
{
spider.Site.StartRequests = new List <Request> {
spider.Site.StartRequests[0]
};
}
spider.InitComponent();
return(spider);
}
case Configuration.Scheduler.Types.Redis:
{
var scheduler = (Scheduler.RedisScheduler)(SpiderContext.Scheduler.GetScheduler());
string key = "locker-" + Name;
if (args != null && args.Length > 0)
{
if (args.Contains("rerun"))
{
if (redis != null)
{
redis.KeyDelete(key);
redis.HashDelete("init-status", Name);
redis.HashDelete("validate-status", Name);
redis.HashDelete(Scheduler.RedisScheduler.TaskStatus, Name);
redis.SortedSetRemove(Scheduler.RedisScheduler.TaskList, Name);
}
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetQueueKey(Name));
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetSetKey(Name));
scheduler.Redis.KeyDelete(Scheduler.RedisScheduler.GetItemKey(Name));
}
if (args.Contains("noconsole"))
{
Log.WriteLine("No console log info.");
Log.NoConsole = true;
}
}
try
{
if (redis != null)
{
while (!redis.LockTake(key, "0", TimeSpan.FromMinutes(10)))
{
Thread.Sleep(1000);
}
}
var lockerValue = redis?.HashGet(InitStatusSetName, Name);
bool needInitStartRequest = lockerValue != "init finished";
if (needInitStartRequest)
{
PrepareSite();
}
else
{
Logger.Info("Site 已经初始化");
SpiderContext.Site.ClearStartRequests();
}
Logger.Info("创建爬虫...");
var spider = GenerateSpider(scheduler);
spider.SaveStatus = true;
SpiderMonitor.Default.Register(spider);
Logger.Info("构建内部模块...");
if (isTestSpider && spider.Site.StartRequests.Count > 0)
{
spider.Site.StartRequests = new List <Request> {
spider.Site.StartRequests[0]
};
}
spider.InitComponent();
if (needInitStartRequest)
{
redis?.HashSet(InitStatusSetName, Name, "init finished");
}
return(spider);
}
catch (Exception e)
{
Logger.Error(e.Message, e);
return(null);
}
finally
{
try
{
redis?.LockRelease(key, 0);
}
catch
{
// ignored
}
}
}
}
throw new SpiderExceptoin("初始化失败.");
}
