public static SpiderTask GetTask()
{
SpiderTask task = TaskQueue.GetTask();
if (task == null)
{
return(null);
}
if (!String.IsNullOrEmpty(task.ConvertorKey))
{
ITaskConvert convert = Converts[task.ConvertorKey];
convert.Deal(task);
}
if (task.IsCompleted)
{
return(null);
}
if (!String.IsNullOrEmpty(task.SpiderKey))
{
task.Executor = SpiderActions[task.SpiderKey];
}
return(task);
}
public static void Run()
{
for (int i = 0; i < Hallocation.ProduceCount; i += 1)
{
Spider spider = new Spider(i);
Spiders.Add(spider);
}
for (int i = 0; i < Hallocation.ProduceCount; i += 1)
{
Spider spider = Spiders[i];
ThreadPool.QueueUserWorkItem((obj =>
{
#if DEBUG
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine("[Runner]\t{0}号蜘蛛被唤醒\r\n", spider.Index);
#endif
spider.Start();
}));
}
ThreadPool.QueueUserWorkItem((obj) =>
{
CheckEnd();
});
for (int i = 0; i < Hallocation.CustomerCount; i += 1)
{
int j = i;
ThreadPool.QueueUserWorkItem((obj) =>
{
while (true)
{
SpiderTask task = GetResult();
if (task == null)
{
Thread.Sleep(400);
continue;
}
#if DEBUG
Console.ForegroundColor = ConsoleColor.DarkGreen;
Console.WriteLine("[Executor]\t拿到结果并交给{0}来处理\r\n", task.ProcessorKey);
#endif
if (!String.IsNullOrEmpty(task.FilterKey))
{
ITaskFilter filter = Filters[task.FilterKey];
filter.Run(task);
}
if (!String.IsNullOrEmpty(task.ProcessorKey))
{
ITaskProcessor processor = Processors[task.ProcessorKey];
processor.Run(task);
}
PostTask(task);
}
});
}
}
public static void PostTask(SpiderTask task)
{
TasksCompleted.Add(task);
if (!task.IsCompleted)
{
#if DEBUG
Console.ForegroundColor = ConsoleColor.DarkCyan;
Console.WriteLine("[TaskQueue]\t接受任务:{0}\r\n", task.Url);
#endif
TaskQueue.PostTask(task);
}
else
{
#if DEBUG
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("[TaskQueue]\t驳回任务:{0}\r\n", task.Url);
#endif
}
}
public static void PostResult(SpiderTask task)
{
TaskQueue.PostResult(task);
}