Exemple #1
0
        public static SpiderTask GetTask()
        {
            SpiderTask task = TaskQueue.GetTask();

            if (task == null)
            {
                return(null);
            }

            if (!String.IsNullOrEmpty(task.ConvertorKey))
            {
                ITaskConvert convert = Converts[task.ConvertorKey];
                convert.Deal(task);
            }
            if (task.IsCompleted)
            {
                return(null);
            }
            if (!String.IsNullOrEmpty(task.SpiderKey))
            {
                task.Executor = SpiderActions[task.SpiderKey];
            }

            return(task);
        }
Exemple #2
0
        public static void Run()
        {
            for (int i = 0; i < Hallocation.ProduceCount; i += 1)
            {
                Spider spider = new Spider(i);
                Spiders.Add(spider);
            }
            for (int i = 0; i < Hallocation.ProduceCount; i += 1)
            {
                Spider spider = Spiders[i];
                ThreadPool.QueueUserWorkItem((obj =>
                {
#if DEBUG
                    Console.ForegroundColor = ConsoleColor.Blue;
                    Console.WriteLine("[Runner]\t{0}号蜘蛛被唤醒\r\n", spider.Index);
#endif
                    spider.Start();
                }));
            }
            ThreadPool.QueueUserWorkItem((obj) =>
            {
                CheckEnd();
            });

            for (int i = 0; i < Hallocation.CustomerCount; i += 1)
            {
                int j = i;
                ThreadPool.QueueUserWorkItem((obj) =>
                {
                    while (true)
                    {
                        SpiderTask task = GetResult();
                        if (task == null)
                        {
                            Thread.Sleep(400);
                            continue;
                        }
#if DEBUG
                        Console.ForegroundColor = ConsoleColor.DarkGreen;
                        Console.WriteLine("[Executor]\t拿到结果并交给{0}来处理\r\n", task.ProcessorKey);
#endif
                        if (!String.IsNullOrEmpty(task.FilterKey))
                        {
                            ITaskFilter filter = Filters[task.FilterKey];
                            filter.Run(task);
                        }
                        if (!String.IsNullOrEmpty(task.ProcessorKey))
                        {
                            ITaskProcessor processor = Processors[task.ProcessorKey];
                            processor.Run(task);
                        }
                        PostTask(task);
                    }
                });
            }
        }
Exemple #3
0
        public static void PostTask(SpiderTask task)
        {
            TasksCompleted.Add(task);
            if (!task.IsCompleted)
            {
#if DEBUG
                Console.ForegroundColor = ConsoleColor.DarkCyan;
                Console.WriteLine("[TaskQueue]\t接受任务:{0}\r\n", task.Url);
#endif
                TaskQueue.PostTask(task);
            }
            else
            {
#if DEBUG
                Console.ForegroundColor = ConsoleColor.DarkRed;
                Console.WriteLine("[TaskQueue]\t驳回任务:{0}\r\n", task.Url);
#endif
            }
        }
Exemple #4
0
 public static void PostResult(SpiderTask task)
 {
     TaskQueue.PostResult(task);
 }