示例#1
0
        /// <summary>
        /// 执行任务
        /// </summary>
        /// <param name="allInfoUrls">任务源集合</param>
        public void DoTask(List <TaskUrlConfig> allInfoUrls)
        {
            List <Action> lss = new List <Action>();

            for (int i = 0; i < allInfoUrls.Count; i++)
            {
                //爬虫请求次数计数
                StaticConfig.Spiderinfo.RequestMount++;
                StaticConfig.CountConfig.Itotal++;
                MyParallel grabAllInfo = new MyParallel
                {
                    //接收任务对象
                    TaskUrl = allInfoUrls[i],
                    //代理初始化,默认使用本地代理
                    //需要用到阿布云代理的,可以开启fiddler代理或者把本地代理改成阿布云的代理
                    ProxyConfig =
                    {
                        // [-or-] 本地 fiddler代理阿布云
                        Proxy     = new WebProxy("192.168.1.1"),
                        // [-or-] 阿布云
                        //Proxy = new WebProxy("http://http-dyn.abuyun.com:9020"),
                        ProxyUser = "******",
                        ProxyPass = "******"
                    },
                    //初始化账号Cookie信息,下面为不需要用到账号的情况
                    CkConfig =
                    {
                        Cookie = ""
                    }
                };
                lss.Add(new MyParallel(grabAllInfo).MyRequest);
                //此处控制并发数量,并发数量可以在配置文件设置
                if (i % StaticConfig.SpConfig.ActionLssNum == StaticConfig.SpConfig.ActionLssNum - 1 ||
                    i == allInfoUrls.Count - 1)
                {
                    try
                    {
                        //开始并发
                        System.Threading.Tasks.Parallel.Invoke(lss.ToArray());
                        Console.WriteLine("***********************************************************");
                        //TODO:回收任务
                        GcTask(StaticConfig.TaskUrls, StaticConfig.AddTaskUrls);
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("并发出错:{0}>>>{1}", ex.Message, DateTime.Now);
                        CLog.DiaryLog(ex.Message,
                                      $"\\{StaticConfig.Spiderinfo.TaskName}并发出错\\{StaticConfig.Spiderinfo.TaskName}并发出错_{DateTime.Now:yyyyMMdd}.txt");
                        System.Threading.Thread.Sleep(new TimeSpan(0, 1, 0));
                    }
                    //初始化变量
                    lss = new List <Action>();
                    StaticConfig.TaskUrls    = new List <TaskUrlConfig>();
                    StaticConfig.AddTaskUrls = new List <TaskUrlConfig>();
                    System.Threading.Thread.Sleep(new TimeSpan(0, 0, StaticConfig.SpConfig.ActionSleepTime));
                    System.GC.Collect();

                    //TODO:发送请求异常处理的邮件通知
                    FixException(StaticConfig.SpConfig.ActionEmail,
                                 StaticConfig.Spiderinfo.TaskName, StaticConfig.CountConfig.ErrorCodeStr);
                    Console.WriteLine("***********************************************************");
                }
                //程序界面头部显示的信息,方便查看数据情况
                StaticConfig.CountConfig.EMessage = string.Format("{0}[{1:MMddHHmm}]【{2}/{3}>{4:0.000}-{5}】[{6}/{7}]",
                                                                  StaticConfig.Spiderinfo.TaskName, StaticConfig.SpConfig.DateKs, StaticConfig.CountConfig.Itrue,
                                                                  StaticConfig.CountConfig.Itotal,
                                                                  ((double)StaticConfig.CountConfig.Itrue / StaticConfig.CountConfig.Itotal),
                                                                  StaticConfig.CountConfig.Ifasle, i, allInfoUrls.Count);
                Console.Title = StaticConfig.CountConfig.EMessage;
                #region 通知邮件(每8小时统计一次)
                //可以根据实际情况选择性给自己发邮件的,不强制
                if (DateTime.Now.Hour % 8 == 0 && DateTime.Now.Minute == 0)
                {
                    MessageEail("每8个小时反馈");
                }
                #endregion
            }
        }
示例#2
0
 /// <summary>
 /// 带参类构造函数
 /// </summary>
 /// <param name="myParallel">我的并发类对象</param>
 public MyParallel(MyParallel myParallel)
 {
     TaskUrl     = myParallel.TaskUrl;
     ProxyConfig = myParallel.ProxyConfig;
     CkConfig    = myParallel.CkConfig;
 }