예제 #1
0
        private static void BjCrawler()
        {
            TaskEntity model = new TaskEntity
            {
                OperatorName   = "张雪艳",
                Unique         = Guid.NewGuid(),
                CreateTime     = DateTime.Now,
                TaskName       = "测试任务",
                TaskStateDicId = 1,
            };

            //Proxy.Proxy.GetInstance().ProxySave(100, 10);
            //Proxy.Proxy.GetInstance().ValidateCanUse(7);
            var OperatorName = "张雪艳"; //操作人

            //待查数据初始化
            var lists = File.ReadAllLines("E:/1.txt", Encoding.Default).ToList();

            new BaseData(model).InsertMetadata(lists, OperatorName, model, taskEntity =>
            {
                //抓取数据
                var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
                bjqyxy.CrawlerWork();
            });
        }
예제 #2
0
        public ActionResult Create(TaskEntity model, HttpPostedFileBase txtfile)
        {
            var stream      = txtfile.InputStream;
            var streamread  = new StreamReader(stream, Encoding.Default);
            var companyList = new List <string>();

            while (!streamread.EndOfStream)
            {
                companyList.Add(streamread.ReadLine());
            }
            model.TaskType       = EnumTaskType.BjCrawler;
            model.TaskStateDicId = 1;
            model.TaskNum        = companyList.Count;
            model.CreateTime     = DateTime.Now;
            model.IsSingelSearch = false;
            new TaskDomain().Add(model);
            new Task(() =>
            {
                new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
                {
                    var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
                    new Task(() => { bjqyxy.CrawlerWork(); }).Start();
                });
            }).Start();
            return(Json(new { msg = $"成功上传了任务文件,系统接受到{companyList.Count}条记录,正在导入系统中。。。" }));
        }
예제 #3
0
        public ActionResult GoGather(TaskEntity model)
        {
            //Thread.Sleep(1000 * 3); //默认等待三秒
            //抓取数据
            var bjqyxy = new Crawler.Bjqyxy.BjCrawler(model, t => t.TaskGuid.Equals(model.Unique));

            new Task(() => { bjqyxy.CrawlerWork(); }).Start();
            return(Json(new { state = "nothion" }));
        }
예제 #4
0
        public ActionResult SingelSearch(string guid = null, string searchInfo = null)
        {
            CrawlerEntity crawlerEntity = null;

            if (string.IsNullOrWhiteSpace(guid) && string.IsNullOrWhiteSpace(searchInfo))
            {
                ViewBag.Guid = Guid.NewGuid();
            }
            else
            {
                ViewBag.Guid = guid;
                var count = new TargeCompanyDomain().Get(t => t.CompanyName.Equals(searchInfo))?.Count;
                if (count > 0)
                {
                    //历史记录中已存在
                }
                else
                {
                    //上网检索
                    List <string> companyList = new List <string> {
                        searchInfo
                    };
                    TaskEntity model = new TaskEntity();
                    model.TaskType       = EnumTaskType.BjCrawler;
                    model.TaskName       = $"单个任务[{DateTime.Now.ToString("G")}]";
                    model.Unique         = Conv.ToGuid(guid);
                    model.TaskStateDicId = 1;
                    model.TaskNum        = 1;
                    model.CreateTime     = DateTime.Now;
                    model.IsSingelSearch = true;
                    new TaskDomain().Add(model);
                    new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
                    {
                        Task[] tasks = new Task[4];
                        for (int i = 0; i < 4; i++)
                        {
                            tasks[i] = new Task(() =>
                            {
                                var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
                                bjqyxy.SingelSearch(searchInfo);
                            });
                            tasks[i].Start();
                        }
                        Task.WaitAny(tasks);
                    });
                }
                crawlerEntity = new CrawlerDomain().Get(t => t.搜索名称 == searchInfo && t.称 != null).FirstOrDefault();
            }
            return(View(crawlerEntity));
        }
예제 #5
0
        public ActionResult SingelSearch(string guid = null, string searchInfo = null)
        {
            CrawlerEntity crawlerEntity = null;
            if (string.IsNullOrWhiteSpace(guid) && string.IsNullOrWhiteSpace(searchInfo))
            {
                ViewBag.Guid = Guid.NewGuid();
            }
            else
            {
                ViewBag.Guid = guid;
                var count = new TargeCompanyDomain().Get(t => t.CompanyName.Equals(searchInfo))?.Count;
                if (count > 0)
                {
                    //历史记录中已存在
                }
                else
                {
                    //上网检索
                    List<string> companyList = new List<string> { searchInfo };
                    TaskEntity model = new TaskEntity();
                    model.TaskType = EnumTaskType.BjCrawler;
                    model.TaskName = $"单个任务[{DateTime.Now.ToString("G")}]";
                    model.Unique = Conv.ToGuid(guid);
                    model.TaskStateDicId = 1;
                    model.TaskNum = 1;
                    model.CreateTime = DateTime.Now;
                    model.IsSingelSearch = true;
                    new TaskDomain().Add(model);
                    new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
                    {
                        Task[] tasks = new Task[4];
                        for (int i = 0; i < 4; i++)
                        {
                            tasks[i] = new Task(() =>
                            {
                                var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
                                bjqyxy.SingelSearch(searchInfo);
                            });
                            tasks[i].Start();
                        }
                        Task.WaitAny(tasks);
                    });

                }
                crawlerEntity = new CrawlerDomain().Get(t => t.搜索名称 == searchInfo && t.名称 != null).FirstOrDefault();
            }
            return View(crawlerEntity);
        }
예제 #6
0
파일: Program.cs 프로젝트: CBDlkl/LiGather
        private static void BjCrawler()
        {
            TaskEntity model = new TaskEntity
            {
                OperatorName = "张雪艳",
                Unique = Guid.NewGuid(),
                CreateTime = DateTime.Now,
                TaskName = "测试任务",
                TaskStateDicId = 1,
            };

            //Proxy.Proxy.GetInstance().ProxySave(100, 10);
            //Proxy.Proxy.GetInstance().ValidateCanUse(7);
            var OperatorName = "张雪艳"; //操作人

            //待查数据初始化
            var lists = File.ReadAllLines("E:/1.txt", Encoding.Default).ToList();
            new BaseData(model).InsertMetadata(lists, OperatorName, model, taskEntity =>
            {
                //抓取数据
                var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
                bjqyxy.CrawlerWork();
            });
        }
예제 #7
0
 public ActionResult Create(TaskEntity model, HttpPostedFileBase txtfile)
 {
     var stream = txtfile.InputStream;
     var streamread = new StreamReader(stream, Encoding.Default);
     var companyList = new List<string>();
     while (!streamread.EndOfStream)
         companyList.Add(streamread.ReadLine());
     model.TaskType = EnumTaskType.BjCrawler;
     model.TaskStateDicId = 1;
     model.TaskNum = companyList.Count;
     model.CreateTime = DateTime.Now;
     model.IsSingelSearch = false;
     new TaskDomain().Add(model);
     new Task(() =>
     {
         new BaseData(model).InsertMetadata(companyList.ToList(), model.TaskName, model, taskEntity =>
         {
             var bjqyxy = new Crawler.Bjqyxy.BjCrawler(taskEntity, t => t.TaskGuid.Equals(taskEntity.Unique));
             new Task(() => { bjqyxy.CrawlerWork(); }).Start();
         });
     }).Start();
     return Json(new { msg = $"成功上传了任务文件,系统接受到{companyList.Count}条记录,正在导入系统中。。。" });
 }
예제 #8
0
 public ActionResult GoGather(TaskEntity model)
 {
     //Thread.Sleep(1000 * 3); //默认等待三秒
     //抓取数据
     var bjqyxy = new Crawler.Bjqyxy.BjCrawler(model, t => t.TaskGuid.Equals(model.Unique));
     new Task(() => { bjqyxy.CrawlerWork(); }).Start();
     return Json(new { state = "nothion" });
 }