public void StartJobs() { //1. 通过ConductAPI获取一批账号的AuthorID string[] AccountIDs = ConductorClient.GetWeiboSyncTasks(CrawlerName, DCPCount).ToArray(); //2. 创建这批账号的同步任务 if (AccountIDs != null) { //提交到分析程序 ParallelOptions options = new ParallelOptions() { MaxDegreeOfParallelism = 1000 }; //最大线程数1000 Parallel.ForEach(AccountIDs, options, Item => { CrawlerFactory.AddPipeline(new Crawler.Core.Crawler.WeiboSyncCrawler(Item, CrawlerManager.CrawlerFactory), DateTime.Now); }); } //3. 将任务情况反馈到界面 //4. 启动任务 Start(); }
static void Main(string[] args) { InitWork(); //NormalizeAccount(); //TestAccount(); //RemoveAll(); //AddSubscribe(); //Console.WriteLine(DatabaseRecovery.Count()); //AddCBDs(); //DatabaseRecovery.Work(); //AuthorLocHistWorker.BackToOrigin(); //StatusSubscribeWorker.BackToOrigin(); //CBDWorker.BackToOrigin(); //AuthorCensusWorker.BackToOrigin(); //Console.WriteLine("all is well"); //return; scheduler = new Scheduler("SinaWeibo", new SchedulerSetting()); for (int j = 0; j < int.Parse(ConfigurationManager.AppSettings["WorkerThreadCount"]); ++j) { IPipeline worker = null; switch (ConfigurationManager.AppSettings["WorkerType"]) { case "WeiboAuthorCrawler": { //var tmp = new AuthorCensusWorker("WeiboUserCensus_" + j, scheduler); WeiboAuthorCrawler work = new WeiboAuthorCrawler("WeiboUserCensus_" + j, scheduler); work.DoOneJob(work); //worker = tmp; //JobCounter = tmp.CntData; //JobCounterDesc = "用户普查"; break; } case "WeiboSub": { var tmp = new Crawler.Core.Crawler.WeiboSubscribeCrawler("WeiboSub_" + j, scheduler); worker = tmp; tmp.DoOneJob(tmp); //JobCounter = tmp.CntData; JobCounterDesc = "微博订阅"; break; } case "WeiboCBD": { var tmp = new CBDWorker("WeiboCBD_" + j, scheduler); worker = tmp; WeiboCBDCrawler work = new WeiboCBDCrawler("WeiboCBD_" + j, scheduler); work.DoOneJob(work); JobCounter = tmp.CntData; JobCounterDesc = "CBD扫描"; break; } case "WeiboLocHistory": { var tmp = new AuthorLocHistWorker("WeiboLocHistory_" + j, scheduler); worker = tmp; WeiboAuthorLocHistoryCrawler work = new WeiboAuthorLocHistoryCrawler("WeiboLocHistory_" + j, scheduler); work.DoOneJob(work); JobCounter = tmp.CntData; JobCounterDesc = "地理信息"; break; } case "Forward": { WeiboForwardTrackingCrawler work = new WeiboForwardTrackingCrawler("Forward", scheduler); work.DoOneJob(work); var tmp = new LoginAccountWorker("Forward", scheduler); worker = tmp; JobCounter = tmp.CntData; JobCounterDesc = "关注订阅"; break; } case "Reply": { WeiboReplyTrackingCrawler work = new WeiboReplyTrackingCrawler("Reply" + j, scheduler); work.DoOneJob(work); var tmp = new RelationshipWorker("Reply" + j, scheduler); worker = tmp; JobCounter = tmp.CntData; JobCounterDesc = "粉丝&关注"; break; } default: break; } if (worker == null) { Console.WriteLine("工人类型配置项错误,请输入exit退出程序"); return; } scheduler.AddPipeline(worker, DateTime.Now); if (ConfigurationManager.AppSettings["WorkerType"] == "LoginAccountWorker") { break; } } _statusMonitor.ConnectToMonitorServer(); scheduler.Start(); }