public void StartJobs()
{
//1. 通过ConductAPI获取一批账号的AuthorID
string[] AccountIDs = ConductorClient.GetWeiboSyncTasks(CrawlerName, DCPCount).ToArray();
//2. 创建这批账号的同步任务
if (AccountIDs != null)
{
//提交到分析程序
ParallelOptions options = new ParallelOptions()
{
MaxDegreeOfParallelism = 1000
}; //最大线程数1000
Parallel.ForEach(AccountIDs, options, Item =>
{
CrawlerFactory.AddPipeline(new Crawler.Core.Crawler.WeiboSyncCrawler(Item, CrawlerManager.CrawlerFactory), DateTime.Now);
});
}
//3. 将任务情况反馈到界面
//4. 启动任务
Start();
}
static void Main(string[] args)
{
InitWork();
//NormalizeAccount();
//TestAccount();
//RemoveAll();
//AddSubscribe();
//Console.WriteLine(DatabaseRecovery.Count());
//AddCBDs();
//DatabaseRecovery.Work();
//AuthorLocHistWorker.BackToOrigin();
//StatusSubscribeWorker.BackToOrigin();
//CBDWorker.BackToOrigin();
//AuthorCensusWorker.BackToOrigin();
//Console.WriteLine("all is well");
//return;
scheduler = new Scheduler("SinaWeibo", new SchedulerSetting());
for (int j = 0; j < int.Parse(ConfigurationManager.AppSettings["WorkerThreadCount"]); ++j)
{
IPipeline worker = null;
switch (ConfigurationManager.AppSettings["WorkerType"])
{
case "WeiboAuthorCrawler":
{
//var tmp = new AuthorCensusWorker("WeiboUserCensus_" + j, scheduler);
WeiboAuthorCrawler work = new WeiboAuthorCrawler("WeiboUserCensus_" + j, scheduler);
work.DoOneJob(work);
//worker = tmp;
//JobCounter = tmp.CntData;
//JobCounterDesc = "用户普查";
break;
}
case "WeiboSub":
{
var tmp = new Crawler.Core.Crawler.WeiboSubscribeCrawler("WeiboSub_" + j, scheduler);
worker = tmp;
tmp.DoOneJob(tmp);
//JobCounter = tmp.CntData;
JobCounterDesc = "微博订阅";
break;
}
case "WeiboCBD":
{
var tmp = new CBDWorker("WeiboCBD_" + j, scheduler);
worker = tmp;
WeiboCBDCrawler work = new WeiboCBDCrawler("WeiboCBD_" + j, scheduler);
work.DoOneJob(work);
JobCounter = tmp.CntData;
JobCounterDesc = "CBD扫描";
break;
}
case "WeiboLocHistory":
{
var tmp = new AuthorLocHistWorker("WeiboLocHistory_" + j, scheduler);
worker = tmp;
WeiboAuthorLocHistoryCrawler work = new WeiboAuthorLocHistoryCrawler("WeiboLocHistory_" + j, scheduler);
work.DoOneJob(work);
JobCounter = tmp.CntData;
JobCounterDesc = "地理信息";
break;
}
case "Forward":
{
WeiboForwardTrackingCrawler work = new WeiboForwardTrackingCrawler("Forward", scheduler);
work.DoOneJob(work);
var tmp = new LoginAccountWorker("Forward", scheduler);
worker = tmp;
JobCounter = tmp.CntData;
JobCounterDesc = "关注订阅";
break;
}
case "Reply":
{
WeiboReplyTrackingCrawler work = new WeiboReplyTrackingCrawler("Reply" + j, scheduler);
work.DoOneJob(work);
var tmp = new RelationshipWorker("Reply" + j, scheduler);
worker = tmp;
JobCounter = tmp.CntData;
JobCounterDesc = "粉丝&关注";
break;
}
default: break;
}
if (worker == null)
{
Console.WriteLine("工人类型配置项错误，请输入exit退出程序");
return;
}
scheduler.AddPipeline(worker, DateTime.Now);
if (ConfigurationManager.AppSettings["WorkerType"] == "LoginAccountWorker")
{
break;
}
}
_statusMonitor.ConnectToMonitorServer();
scheduler.Start();
}
