/// <summary>
/// 获取一个不同域名的的生产线任务
/// </summary>
/// <param name="pInput获取生产线任务列表"></param>
/// <returns></returns>
internal static LwbResult GetCrawlTask(Input获取生产线任务列表 pInput获取生产线任务列表)
{
if (mAuthorityList == null || mAuthorityList.Count == 0)
{
mAuthorityList.Add("http://127.0.0.1:8080");
}
if (mAuthorityList.Count > 0)
{
mPos++;
if (mPos >= mAuthorityList.Count)
{
mPos = 0;
}
string sAuthority = mAuthorityList[mPos];
LwbResult sLwbResult = LwbProcess(pInput获取生产线任务列表, (int)CrawlCmd.获取生产线任务列表, sAuthority);
//先看看结果
if (sLwbResult.ResultType != LwbResultType.Success)
{
return(sLwbResult);
}
//转换结果
List <CrawlTask> sCrawlTaskList = sLwbResult.Data as List <CrawlTask>;
if (sCrawlTaskList == null)
{
return(new LwbResult(LwbResultType.Error, "爬虫抓取返回数据格式错误"));
}
if (sCrawlTaskList.Count == 0)
{
return(new LwbResult(LwbResultType.Success, "爬虫获取到的任务数量为0"));
}
sCrawlTaskList.ForEach(t =>
{
t.Authority = sAuthority;
});
return(sLwbResult);
}
else
{
return(new LwbResult(LwbResultType.Error, "未找到远程数据中心服务器"));
}
}
/// <summary>
/// 爬虫去干活去喽
/// </summary>
/// <returns></returns>
public static LwbResult DbAdapter()
{
if (!mCanAttemper)
{
return(new LwbResult(LwbResultType.Success, "爬虫正在获取任务中，请勿累死爬虫"));
}
try
{
//锁定爬虫，让其暂时不接受任务
mCanAttemper = false;
int sMax;
lock (mLocker)
{
sMax = (MaxThreads - mTaskPool.Count) > 5 ? 5 : (MaxThreads - mTaskPool.Count);
}
//爬虫已经在干任务达到30个
if (sMax == 0)
{
return(new LwbResult(LwbResultType.Success, "爬虫已经正在干将近" + MaxThreads + "个任务，让他歇会吧"));
}
List <string> sList = new List <string>();
lock (mLocker)
{
foreach (KeyValuePair <string, HostStatus> sKp in mHostDic)
{
if (sKp.Value.Busy)
{
sList.Add(sKp.Key);
}
}
}
//去远程服务器取任务
LwbResult sLwbResult = WCFServer.GetCrawlTask(new Input获取生产线任务列表 {
RuningTaskHost = sList, TaskMax = sMax
});
List <CrawlTask> sCrawlTaskList = sLwbResult.Data as List <CrawlTask>;
if (sCrawlTaskList == null)
{
return(sLwbResult);
}
sCrawlTaskList.ForEach(t =>
{
lock (mLocker)
{
//任务包缓冲池
mTaskPool[t.ID] = t;
HostStatus sHostStatus;
if (mHostDic.TryGetValue(t.Host, out sHostStatus) == false)
{
sHostStatus = new HostStatus(t.Host);
mHostDic[t.Host] = sHostStatus;
}
sHostStatus.TaskCount++;
}
ThreadPool.QueueUserWorkItem(new WaitCallback(ExeTask), t);
});
mLastAddTaskDt = DateTime.Now;
return(new LwbResult(LwbResultType.Success, "爬虫获取任务完毕，很开心"));
}
catch (Exception ee)
{
return(new LwbResult(LwbResultType.Error, "爬虫在获取任务中生病了" + ee.Message));
}
finally
{
mCanAttemper = true;
}
}
/// <summary>
/// 爬虫的执行事件
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
static void mTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
LwbResult sLwbResult = CrawlerManager.DbAdapter();
Console.WriteLine(sLwbResult.ToString());
}
