/// <summary> /// 将数据库实体类列表转化成人物类实体列表 /// </summary> /// <param name="cTask"></param> /// <returns></returns> public CrawlTaskDetail ReadInner(CTask cTask) { CrawlTaskDetail crawlTaskDetail = new CrawlTaskDetail(); crawlTaskDetail.ID = cTask.Id; crawlTaskDetail.Url = cTask.Url; return(crawlTaskDetail); }
internal void RecieveCrawlResult(CrawlResult pResult) { CrawlTask sCrawlTask; lock (mLocker) { if (mRunningTaskDic.TryGetValue(pResult.TaskID, out sCrawlTask) == false || sCrawlTask.List.Count != pResult.List.Count) //未注册的任务 { return; } else { //该任务算是完成了 mRunningTaskDic.Remove(pResult.TaskID); for (int i = 0; i < sCrawlTask.List.Count; i++) { mRunningTaskDetailDic.Remove(sCrawlTask.List[i].Key); } } } //最后时间 mLastProduceDt = DateTime.Now; //将任务和结果合并成一个对象 List <CrawlOriData> sCrawlOriDataList = new List <CrawlOriData>(); //任务包完成的细节任务集合ID List <int> mFinishedIdList = new List <int>(); for (int i = 0; i < sCrawlTask.List.Count; i++) { CrawlResultDetail sCrawlResultDetail = pResult.List[i]; CrawlTaskDetail sCrawlTaskDetail = sCrawlTask.List[i]; //将完成ID mFinishedIdList.Add(sCrawlTaskDetail.ID); CrawlOriData sCrawlOriData = new CrawlOriData(sCrawlTaskDetail, sCrawlResultDetail, (byte)("utf-8".Equals(Chaset, StringComparison.OrdinalIgnoreCase) ? 0 : 1)); //存储原始数据 crawlDbAdapter.InsertCrawlResult(sCrawlResultDetail); // sCrawlOriDataList.Add(sCrawlOriData); } //更新数据库标示,已经完成任务 mFinishedIdList.ForEach(t => { crawlDbAdapter.UpdateCTask(t); }); AddCrawlDatasWaitDrill(sCrawlOriDataList); }
/// <summary> /// 添加详细采集任务 /// </summary> /// <param name="pTaskDetail"></param> internal void AddCrawlTaskDetail(CrawlTaskDetail pTaskDetail) { }