internal void RecieveCrawlResult(CrawlResult pResult) { CrawlTask sCrawlTask; lock (mLocker) { if (mRunningTaskDic.TryGetValue(pResult.TaskID, out sCrawlTask) == false || sCrawlTask.List.Count != pResult.List.Count) //未注册的任务 { return; } else { //该任务算是完成了 mRunningTaskDic.Remove(pResult.TaskID); for (int i = 0; i < sCrawlTask.List.Count; i++) { mRunningTaskDetailDic.Remove(sCrawlTask.List[i].Key); } } } //最后时间 mLastProduceDt = DateTime.Now; //将任务和结果合并成一个对象 List <CrawlOriData> sCrawlOriDataList = new List <CrawlOriData>(); //任务包完成的细节任务集合ID List <int> mFinishedIdList = new List <int>(); for (int i = 0; i < sCrawlTask.List.Count; i++) { CrawlResultDetail sCrawlResultDetail = pResult.List[i]; CrawlTaskDetail sCrawlTaskDetail = sCrawlTask.List[i]; //将完成ID mFinishedIdList.Add(sCrawlTaskDetail.ID); CrawlOriData sCrawlOriData = new CrawlOriData(sCrawlTaskDetail, sCrawlResultDetail, (byte)("utf-8".Equals(Chaset, StringComparison.OrdinalIgnoreCase) ? 0 : 1)); //存储原始数据 crawlDbAdapter.InsertCrawlResult(sCrawlResultDetail); // sCrawlOriDataList.Add(sCrawlOriData); } //更新数据库标示,已经完成任务 mFinishedIdList.ForEach(t => { crawlDbAdapter.UpdateCTask(t); }); AddCrawlDatasWaitDrill(sCrawlOriDataList); }
/// <summary> /// 单条插入数据库 /// </summary> /// <param name="pCrawlResultDetailList"></param> public void InsertCrawlResult(CrawlResultDetail pCrawlResultDetail) { using (var sqlConnection = new SqlConnection(Constant.DatabaseConnection)) { string sql = string.Format("Insert into CTaskResult values(@url,@rcontent,@isdeleted,@createdtime)"); sqlConnection.Open(); sqlConnection.Execute(sql, new { url = "", rcontent = pCrawlResultDetail.Content, isdeleted = false, createdtime = DateTime.Now }); sqlConnection.Close(); } }