示例#1
0
        private void DrillProcess(CrawlOriData pCrawlOriData)
        {
            List<DrillResult> sResultList = null;

            sResultList = DrillRegularResult(pCrawlOriData);

            sResultList.ForEach(t => {
                t.Records.ForEach(tt => {
                    crawlDbAdapter.InsertCrawlDetailResult(tt);
                });
            });
        }
示例#2
0
        private List<DrillResult> DrillRegularResult(CrawlOriData pCrawlOriData)
        {
            StringBuilder sHtmlSb = new StringBuilder(pCrawlOriData.Data.ToString());
            //执行清洗操作
            if (CleanRule != null && CleanRule.Length > 0)
            {
                string[] sCleanRules = CleanRule.Split(new string[] { "\n", "\r" }, StringSplitOptions.RemoveEmptyEntries);
                if (sCleanRules.Length > 0)
                {
                    for (int i = 0; i < sCleanRules.Length; i++)
                    {
                        string[] sCleanRule = sCleanRules[i].Split('|');
                        if (sCleanRule.Length == 1)
                        {
                            sHtmlSb.Replace(sCleanRule[0], "");
                        }
                        else if (sCleanRule.Length == 2)
                        {
                            sHtmlSb.Replace(sCleanRule[0], sCleanRule[1]);
                        }
                    }
                }
            }

            List<DrillResult> Records = new List<DrillResult>();
            //对当前html进行一个规则实例化-富血模型类
            RegScriptTransactor sRegScriptTransactor = new RegScriptTransactor(sHtmlSb.ToString());
            //(多个)记录区-规则提取
            DrillRegularRules.ForEach(t =>
            {
                if (sRegScriptTransactor.CanExe(t))
                {
                    if (t.DrillType == 0)
                    {
                        string sName = t.FeatureType == 0 ? LineFeatureType.链接.ToString() : LineFeatureType.图片.ToString();
                        DrillResult sDrillResult = new DrillResult();
                        string[] sRdData = sRegScriptTransactor.GetUrls(t, pCrawlOriData.Url);
                        //是否能找到记录区
                        if (sRdData != null && sRdData.Length > 0)
                        {
                            RegularMetaFeild sFeild = null;
                            if (t.Feilds != null && t.Feilds.Count >= 5) { sFeild = t.Feilds[1]; }
                            //生成结果集
                            for (int j = 0; j < sRdData.Length; j++)
                            {
                                sDrillResult.Records.Add(new DrillCRecord(Plot.Name, sName, sRdData[j]));
                            }
                        }
                        Records.Add(sDrillResult);
                    }
                    else
                    {
                        //高级自定义提取
                        //规则结果对象
                        DrillResult sDrillResult = new DrillResult();
                        //获取记录区片段
                        string[] sRegionHtmls = sRegScriptTransactor.GetRecordHtmls(t);
                        if (sRegionHtmls != null)
                        {
                            for (int j = 0; j < sRegionHtmls.Length; j++)
                            {
                                RegScriptTransactor sRegionTransactor = new RegScriptTransactor(sRegionHtmls[j]);
                                DrillCRecord sDrillCRecord = new DrillCRecord();
                                //sDrillCRecord.DbModelID = sDrillRule.MetaModalID;
                                SRecord sCRecord = new SRecord();
                                sDrillCRecord.Record = sCRecord;
                                //sCRecord.DbID = sDrillRule.DbID;
                                //sCRecord.Meta = new string[sDrillRule.Feilds.Count];
                                for (int k = 0; k < t.Feilds.Count; k++)
                                {
                                    RegularMetaFeild sRegularMetaFeild = t.Feilds[k];
                                    if (sRegularMetaFeild.Name == "来源链接")
                                    {
                                        sCRecord.Url = pCrawlOriData.Url;
                                    }
                                    else if (sRegularMetaFeild.Rule != null)
                                    {
                                        string sValue = sRegionTransactor.Exe(sRegularMetaFeild.Rule);
                                        sCRecord.Url = sValue;
                                        if (sRegularMetaFeild.BindType > 0 && sValue != null && sValue.Trim().Length > 0)
                                        {
                                            string[] sUrls = sValue.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
                                            if (sUrls != null && sUrls.Length > 0)
                                            {
                                                Dictionary<string, string> sUrlDic = new Dictionary<string, string>();
                                                for (int ii = 0; ii < sUrls.Length; ii++)
                                                {
                                                    string sUrl = sUrls[ii].Trim();
                                                    if (sUrlDic.ContainsKey(sUrl.ToLower()) == false)
                                                    {
                                                        sUrlDic[sUrl.ToLower()] = sUrl;
                                                        string[] sUrlSpans = sUrl.Split('\t');
                                                        if (sUrlSpans.Length > 1 && (sUrlSpans[1].StartsWith("http://") || sUrlSpans[1].StartsWith("https://")))
                                                        {
                                                            //sDrillCRecord.AddDownload(sUrlSpans[1], pData.Url, sRegularMetaFeild.BindType);
                                                        }
                                                        else if (sUrlSpans[0].StartsWith("http://") || sUrlSpans[0].StartsWith("https://"))
                                                        {
                                                            //sDrillCRecord.AddDownload(sUrlSpans[0], pData.Url, sRegularMetaFeild.BindType);
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                                sDrillResult.Records.Add(sDrillCRecord);
                            }
                        }
                        Records.Add(sDrillResult);
                    };
                }
            });
            return Records;
        }
示例#3
0
        internal void RecieveCrawlResult(CrawlResult pResult)
        {
            CrawlTask sCrawlTask;
            lock (mLocker)
            {
                if (mRunningTaskDic.TryGetValue(pResult.TaskID, out sCrawlTask) == false || sCrawlTask.List.Count != pResult.List.Count)  //未注册的任务
                {
                    return;
                }
                else
                {
                    //该任务算是完成了
                    mRunningTaskDic.Remove(pResult.TaskID);
                    for (int i = 0; i < sCrawlTask.List.Count; i++)
                    {
                        mRunningTaskDetailDic.Remove(sCrawlTask.List[i].Key);
                    }
                }
            }
            //最后时间
            mLastProduceDt = DateTime.Now;
            //将任务和结果合并成一个对象
            List<CrawlOriData> sCrawlOriDataList = new List<CrawlOriData>();
            //任务包完成的细节任务集合ID
            List<int> mFinishedIdList = new List<int>();
            for (int i = 0; i < sCrawlTask.List.Count; i++)
            {
                CrawlResultDetail sCrawlResultDetail = pResult.List[i];
                CrawlTaskDetail sCrawlTaskDetail = sCrawlTask.List[i];
                //将完成ID
                mFinishedIdList.Add(sCrawlTaskDetail.ID);
                CrawlOriData sCrawlOriData = new CrawlOriData(sCrawlTaskDetail, sCrawlResultDetail, (byte)("utf-8".Equals(Chaset, StringComparison.OrdinalIgnoreCase) ? 0 : 1));

                //存储原始数据
                crawlDbAdapter.InsertCrawlResult(sCrawlResultDetail);
                //
                sCrawlOriDataList.Add(sCrawlOriData);

            }
            //更新数据库标示,已经完成任务
            mFinishedIdList.ForEach(t => {
                crawlDbAdapter.UpdateCTask(t);
            });

            AddCrawlDatasWaitDrill(sCrawlOriDataList);
        }