/// <summary> /// 获取WebConfig.Spides下的某个子节点 /// </summary> /// <returns></returns> public IList <SpiderInfo> GetSpiderConfigInfo() { IList <SpiderInfo> spiderInfoList = new List <SpiderInfo>(); var imagesSection = ConfigurationManager.GetSection("spiders") as XmlNode; XmlNodeList gradesNode = imagesSection.ChildNodes; for (int i = 0; i < gradesNode.Count; i++) { SpiderInfo spiderInfo = new SpiderInfo(); List <string> urlList = new List <string>(); XmlNode spiderNode = gradesNode[i]; for (int j = 0; j < spiderNode.Attributes.Count; j++) { string name = spiderNode.Attributes[j].Name; string value = spiderNode.Attributes[name].Value; switch (name) { case "name": spiderInfo.Name = value; break; case "phone": spiderInfo.Phone = value.Trim(); break; case "template": spiderInfo.Template = Convert.ToInt32(value.Trim()); break; } } XmlNodeList childNode = spiderNode.ChildNodes; for (int m = 0; m < childNode.Count; m++) { XmlNode pageNode = childNode[m]; for (int n = 0; n < pageNode.Attributes.Count; n++) { string name = pageNode.Attributes[n].Name; string value = pageNode.Attributes[name].Value; switch (name) { case "url": urlList.Add(value); break; } } } spiderInfo.UrlList = urlList; spiderInfoList.Add(spiderInfo); } return(spiderInfoList); }
/// <summary> /// www.jxdyf.com 网站页面指定抓取 /// </summary> public void GetAppointWebSiteForJinXiang() { HttpHelper helper = new HttpHelper(); IList <SpiderInfo> spiderInfoList = GetSpiderConfigInfo(); for (int i = 0; i < spiderInfoList.Count; i++) { SpiderInfo spiderInfo = spiderInfoList[i]; for (int j = 0; j < spiderInfo.UrlList.Count; j++) { HttpItem item = new HttpItem(); item.URL = spiderInfo.UrlList[j]; item.Timeout = 10000; item.ResultType = ResultType.String; item.UserAgent = "Baiduspider+(+http://www.baidu.com/search/spider.htm)"; HttpResult detailResult = helper.GetHtml(item); if (detailResult.StatusCode != HttpStatusCode.OK) { int status = (int)detailResult.StatusCode; if (status == 0) { //发送给李传俊 服务器断网 SMSHelper.Send(string.Format("金象网页异常状态:{0},地址:{1}", detailResult.StatusDescription, item.URL), "13718179873"); } else { //发送给配置文件指定手机 SMSHelper.Send(string.Format(spiderInfo.Name + " 你好,金象网页异常状态:{0},地址:{1}", detailResult.StatusDescription, item.URL), spiderInfo.Phone); } return; } bool isExist = ExistAppointJinXiangDetailHtml(spiderInfo.Template, detailResult); if (isExist == false) { //发送短信 Log(string.Format("已向 责任人:{0} 手机:{1}发送了", spiderInfo.Name, spiderInfo.Phone)); Log(string.Format("目标地址:{0} 异常信息", item.URL)); SMSHelper.Send(string.Format("页面访问正常,但页面缺少必要的数据!,地址:{0}", item.URL), spiderInfo.Phone); break; } else { Console.WriteLine("责任人:{0},目标地址:{1} 正常打开", spiderInfo.Name, item.URL); } } } }