/// <summary>
/// 获取WebConfig.Spides下的某个子节点
/// </summary>
/// <returns></returns>
public IList <SpiderInfo> GetSpiderConfigInfo()
{
IList <SpiderInfo> spiderInfoList = new List <SpiderInfo>();
var imagesSection = ConfigurationManager.GetSection("spiders") as XmlNode;
XmlNodeList gradesNode = imagesSection.ChildNodes;
for (int i = 0; i < gradesNode.Count; i++)
{
SpiderInfo spiderInfo = new SpiderInfo();
List <string> urlList = new List <string>();
XmlNode spiderNode = gradesNode[i];
for (int j = 0; j < spiderNode.Attributes.Count; j++)
{
string name = spiderNode.Attributes[j].Name;
string value = spiderNode.Attributes[name].Value;
switch (name)
{
case "name":
spiderInfo.Name = value; break;
case "phone":
spiderInfo.Phone = value.Trim(); break;
case "template":
spiderInfo.Template = Convert.ToInt32(value.Trim()); break;
}
}
XmlNodeList childNode = spiderNode.ChildNodes;
for (int m = 0; m < childNode.Count; m++)
{
XmlNode pageNode = childNode[m];
for (int n = 0; n < pageNode.Attributes.Count; n++)
{
string name = pageNode.Attributes[n].Name;
string value = pageNode.Attributes[name].Value;
switch (name)
{
case "url":
urlList.Add(value); break;
}
}
}
spiderInfo.UrlList = urlList;
spiderInfoList.Add(spiderInfo);
}
return(spiderInfoList);
}
/// <summary>
/// www.jxdyf.com 网站页面指定抓取
/// </summary>
public void GetAppointWebSiteForJinXiang()
{
HttpHelper helper = new HttpHelper();
IList <SpiderInfo> spiderInfoList = GetSpiderConfigInfo();
for (int i = 0; i < spiderInfoList.Count; i++)
{
SpiderInfo spiderInfo = spiderInfoList[i];
for (int j = 0; j < spiderInfo.UrlList.Count; j++)
{
HttpItem item = new HttpItem();
item.URL = spiderInfo.UrlList[j];
item.Timeout = 10000;
item.ResultType = ResultType.String;
item.UserAgent = "Baiduspider+(+http://www.baidu.com/search/spider.htm)";
HttpResult detailResult = helper.GetHtml(item);
if (detailResult.StatusCode != HttpStatusCode.OK)
{
int status = (int)detailResult.StatusCode;
if (status == 0)
{
//发送给李传俊 服务器断网
SMSHelper.Send(string.Format("金象网页异常状态:{0},地址:{1}", detailResult.StatusDescription, item.URL), "13718179873");
}
else
{
//发送给配置文件指定手机
SMSHelper.Send(string.Format(spiderInfo.Name + " 你好,金象网页异常状态:{0},地址:{1}", detailResult.StatusDescription, item.URL), spiderInfo.Phone);
}
return;
}
bool isExist = ExistAppointJinXiangDetailHtml(spiderInfo.Template, detailResult);
if (isExist == false)
{
//发送短信
Log(string.Format("已向 责任人:{0} 手机:{1}发送了", spiderInfo.Name, spiderInfo.Phone));
Log(string.Format("目标地址:{0} 异常信息", item.URL));
SMSHelper.Send(string.Format("页面访问正常,但页面缺少必要的数据！,地址:{0}", item.URL), spiderInfo.Phone);
break;
}
else
{
Console.WriteLine("责任人:{0},目标地址:{1} 正常打开", spiderInfo.Name, item.URL);
}
}
}
}
