static void GetPicUrlsFromBeautyPersonalPage(ImageTag imgNode, int fileIndex, int type)
{
if (imgNode.Attributes.ContainsKey("SRC") || imgNode.Attributes.ContainsKey("DATA-CFSRC"))
{
string imgUrl = imgNode.Attributes.ContainsKey("SRC") ? imgNode.GetAttribute("SRC") : imgNode.GetAttribute("DATA-CFSRC");
//2014年5月16日根据网页结构修改
//if (imgUrl.Contains("/250x0/"))
//{
// imgUrl = imgUrl.Substring(imgUrl.IndexOf("/250x0/") + 7);
// imgUrl = "http://" + imgUrl;
//}
//int startIndex = imgUrl.IndexOf("/media.curator.im/images/");
//imgUrl = "http:/" + imgUrl.Substring(startIndex);
if (!imgFileNameSet.Contains(imgUrl))
{
string imgName = "";
if (imgNode.Attributes.ContainsKey("ALT"))
{
imgName = imgNode.GetAttribute("ALT");
if (type == 2)//type为2是爬取“正妹流”中的妹子的网页的情况
{
imgName = imgName.Substring(4);
}
}
else
{
Console.WriteLine("第" + fileIndex + "张图片无法获取alt属性！");
return;
}
imgFileNameSet.Add(imgUrl);
//因为要把美女的名字作为文件夹名，所以要排除所有不能用于文件夹的字符
int invalideCharIndex = imgName.IndexOfAny(Path.GetInvalidPathChars());
while (invalideCharIndex != -1)
{
imgName = imgName.Remove(invalideCharIndex, 1);
invalideCharIndex = imgName.IndexOfAny(Path.GetInvalidPathChars());
}
//因为要把美女的名字作为文件名，所以要排除所有不能用于文件名的字符
invalideCharIndex = imgName.IndexOfAny(Path.GetInvalidFileNameChars());
while (invalideCharIndex != -1)
{
imgName = imgName.Remove(invalideCharIndex, 1);
invalideCharIndex = imgName.IndexOfAny(Path.GetInvalidFileNameChars());
}
string completeImgName = type == 1 ? saveOneDayOneBeautyBasePath + imgName : saveBeautyFlowBasePath + imgName;//和上面类似，用type来区别图片保存路径
if (!Directory.Exists(completeImgName))
{
Directory.CreateDirectory(completeImgName);
}
currentImgFileNameSet.Add(imgUrl, completeImgName + "\\" + imgName + " (" + fileIndex + ").jpg");
thunderAgent.AddTask2(imgUrl, imgName + " (" + fileIndex + ").jpg", "D:\\Download\\" + completeImgName + "\\", "", "", 1, 0, 1);
fileIndex++;
}
}
else
{
Console.WriteLine("无法获取第" + fileIndex + "张图片！");
return;
}
}
/// <summary>
/// 把内容中的图片，相对地址变为绝对地址。
/// </summary>
/// <param name="tmp_content"></param>
/// <param name="url"></param>
public static void ImageSrc(ref string tmp_content, string url)
{
//如果遇到使用绝对路径的图片，转换为全路径。
NodeList htmlNodes = new Parser(new Lexer(tmp_content.Replace("<IMG", "<img").Replace("<Img", "<img"))).Parse(new TagNameFilter("img"));
for (int j = htmlNodes.Count - 1; j >= 0; j--)
{
ImageTag link = (ImageTag)htmlNodes.ElementAt(j);
string urlpart = link.GetAttribute("src");
if (!string.IsNullOrEmpty(urlpart) && !new Regex(@"^http:").IsMatch(urlpart))
{
urlpart = new xkHttp().getDealUrl(url, urlpart);
string oldlink = link.ToHtml();
link.RemoveAttribute("src");
link.RemoveAttribute("onclick");
string newsrc = "src=\"" + urlpart + "\" ";
string newlink = link.ToHtml();
newlink = newlink.Insert(5, newsrc);
tmp_content = tmp_content.Replace(oldlink, newlink);
//EchoHelper.Echo("成功转换了一个图片的SRC属性！", "", EchoHelper.EchoType.普通信息);
}
if (!string.IsNullOrEmpty(urlpart) && new Regex(@"^\.\.").IsMatch(urlpart))
{
string oldlink = link.ToHtml();
tmp_content = tmp_content.Replace(oldlink, "");
}
}
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.UTF8);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("a"), new HasAttributeFilter("id", "PageDataList__ctl7_LinkButton1")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList.AsString();
pageInt = Convert.ToInt32(temp.GetRegexBegEnd("共", "页"));
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(
new string[] {
"__EVENTTARGET", "__EVENTARGUMENT", "__VIEWSTATE", "head1:username",
"head1:Password", "head1:rbLoginType", "Tb_keyword", "ddlNewsType", "ddlistaddnewsdate"
},
new string[] {
"PageDataList$_ctl" + (i + 1).ToString() + "$LinkButton1", "", viewState, "", "", "unit", "", "20", ""
}
);
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, nvc, Encoding.UTF8);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", " tb_list")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "通知公告";
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
headName = tr.Columns[1].ToNodePlainString();
infoUrl = "http://www.szpark.com.cn" + tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = ToolHtml.GetHtmlByUrl(infoUrl, Encoding.UTF8).GetJsString();
}
catch { }
parser = new Parser(new Lexer(htldtl));
NodeList noList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "newsinfo")));
if (noList != null && noList.Count > 0)
{
ctxHtml = noList.AsHtml().Replace("<br/>", "\r\n").Replace("<BR/>", "");
infoCtx = ctxHtml.ToCtxString().Replace(" ", "").Replace(" ", "").Replace("\t\t", "\t").Replace("\t\t", "\t");
infoCtx = Regex.Replace(infoCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace(" ", "").Replace("\t", "").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n");
msgType = MsgTypeCosnt.ShenZhenFJYLMsgType;
infoScorce = infoScorce.Replace(" ", "");
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "深圳市工程", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int m = 0; m < imgList.Count; m++)
{
try
{
ImageTag img = imgList[m] as ImageTag;
string src = img.GetAttribute("src");
if (src.ToLower().Contains(".gif"))
{
continue;
}
BaseAttach obj = null;
if (src.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(src, headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.szpark.com.cn" + src.Replace("../", "/").Replace("./", "/"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
string href = aTag.GetATagHref();
if (href.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(href, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.szpark.com.cn" + href.Replace("../", "/").Replace("./", "/"), aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <NotifyInfo>();
Dictionary <string, string> dic = new Dictionary <string, string>();
//dic.Add("盐田区", "http://yt.szzfcg.cn/portal/topicView.do?method=view&id=50074439");
dic.Add("龙华新区", "http://lhxq.szzfcg.cn/portal/topicView.do?method=view&id=110074439");
dic.Add("大鹏新区", "http://dp.szzfcg.cn/portal/topicView.do?method=view&id=100074439");
dic.Add("坪山新区", "http://ps.szzfcg.cn/portal/topicView.do?method=view&id=90074439");
dic.Add("龙岗区", "http://lg.szzfcg.cn/portal/topicView.do?method=view&id=70074439");
dic.Add("光明新区", "http://gm.szzfcg.cn/portal/topicView.do?method=view&id=10170626");
dic.Add("福田区", "http://ft.szzfcg.cn/portal/topicView.do?method=view&id=30074439");
dic.Add("罗湖区", "http://lh.szzfcg.cn/portal/topicView.do?method=view&id=20074439");
dic.Add("南山区", "http://ns.szzfcg.cn/portal/topicView.do?method=view&id=40074439");
Dictionary <string, string> dicCity = new Dictionary <string, string>();
//dicCity.Add("盐田区", "yt");
dicCity.Add("龙华新区", "lhxq");
dicCity.Add("大鹏新区", "dp");
dicCity.Add("坪山新区", "ps");
dicCity.Add("龙岗区", "lg");
dicCity.Add("光明新区", "gm");
dicCity.Add("福田区", "ft");
dicCity.Add("罗湖区", "lh");
dicCity.Add("南山区", "ns");
foreach (string key in dic.Keys)
{
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(dic[key]);
}
catch { continue; }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("select"), new HasAttributeFilter("name", "__ec_pages")));
if (pageNode != null && pageNode.Count > 0)
{
SelectTag select = pageNode[0] as SelectTag;
try
{
pageInt = int.Parse(select.OptionTags[select.OptionTags.Length - 1].Value);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
string id = dic[key].Substring(dic[key].IndexOf("id"), dic[key].Length - dic[key].IndexOf("id")).Replace("id=", "");
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"ec_i",
"topicChrList_20070702_crd",
"topicChrList_20070702_f_a",
"topicChrList_20070702_p",
"topicChrList_20070702_s_name",
"topicChrList_20070702_s_topName",
"id",
"method",
"__ec_pages",
"topicChrList_20070702_rd",
"topicChrList_20070702_f_name",
"topicChrList_20070702_f_topName",
"topicChrList_20070702_f_ldate",
}, new string[] {
"topicChrList_20070702",
"20",
"",
i.ToString(),
"",
"",
id,
"view",
i.ToString(),
"20",
"",
"",
""
});
html = this.ToolWebSite.GetHtmlByUrl(dic[key], nvc);
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "topicChrList_20070702_table")));
if (listNode != null & listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 3; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
headName = tr.Columns[1].ToNodePlainString();
releaseTime = tr.Columns[3].ToPlainTextString();
infoType = "通知公告";
msgType = "深圳市" + key + "政府采购中心";
infoUrl = "http://" + dicCity[key] + ".szzfcg.cn" + tr.Columns[1].GetATagHref();
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("align", "center")));
if (dtlNode != null && dtlNode.Count > 0)
{
ctxHtml = dtlNode[0].ToHtml();
infoCtx = ctxHtml.ToCtxString();
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "深圳政府采购", key, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
goto type;
}
ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate);
}
else
{
parser.Reset();
NodeList bodyNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("body"));
if (bodyNode != null && bodyNode.Count > 0)
{
ctxHtml = bodyNode.AsHtml();
infoCtx = ctxHtml.ToCtxString();
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "深圳政府采购", key, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate))
{
parser.Reset();
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int m = 0; m < imgList.Count; m++)
{
try
{
ImageTag img = imgList[m] as ImageTag;
string src = img.GetAttribute("src");
BaseAttach obj = null;
if (src.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(src, headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://" + dicCity[key] + ".szzfcg.cn" + src, headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
type : continue;
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("select"), new HasAttributeFilter("id", "dnn_ctr467_ArticleList_cboPages")), true), new TagNameFilter("option")));
if (pageList != null && pageList.Count > 0)
{
try
{
pageInt = pageList.Count;
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(
new string[] { "__EVENTARGUMENT", "dnn:ctr467:ArticleList:cboPages",
"ScrollTop", "__dnnVariable", "__VIEWSTATE" },
new string[] { "", (i - 1).ToString(), "", "", viewState }
);
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, nvc, Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "dnn_ctr467_ArticleList_PanelA")), true), new TagNameFilter("table")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "通知公告";
releaseTime = "20" + tr.Columns[2].ToPlainTextString().GetDateRegex("yy-MM-dd");
headName = tr.Columns[1].ToNodePlainString();
infoUrl = "http://www.szmea.net" + tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = ToolHtml.GetHtmlByUrl(SiteUrl, infoUrl, Encoding.Default).GetJsString(); //ToolHtml.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch { }
parser = new Parser(new Lexer(htldtl));
NodeList noList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "dnn_ctr391_ArticleShow_lblContent")));
if (noList != null && noList.Count > 0)
{
ctxHtml = noList.AsHtml().Replace("<br/>", "\r\n").Replace("<BR/>", "");
infoCtx = ctxHtml.ToCtxString().Replace(" ", "").Replace(" ", "").Replace("\t\t", "\t").Replace("\t\t", "\t");
infoCtx = Regex.Replace(infoCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace(" ", "").Replace("\t", "").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n");
msgType = MsgTypeCosnt.ShenZhenJLGCMsgType;
infoScorce = infoScorce.Replace(" ", "");
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "深圳市工程", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int m = 0; m < imgList.Count; m++)
{
try
{
ImageTag img = imgList[m] as ImageTag;
string src = img.GetAttribute("src");
if (src.ToLower().Contains(".gif"))
{
continue;
}
BaseAttach obj = null;
if (src.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(src, headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.szmea.net" + src.Replace("../", "/").Replace("./", "/"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
string href = aTag.GetATagHref();
if (href.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(href, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.szmea.net" + href.Replace("../", "/").Replace("./", "/"), aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { continue; }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "mtop pages")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("1/", "页");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.gsei.com.cn/index.php/cms/item-list-category-1337-page-" + i + ".shtml", Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "label_ul_b")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
ATag aTag = listNode[j].GetATag();
if (aTag == null)
{
continue;
}
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
prjName = aTag.GetAttribute("title");
beginDate = listNode[j].ToPlainTextString().GetDateRegex();
InfoUrl = aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "p8_content_show")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.ToLower().GetReplace("</p>,<br/>", "\r\n").ToCtxString();
bidUnit = bidCtx.GetBidRegex();
if (string.IsNullOrWhiteSpace(bidUnit))
{
bidUnit = bidCtx.GetRegex("第一成交候选人,第一名,中标人为,中标单位名称");
}
bidMoney = bidCtx.GetMoneyRegex(null, false, "万元");
if (string.IsNullOrWhiteSpace(bidMoney) || bidMoney == "0")
{
bidMoney = bidCtx.GetMoneyRegex(new string[] { "中标造价" }, false, "万元");
}
prjMgr = bidCtx.GetMgrRegex();
buildUnit = bidCtx.GetBuildRegex();
code = bidCtx.GetCodeRegex().GetCodeDel();
if (string.IsNullOrEmpty(bidUnit))
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
if (tableNode != null && tableNode.Count > 0)
{
TableTag tag = tableNode[0] as TableTag;
string ctx = string.Empty;
for (int r = 0; r < tag.RowCount; r++)
{
string rowName = tag.Rows[r].ToNodePlainString();
if (rowName.Contains("中标候选人名称") || rowName.Contains("中标价"))
{
for (int c = 0; c < 7; c++)
{
try
{
if (c < 3)
{
ctx += tag.Rows[r].Columns[c].ToNodePlainString().GetReplace(":,：") + "：";
}
else
{
ctx += tag.Rows[r + 1].Columns[c - 3].ToNodePlainString().GetReplace(":,：") + "：";
}
ctx += tag.Rows[r + 2].Columns[c].ToNodePlainString().GetReplace(":,：") + "\r\n";
}
catch { }
}
}
else
{
for (int c = 0; c < tag.Rows[r].ColumnCount; c++)
{
string temp = tag.Rows[r].Columns[c].ToNodePlainString();
if ((c + 1) % 2 == 0)
{
ctx += temp.GetReplace(":,：") + "\r\n";
}
else
{
ctx += temp.GetReplace(":,：") + "：";
}
}
}
if (rowName.Contains("中标候选人名称") || rowName.Contains("中标价"))
{
break;
}
}
bidUnit = ctx.GetBidRegex().GetReplace("第一名,第二名,第三名,名次");
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = ctx.GetRegex("中标候选人名称");
}
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex();
}
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = ctx.GetBuildRegex();
}
if (string.IsNullOrEmpty(prjMgr) || prjMgr.IsNumber())
{
prjMgr = ctx.GetMgrRegex();
}
if (string.IsNullOrEmpty(code))
{
code = ctx.GetCodeRegex().GetCodeDel();
}
if (string.IsNullOrEmpty(bidUnit) || bidUnit.IsNumber())
{
ctx = string.Empty;
for (int r = 0; r < tag.RowCount; r++)
{
for (int c = 0; c < tag.Rows[r].ColumnCount; c++)
{
string temp = tag.Rows[r].Columns[c].ToNodePlainString();
if ((c + 1) % 2 == 0)
{
ctx += temp.GetReplace(":,：") + "\r\n";
}
else
{
ctx += temp.GetReplace(":,：") + "：";
}
}
}
bidUnit = ctx.GetBidRegex().GetReplace("第一名,第二名,第三名,名次");
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex();
}
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = ctx.GetBuildRegex();
}
if (string.IsNullOrEmpty(prjMgr) || prjMgr.IsNumber())
{
prjMgr = ctx.GetMgrRegex();
}
if (string.IsNullOrEmpty(code))
{
code = ctx.GetCodeRegex().GetCodeDel();
}
if (string.IsNullOrEmpty(bidUnit) || bidUnit.IsNumber())
{
ctx = string.Empty;
for (int c = 0; c < tag.Rows[0].ColumnCount; c++)
{
try
{
ctx += tag.Rows[0].Columns[c].ToNodePlainString().GetReplace(":,：") + "：";
ctx += tag.Rows[1].Columns[c].ToNodePlainString().GetReplace(":,：") + "\r\n";
}
catch { }
}
bidUnit = ctx.GetBidRegex().GetReplace("第一名,第二名,第三名,名次");
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = ctx.GetRegex("中标候选人名称");
}
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex();
}
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = ctx.GetBuildRegex();
}
if (string.IsNullOrEmpty(prjMgr) || prjMgr.IsNumber())
{
prjMgr = ctx.GetMgrRegex();
}
if (string.IsNullOrEmpty(code))
{
code = ctx.GetCodeRegex().GetCodeDel();
}
}
if (string.IsNullOrEmpty(bidUnit) || bidUnit.IsNumber())
{
for (int r = 0; r < tag.RowCount; r++)
{
string rowName = tag.Rows[r].ToNodePlainString();
if (rowName.Contains("中标候选人名称") || rowName.Contains("中标价"))
{
for (int c = 0; c < tag.Rows[r].ColumnCount; c++)
{
try
{
ctx += tag.Rows[r].Columns[c].ToNodePlainString().GetReplace(":,：") + "：";
ctx += tag.Rows[r + 1].Columns[c].ToNodePlainString().GetReplace(":,：") + "\r\n";
}
catch { }
}
}
else
{
for (int c = 0; c < tag.Rows[r].ColumnCount; c++)
{
string temp = tag.Rows[r].Columns[c].ToNodePlainString();
if ((c + 1) % 2 == 0)
{
ctx += temp.GetReplace(":,：") + "\r\n";
}
else
{
ctx += temp.GetReplace(":,：") + "：";
}
}
}
if (rowName.Contains("中标候选人名称") || rowName.Contains("中标价"))
{
break;
}
}
bidUnit = ctx.GetBidRegex().GetReplace("第一名,第二名,第三名,名次");
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = ctx.GetRegex("中标候选人名称");
}
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex();
}
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = ctx.GetBuildRegex();
}
if (string.IsNullOrEmpty(prjMgr) || prjMgr.IsNumber())
{
prjMgr = ctx.GetMgrRegex();
}
if (string.IsNullOrEmpty(code))
{
code = ctx.GetCodeRegex().GetCodeDel();
}
}
}
}
}
if (bidUnit.Contains("公司"))
{
bidUnit = bidUnit.Remove(bidUnit.IndexOf("公司")) + "公司";
}
if (bidUnit.Contains("中标价"))
{
bidUnit = "";
}
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址"));
}
List <string> imgList = new List <string>();
parser = new Parser(new Lexer(HtmlTxt));
NodeList imgNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgNode != null && imgNode.Count > 0)
{
for (int m = 0; m < imgNode.Count; m++)
{
ImageTag tag = imgNode[m] as ImageTag;
string link = tag.GetAttribute("src");
string webLink = "http://www.gsei.com.cn/" + link;
HtmlTxt = HtmlTxt.GetReplace(link, webLink);
imgList.Add(webLink);
}
}
if (!bidUnit.Contains("公司") && !bidUnit.Contains("研究院") && !bidUnit.Contains("管理局") && !bidUnit.Contains("院"))
{
bidUnit = "";
}
msgType = "甘肃省信息中心";
specType = "政府采购";
bidType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("甘肃省", "甘肃省及地市", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
if (imgList.Count > 0)
{
foreach (string img in imgList)
{
string linkName = string.Empty;
if (img.Contains("/"))
{
linkName = img.Substring(img.LastIndexOf("/"));
}
else
{
linkName = img;
}
BaseAttach attach = ToolDb.GenBaseAttach(linkName, info.Id, img);
base.AttachList.Add(attach);
}
}
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.gsei.com.cn/" + a.Link.GetReplace("../,./");
}
if (Encoding.Default.GetByteCount(link) > 500)
{
continue;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected List <BidInfo> GetBidInfo(string url)
{
List <BidInfo> list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(url, Encoding.Default);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "cn6")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().Replace("(", "kdxx").GetRegexBegEnd("kdxx", ",");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(url + "index_" + (i - 1).ToString() + ".htm", Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "position2")), true), new TagNameFilter("li")));
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty,
city = string.Empty;
prjName = nodeList[j].GetATagValue("title");
if (string.IsNullOrEmpty(prjName))
{
continue;
}
if (prjName.Contains("项目所在地区"))
{
city = "广州市区";
prjName = prjName.Replace("[", "").Replace("]", "").Replace("my:项目所在地区notset区", "");
}
if (prjName.Contains("广东省"))
{
city = "广州市区";
prjName = prjName.Replace("[", "").Replace("]", "").Replace("广东省", "");
}
else
{
string temp = prjName.Replace("[", "kdxx").Replace("]", "xxdk").GetRegexBegEnd("kdxx", "xxdk");
if (!string.IsNullOrEmpty(temp))
{
prjName = prjName.Replace("[", "").Replace("]", "").Replace(temp, "");
city = temp + "区";
}
else
{
prjName = prjName.Replace("[", "").Replace("]", "");
city = "广州市区";
}
}
prjName = prjName.Replace("--中标结果", "");
if (!string.IsNullOrEmpty(prjName))
{
if (prjName[0] == '-')
{
prjName = prjName.Substring(1, prjName.Length - 1);
}
}
InfoUrl = url + nodeList[j].GetATagHref().Replace("../", "").Replace("./", "");
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch
{
Logger.Error(nodeList[j].ToNodePlainString());
continue;
}
parser = new Parser(new Lexer(htldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("cellSpacing", "1")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml().Replace("<br/>", "\r\n").Replace("<br>", "\r\n").Replace("<BR/>", "\r\n").Replace("<BR>", "\r\n").Replace("<p>", "\r\n").Replace("</p>", "\r\n").Replace("<P>", "\r\n").Replace("</P>", "\r\n");
bidCtx = HtmlTxt.ToCtxString();
buildUnit = bidCtx.GetBuildRegex();
code = bidCtx.GetCodeRegex();
prjAddress = bidCtx.GetAddressRegex();
bidType = prjName.GetInviteBidType();
bidUnit = bidCtx.GetBidRegex();
bidMoney = bidCtx.GetMoneyRegex(null, false, "万元");
if (bidMoney != "0")
{
try
{
int money = int.Parse(bidMoney);
if (money > 100000)
{
bidMoney = bidCtx.GetMoneyRegex();
}
}
catch { }
}
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = bidCtx.GetRegexBegEnd("评标结果，", "为该项目");
}
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = bidCtx.GetRegexBegEnd("第一中标候选人，", "为本工程");
}
if (string.IsNullOrEmpty(bidUnit))
{
string ctx = string.Empty;
parser = new Parser(new Lexer(HtmlTxt));
NodeList bidNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "MsoNormalTable")));
if (bidNode != null && bidNode.Count > 0)
{
TableTag table = bidNode[0] as TableTag;
if (table.RowCount > 1)
{
try
{
for (int cell = 0; cell < table.Rows[0].ColumnCount; cell++)
{
ctx += table.Rows[0].Columns[cell].ToNodePlainString().Replace(" ", "").Replace(" ", "") + "："; ctx += table.Rows[1].Columns[cell].ToNodePlainString().Replace(" ", "").Replace(" ", "") + "\r\n";
}
}
catch { }
}
else
{
parser.Reset();
bidNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "MsoTableGrid")));
if (bidNode != null && bidNode.Count > 0)
{
table = bidNode[0] as TableTag;
if (table.RowCount > 1)
{
try
{
for (int r = 0; r < table.RowCount; r++)
{
ctx += table.Rows[r].Columns[0].ToNodePlainString() + "：";
ctx += table.Rows[r].Columns[1].ToNodePlainString() + "\r\n";
}
}
catch { }
}
}
}
bidUnit = ctx.GetRegex("投标单位,第一候选人,单位名称");
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex(null, false, "万元");
if (bidMoney != "0")
{
try
{
int money = int.Parse(bidMoney);
if (money > 100000)
{
bidMoney = ctx.GetMoneyRegex();
}
}
catch { }
}
}
}
else
{
parser.Reset();
bidNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "MsoTableGrid")));
if (bidNode == null || bidNode.Count < 1)
{
parser.Reset();
bidNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
}
if (bidNode != null && bidNode.Count > 0)
{
ctx = "";
TableTag table = null;
if (bidNode.Count > 1)
{
table = bidNode[1] as TableTag;
}
else
{
table = bidNode[bidNode.Count - 1] as TableTag;
}
if (table.RowCount > 1)
{
try
{
for (int r = 0; r < table.RowCount; r++)
{
ctx += table.Rows[r].Columns[0].ToNodePlainString() + "：";
ctx += table.Rows[r].Columns[1].ToNodePlainString() + "\r\n";
}
}
catch { }
}
bidUnit = ctx.GetRegex("投标单位,单位名称");
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = ctx.GetMoneyRegex(null, false, "万元");
if (bidMoney != "0")
{
try
{
int money = int.Parse(bidMoney);
if (money > 100000)
{
bidMoney = ctx.GetMoneyRegex();
}
}
catch { }
}
}
}
}
}
if (bidUnit.Contains("中标价"))
{
bidUnit = bidUnit.Remove(bidUnit.IndexOf("中标价"));
}
if (bidUnit.Contains("报价"))
{
bidUnit = bidUnit.Remove(bidUnit.IndexOf("报价"));
}
if (bidUnit.Contains("项目负责人"))
{
bidUnit = bidUnit.Remove(bidUnit.IndexOf("项目负责人"));
}
if (bidUnit.Contains("公司"))
{
bidUnit = bidUnit.Remove(bidUnit.LastIndexOf("公司")) + "公司";
}
bidUnit = bidUnit.Replace("（牵头人）", "");
msgType = "广东省招标投标监管网";
specType = "建设工程";
if (bidCtx.IndexOf("发布日期") != -1)
{
string ctx = bidCtx.Substring(bidCtx.IndexOf("发布日期"), bidCtx.Length - bidCtx.IndexOf("发布日期"));
beginDate = ctx.GetDateRegex();
}
else if (bidCtx.IndexOf("发布时间") != -1)
{
string ctx = bidCtx.Substring(bidCtx.IndexOf("发布时间"), bidCtx.Length - bidCtx.IndexOf("发布时间"));
beginDate = ctx.GetDateRegex();
}
if (string.IsNullOrEmpty(beginDate))
{
beginDate = DateTime.Now.ToString("yyyy-MM-dd");
}
BidInfo info = ToolDb.GenBidInfo("广东省", city, "", string.Empty, code, prjName, buildUnit, beginDate,
bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType,
bidMoney, InfoUrl, prjMgr, HtmlTxt);
parser = new Parser(new Lexer(HtmlTxt));
NodeList imgNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgNode != null && imgNode.Count > 0)
{
for (int img = 0; img < imgNode.Count; img++)
{
ImageTag imgTag = imgNode[img] as ImageTag;
string tagUrl = imgTag.GetAttribute("src");
if (!string.IsNullOrWhiteSpace(tagUrl))
{
string srcUrl = InfoUrl.Remove(InfoUrl.LastIndexOf("/"));
string src = srcUrl + tagUrl.Replace("./", "/");
string attachName = tagUrl.Replace("./", "");
BaseAttach attach = null;
if (Encoding.Default.GetByteCount(attachName) < 400)
{
attach = ToolDb.GenBaseAttach(attachName, info.Id, src);
}
base.AttachList.Add(attach);
info.CtxHtml = info.CtxHtml.Replace(tagUrl, src);
}
}
}
list.Add(info);
if (list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("class", "f12")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList.AsString();
pageInt = Convert.ToInt32(temp.GetRegexBegEnd("/", "页"));
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl + "&page=" + i.ToString(), Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("height", "32")), true), new TagNameFilter("table")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 0; j < table.RowCount - 1; j++)
{
if ((j + 1) % 2 == 0)
{
continue;
}
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "通知公告";
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
headName = tr.Columns[1].ToNodePlainString();
infoUrl = "http://www.jianzhuxh.com/news/" + tr.Columns[1].GetATagValue("onclick").GetRegexBegEnd("'", "'");
string htldtl = string.Empty;
try
{
htldtl = ToolHtml.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList noList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "text18")));
if (noList != null && noList.Count > 0)
{
ctxHtml = noList[0].ToHtml().Replace("<br/>", "\r\n").Replace("<BR/>", "");
infoCtx = ctxHtml.ToCtxString().Replace(" ", "").Replace(" ", "").Replace("\t\t", "\t").Replace("\t\t", "\t");
infoCtx = Regex.Replace(infoCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace(" ", "").Replace("\t", "").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n");
msgType = MsgTypeCosnt.ShenZhenJZYMsgType;
infoScorce = infoScorce.Replace(" ", "");
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "深圳市工程", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int m = 0; m < imgList.Count; m++)
{
try
{
ImageTag img = imgList[m] as ImageTag;
string src = img.GetAttribute("src");
if (src.ToLower().Contains(".gif"))
{
continue;
}
BaseAttach obj = null;
if (src.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(src, headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.jianzhuxh.com" + src.Replace("../", "/").Replace("./", "/"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
string href = aTag.GetATagHref();
if (href.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(href, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.jianzhuxh.com" + href.Replace("../", "/").Replace("./", "/"), aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
int count = 1;
IList list = new List <CorpWarning>();
string htl = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int pageInt = 1;
string eventValidation = string.Empty;
try
{
htl = ToolWeb.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(htl));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("a"), new HasAttributeFilter("id", "lx")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.GetATagHref().GetRegexBegEnd("page=", "&");
pageInt = int.Parse(temp);
}
catch
{ }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
htl = ToolWeb.GetHtmlByUrl(this.SiteUrl + "&page=" + i.ToString(), Encoding.Default);
}
catch
{
continue;
}
}
parser = new Parser(new Lexer(htl));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "bean")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount; j++)
{
string code = string.Empty, warningName = string.Empty, deliveryDate = string.Empty, warningType = string.Empty, punishmentType = string.Empty, prjNumber = string.Empty, totalScore = string.Empty, resultScore = string.Empty, corpType = string.Empty, publicEndDate = string.Empty, warningEndDate = string.Empty, prjName = string.Empty, badInfo = string.Empty, msgType = string.Empty, color = string.Empty;
TableRow tr = table.Rows[j];
ImageTag img = tr.Columns[1].SearchFor(typeof(ImageTag), true)[0] as ImageTag;
color = img.GetAttribute("src").ToLower().Contains("red") ? "1" : "0";
warningName = tr.Columns[2].ToNodePlainString();
prjName = tr.Columns[3].ToNodePlainString();
warningType = "直接红黄色警示";
msgType = "深圳市住房和建设局";
CorpWarning info = ToolDb.GenCorpWarning("广东省", "深圳市区", "", code, warningName, deliveryDate, warningType, punishmentType, prjNumber, totalScore, resultScore, corpType, publicEndDate, warningEndDate, prjName, badInfo, msgType, color);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
count++;
if (count >= 200)
{
count = 1;
Thread.Sleep(480000);
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl), Encoding.UTF8);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "page")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList.AsString().GetRegexBegEnd("HTML", ",");
pageInt = int.Parse(temp.Replace("(", ""));
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "secondrightlistbox")), true), new TagNameFilter("ul")), true), new TagNameFilter("li")));
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty,
infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
headName = nodeList[j].ToNodePlainString().Replace("[", "").Replace("]", "");
releaseTime = nodeList[j].ToPlainTextString().GetDateRegex();
headName = headName.Replace(releaseTime, "");
infoType = "办事指南";
infoUrl = "http://www.fsggzy.cn/" + nodeList[j].GetATagHref().Replace("../", "").Replace("./", "");
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "content2")));
if (dtlList == null || dtlList.Count < 1)
{
infoUrl = "http://www.fsggzy.cn/gcjy/gc_jyzn/" + nodeList[j].GetATagHref().Replace("../", "").Replace("./", "");
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "content2")));
}
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
List <string> imgUrlLen = new List <string>();
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int d = 0; d < imgList.Count; d++)
{
ImageTag img = imgList[d] as ImageTag;
string url = img.GetAttribute("src").Replace("../", "").Replace("./", "");
if (url.ToLower().Contains("http:"))
{
imgUrlLen.Add(url);
}
else
{
string[] strLen = infoUrl.Split('/');
string value = string.Empty;
for (int k = 0; k < strLen.Length - 1; k++)
{
value += strLen[k] + "/";
}
string imgUrl = value + url;
imgUrlLen.Add(imgUrl);
}
}
}
infoCtx = dtlList.AsString().ToCtxString();
msgType = MsgTypeCosnt.FouShanMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "佛山市区", string.Empty, infoCtx, infoType);
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
else
{
sqlCount++;
if (!ToolDb.SaveEntity(info, this.ExistCompareFields))
{
if (imgUrlLen != null && imgUrlLen.Count > 0)
{
for (int c = 0; c < imgUrlLen.Count; c++)
{
try
{
BaseAttach obj = ToolHtml.GetBaseAttach(imgUrlLen[c], headName, info.Id);
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
/// <summary>
/// 从网页版微博中获取微博信息
/// </summary>
/// <param name="fansList">保存爬得的粉丝数组</param>
public void GetInfoFromHtml(List <Fan> fansList)
{
Lexer lexer = new Lexer(currentHtmlContent);
Parser parser = new Parser(lexer);
//获取包含每条微博的div标记列表
NodeList fansNodeList = parser.Parse(fanFilter);
for (int i = 0; i < fansNodeList.Size(); i++)
{
Fan fan = new Fan();
//获取包含一个粉丝的<li>标记
Bullet fanBullet = (Bullet)fansNodeList[i];
#region 获取该粉丝头像
NodeList fanPortraitNodeList = fanBullet.Children.ExtractAllNodesThatMatch(portraitFilter, true);
if (fanPortraitNodeList.Size() == 1)
{
Div fanPortraitDiv = (Div)fanPortraitNodeList[0];
NodeList imgNodeList = fanPortraitDiv.Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ImageTag)), true);
if (imgNodeList.Size() == 1)
{
ImageTag imgNode = (ImageTag)imgNodeList[0];
if (imgNode.Attributes.ContainsKey("SRC") && imgNode.Attributes.ContainsKey("ALT"))
{
string imgUrl = imgNode.GetAttribute("SRC");
string imgName = imgNode.GetAttribute("ALT");
fan.Name = imgName;
WebClient wc = new WebClient();//使用WebClient是因为下载用户头像不用登录cookie
wc.DownloadFileAsync(new Uri(imgUrl), @"portrait\" + imgName + ".jpg");
wc.DownloadFileCompleted += wc_DownloadFileCompleted;
}
else
{
Console.WriteLine("第" + i + "个粉丝中，<img>标记缺少必要的属性！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取img标记出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝头像的标准出错！");
}
#endregion
#region 获取该粉丝的关注数/粉丝数/微博数
NodeList fanConnectNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanConnectFilter, true);
if (fanConnectNodeList.Size() == 1)
{
NodeList ATagList = fanConnectNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)), true);
if (ATagList.Size() == 3)
{
for (int j = 0; j < 3; j++)
{
ATag aTag = (ATag)ATagList[j];
switch (j)
{
case 0:
if (aTag.Attributes.ContainsKey("HREF") && aTag.GetAttribute("HREF").Contains("follow"))
{
fan.FollowCount = Int32.Parse(aTag.StringText);
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝的关注数出错！");
}
break;
case 1:
if (aTag.Attributes.ContainsKey("HREF") && aTag.GetAttribute("HREF").Contains("fans"))
{
fan.FansCount = Int32.Parse(aTag.StringText);
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝的粉丝数出错！");
}
break;
default:
fan.FeedsCount = Int32.Parse(aTag.StringText);
break;
}
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝关注数/粉丝数/微博数的数量出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝关注数/粉丝数/微博数的标准出错！");
}
#endregion
#region 获取该粉丝的简介信息
NodeList fanInfoNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanInfoFilter, true);
if (fanInfoNodeList.Size() == 1)
{
//Console.WriteLine(fanInfoNodeList[0].Parent.ToHtml());
Div fanInfoDiv = (Div)fanInfoNodeList[0];
string intro = fanInfoDiv.StringText;
if (intro.Substring(0, 2).Equals("简介"))
{
fan.Introduction = intro.Substring(3, intro.Length - 3).Replace("\n", " ").Replace("\t", " ");
}
}
else
{
if (fanInfoNodeList.Size() == 0)
{
fan.Introduction = "";
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝简介的标准出错！");
}
}
#endregion
#region 获取该粉丝的UserID、地点和性别信息；校验该粉丝的用户名信息
NodeList fanLocationNodeList = fanBullet.Children.ExtractAllNodesThatMatch(fanNameFilter, true);
if (fanLocationNodeList.Size() == 1)
{
//获取粉丝的UserID信息；校验该粉丝的用户名信息
NodeList aTagNodeList = fanLocationNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)), true);
if (aTagNodeList.Size() >= 1)
{
ATag nameNode = (ATag)aTagNodeList[0];
if (nameNode.Attributes.ContainsKey("USERCARD") && nameNode.Attributes.ContainsKey("HREF"))
{
//获取粉丝的UserID信息
string uidStr = nameNode.GetAttribute("USERCARD");
if (uidStr.Substring(0, 3).Equals("id="))
{
fan.UserID = uidStr.Substring(3, uidStr.Length - 3);
}
//获取粉丝的微博链接
string linkUrl = nameNode.GetAttribute("HREF");
fan.LinkURL = "http://www.weibo.com" + linkUrl;
}
else
{
Console.WriteLine("第" + i + "个粉丝中，包含用户id和链接的<a>标记中缺少必要的属性！");
}
//校验该粉丝的用户名信息
if (!nameNode.StringText.Equals(fan.Name))
{
Console.WriteLine("第" + i + "个粉丝中，用户名与用户头像文字描述不一致！");
}
}
//获取粉丝的性别和地点信息
NodeList locationNodeList = fanLocationNodeList[0].Children.ExtractAllNodesThatMatch(new HasAttributeFilter("class", "addr"), true);
if (locationNodeList.Size() == 1)
{
string locationStr = "";
for (int j = 0; j < locationNodeList[0].Children.Size(); j++)
{
INode node = locationNodeList[0].Children[j];
if (node.GetType().Equals(typeof(TextNode)))
{
TextNode tNode = (TextNode)node;
locationStr += tNode.ToPlainTextString();
}
if (node.GetType().Equals(typeof(TagNode)))
{
TagNode tNode = (TagNode)node;
if (tNode.Attributes.ContainsKey("CLASS"))
{
if (tNode.GetAttribute("CLASS").Contains("female"))//必须先female，因为female中也含有male，如果male在前，则所有用户均符合该条件了= =
{
fan.Gender = "female";
}
else
{
if (tNode.GetAttribute("CLASS").Contains("male"))
{
fan.Gender = "male";
}
else
{
fan.Gender = "unknown";
Console.WriteLine("第" + i + "个粉丝性别不明！");
}
}
}
}
}
fan.Location = locationStr.Trim();
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取粉丝地点的标准出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝的UserID、地点和性别信息的标准出错！");
}
#endregion
#region 获取该粉丝关注用户的方式
NodeList followMethodNodeList = fanBullet.Children.ExtractAllNodesThatMatch(followMethodFilter, true);
if (followMethodNodeList.Size() == 1)
{
NodeList methodNodeList = followMethodNodeList[0].Children.ExtractAllNodesThatMatch(new NodeClassFilter(typeof(ATag)));
if (methodNodeList.Size() == 1)
{
ATag methodNode = (ATag)methodNodeList[0];
fan.FollowMethod = methodNode.StringText.Trim();
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝关注用户的方式的数量出错！");
}
}
else
{
Console.WriteLine("第" + i + "个粉丝中，获取该粉丝关注用户的方式的标准出错！");
}
#endregion
fansList.Add(fan);
}
}
protected override IList ExecuteCrawl(bool crawlAll)
{
int sqlCount = 0;
string htl = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int page = 1;
string eventValidation = string.Empty;
try
{
htl = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.UTF8, ref cookiestr);
}
catch
{
return(null);
}
Parser parser = new Parser(new Lexer(htl));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagination")), true), new TagNameFilter("a")));
if (nodeList != null && nodeList.Count > 0)
{
try
{
string temp = nodeList[nodeList.Count - 1].GetATagHref();
string pageCount = temp.Replace(temp.Remove(temp.IndexOf("=")), "").Replace("=", "");
page = int.Parse(pageCount);
}
catch { }
}
for (int i = 1; i <= page; i++)
{
if (i > 1)
{
try
{
htl = this.ToolWebSite.GetHtmlByUrl(SiteUrl + "?page=" + i.ToString(), Encoding.UTF8, ref cookiestr);
}
catch (Exception ex) { continue; }
}
parser = new Parser(new Lexer(htl));
NodeList liNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "list")), true), new TagNameFilter("li")));
if (liNode != null && liNode.Count > 0)
{
for (int j = 0; j < liNode.Count; j++)
{
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
beginDate = liNode[j].ToPlainTextString().GetDateRegex("yyyy/MM/dd");
prjName = liNode[j].ToPlainTextString().Replace(beginDate, "").ToNodeString().Replace(" ", "").Replace("·", "");
ATag aTag = liNode[j].GetATag();
InfoUrl = "http://www.yjggzy.cn" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "");
}
catch
{
continue;
}
Parser parserdetail = new Parser(new Lexer(htmldetail));
NodeList dtlNode = parserdetail.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("dl"), new HasAttributeFilter("class", "acticlecontent")));
if (dtlNode == null || dtlNode.Count < 1)
{
parserdetail.Reset();
dtlNode = parserdetail.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "nr")));
}
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.ToHtml();
bidCtx = HtmlTxt.ToCtxString().Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
bidCtx = bidCtx.Replace("\r\n\r\n", "\r\n");
bidCtx = bidCtx.Replace("\r\n\t\r\n\t", "\r\n\t");
parser = new Parser(new Lexer(HtmlTxt));
NodeList dtlNodeList = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
if (dtlNodeList != null && dtlNodeList.Count > 0)
{
string ctx = string.Empty;
TableTag tableTag = dtlNodeList[0] as TableTag;
foreach (TableRow row in tableTag.Rows)
{
int colIndex = 0;
foreach (TableColumn col in row.Columns)
{
if (row.Columns.Length == 3)
{
if (colIndex == 0 && col.GetAttribute("colspan") != "2")
{
colIndex++;
continue;
}
else if (col.GetAttribute("colspan") == "2" && colIndex == 1)
{
ctx += col.ToNodePlainString() + "：";
}
else if (!string.IsNullOrEmpty(col.GetAttribute("colspan")) && colIndex == 2)
{
ctx += col.ToNodePlainString() + "\r\n";
}
else if (string.IsNullOrEmpty(col.GetAttribute("colspan")) && colIndex == 1)
{
ctx += col.ToNodePlainString() + "：";
}
else if (string.IsNullOrEmpty(col.GetAttribute("colspan")) && colIndex == 2)
{
ctx += col.ToNodePlainString() + "\r\n";
}
colIndex++;
continue;
}
if (row.Columns.Length == 2)
{
if (colIndex == 0)
{
ctx += col.ToNodePlainString() + "：";
}
else if (colIndex == 1)
{
ctx += col.ToNodePlainString() + "\r\n";
}
colIndex++;
continue;
}
if (colIndex == 0 && col.GetAttribute("colspan") != "2")
{
colIndex++;
continue;
}
else if (colIndex == 1 && col.GetAttribute("colspan") != "2")
{
ctx += col.ToNodePlainString() + "：";
}
else if (colIndex == 2 && col.GetAttribute("colspan") != "2")
{
ctx += col.ToNodePlainString() + "\r\n";
}
else if (col.GetAttribute("colspan") == "2" && colIndex == 0)
{
ctx += col.ToNodePlainString() + "：";
}
else if (!string.IsNullOrEmpty(col.GetAttribute("colspan")) && colIndex == 1)
{
ctx += col.ToNodePlainString() + "\r\n";
}
colIndex++;
}
}
buildUnit = ctx.GetBuildRegex();
bidUnit = ctx.GetBidRegex();
code = ctx.GetCodeRegex();
prjAddress = ctx.GetAddressRegex();
prjMgr = ctx.GetMgrRegex();
if (string.IsNullOrEmpty(prjMgr))
{
prjMgr = ctx.GetRegex("项目负责人姓名", true, 50);
}
bidMoney = ctx.GetMoneyRegex();
}
else
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList imgNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgNode != null && imgNode.Count > 0)
{
ImageTag img = imgNode[0] as ImageTag;
string link = "http://www.yjggzy.cn" + img.GetAttribute("src");
HtmlTxt = HtmlTxt.GetReplace(img.GetAttribute("src"), link);
}
}
msgType = "阳江市建设工程交易中心";
specType = "建设工程";
bidType = ToolHtml.GetInviteTypes(prjName);
BidInfo info = ToolDb.GenBidInfo("广东省", "阳江市区", "", string.Empty, code, prjName, buildUnit, beginDate,
bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType,
bidMoney, InfoUrl, prjMgr, HtmlTxt);
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate, this.ExistsHtlCtx))
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ImageTag img = aNode[a] as ImageTag;
try
{
BaseAttach attach = ToolHtml.GetBaseAttach(img.GetAttribute("src"), prjName, info.Id, "SiteManage\\Files\\InviteAttach\\");
if (attach != null)
{
ToolDb.SaveEntity(attach, "SourceID,AttachServerPath");
}
}
catch { }
}
}
}
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.UTF8);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList sNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "Normal")));
if (sNode != null && sNode.Count > 0)
{
try
{
string temp = sNode.AsString().GetRegexBegEnd("/", "跳");
pageInt = int.Parse(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.szgm.gov.cn/gmbsc/143049/143173/143181/ecdc3e5c-" + i.ToString() + ".html", Encoding.UTF8);
}
catch (Exception ex) { continue; }
}
parser = new Parser(new Lexer(html));
NodeList viewList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "hyxejc")), true), new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "100%"))));
if (viewList != null && viewList.Count > 0)
{
for (int j = 1; j < viewList.Count - 1; j++)
{
TableRow tr = (viewList[j] as TableTag).Rows[0];
ATag aTag = tr.GetATag();
if (aTag == null || tr.ColumnCount != 3)
{
continue;
}
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
beginDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
prjName = aTag.GetAttribute("title");
InfoUrl = "http://www.szgm.gov.cn" + aTag.Link;
string htlDtl = string.Empty;
try
{
htlDtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>|<style[^<]*</style>|<xml[^<]*</xml>");
htlDtl = regexHtml.Replace(htlDtl, "");
}
catch { continue; }
parser = new Parser(new Lexer(htlDtl));
NodeList dtl = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "page_con")));
if (dtl != null && dtl.Count > 0)
{
HtmlTxt = dtl.AsHtml();
bidCtx = HtmlTxt.ToCtxString();
string tempName = bidCtx.GetRegex("工程名称");
if (!string.IsNullOrWhiteSpace(tempName))
{
prjName = tempName;
}
code = bidCtx.GetCodeRegex().GetCodeDel();
buildUnit = bidCtx.GetBuildRegex();
prjAddress = bidCtx.GetAddressRegex();
bidUnit = bidCtx.GetBidRegex();
if (string.IsNullOrWhiteSpace(bidUnit))
{
bidUnit = bidCtx.GetRegex("委托单位");
}
if (string.IsNullOrWhiteSpace(bidUnit))
{
bidUnit = bidCtx.GetRegexBegEnd("确认", "为中标单位");
}
bidMoney = bidCtx.GetMoneyRegex();
if (string.IsNullOrWhiteSpace(bidMoney) || bidMoney == "0")
{
bidMoney = bidCtx.GetRegex("合同价").GetMoney();
}
if (string.IsNullOrWhiteSpace(bidMoney) || bidMoney == "0")
{
bidMoney = bidCtx.GetRegexBegEnd("人民币", "元").GetMoney();
}
if (string.IsNullOrWhiteSpace(bidUnit))
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("border", "1")));
if (tableNode != null && tableNode.Count > 0)
{
TableTag tableBid = tableNode[0] as TableTag;
if (tableBid.RowCount > 1)
{
string ctx = string.Empty;
for (int c = 0; c < tableBid.Rows[0].ColumnCount; c++)
{
try
{
ctx += tableBid.Rows[0].Columns[c].ToNodePlainString() + "：";
ctx += tableBid.Rows[1].Columns[c].ToNodePlainString() + "\r\n";
}
catch { }
}
bidUnit = ctx.GetBidRegex();
if (string.IsNullOrEmpty(bidMoney) || bidMoney == "0")
{
bidMoney = ctx.GetMoneyRegex(null, false, "万元");
}
}
}
}
try
{
if (decimal.Parse(bidMoney) > 50000)
{
bidMoney = (decimal.Parse(bidMoney) / 10000).ToString();
}
}
catch { }
parser = new Parser(new Lexer(HtmlTxt));
NodeList imgNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgNode != null && imgNode.Count > 0)
{
for (int img = 0; img < imgNode.Count; img++)
{
ImageTag image = imgNode[img] as ImageTag;
string url = image.GetAttribute("src");
string saveUrl = "http://www.szgm.gov.cn" + url;
HtmlTxt = HtmlTxt.Replace(url, saveUrl);
}
}
msgType = "深圳市光明新区公明街道办事处";
if (string.IsNullOrEmpty(prjAddress))
{
prjAddress = "见招标信息";
}
specType = "政府采购";
bidType = "小型工程";
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = "深圳市光明新区公明街道办事处";
}
BidInfo info = ToolDb.GenBidInfo("广东省", "深圳区及街道工程", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType,
bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default).GetJsString();
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "font9green2")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList.GetATag(pageList.Count - 1).Link.Replace("&", "kdxx") + "kdxx";
temp = temp.GetRegexBegEnd("page=", "kdxx").Replace("&", "");
pageInt = Convert.ToInt32(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl + "&page=" + i.ToString(), Encoding.Default).GetJsString();
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "font9grey1")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "办事指南";
headName = tr.Columns[1].ToNodePlainString();
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
infoUrl = "http://www.gzzb.gd.cn" + tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "contentDiv")));
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = MsgTypeCosnt.GuangZhouMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "广州市区", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int img = 0; img < imgList.Count; img++)
{
ImageTag imgTag = imgList[img] as ImageTag;
try
{
BaseAttach obj = null;
if (imgTag.GetAttribute("src").Contains("http"))
{
obj = ToolHtml.GetBaseAttach(imgTag.GetAttribute("src"), headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.gzzb.gd.cn" + imgTag.GetAttribute("src"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
if (aTag.Link.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(aTag.Link, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.gzzb.gd.cn" + aTag.Link, aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.UTF8);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "page")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList.AsString().GetRegexBegEnd("HTML", ",").Replace("(", "");
pageInt = Convert.ToInt32(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.fsggzy.cn/gcjy/gc_zcfg/index_" + (i - 1).ToString() + ".html", Encoding.UTF8).GetJsString();
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "secondrightlistbox")), true), new TagNameFilter("table")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount; j++)
{
if (j % 2 == 0)
{
continue;
}
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "政策法规";
headName = tr.Columns[0].ToNodePlainString();
releaseTime = tr.Columns[1].ToPlainTextString().GetDateRegex();
infoUrl = "http://www.fsggzy.cn/gcjy/gc_zcfg/" + tr.Columns[0].GetATagHref().Replace("../", "").Replace("./", "");
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "content2")));
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = MsgTypeCosnt.GuangZhouMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "佛山市区", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
string str = string.Empty;
if (infoUrl.IndexOf("/") != -1)
{
str = infoUrl.Remove(infoUrl.LastIndexOf("/"));
}
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int img = 0; img < imgList.Count; img++)
{
ImageTag imgTag = imgList[img] as ImageTag;
try
{
BaseAttach obj = null;
if (imgTag.GetAttribute("src").Contains("http"))
{
obj = ToolHtml.GetBaseAttach(imgTag.GetAttribute("src"), headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach(str + imgTag.GetAttribute("src").Replace("../", "/").Replace("./", "/"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
if (aTag.Link.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(aTag.Link, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach(str + aTag.Link.Replace("../", "/").Replace("./", "/"), aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "scott")), true), new TagNameFilter("a")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList[pageList.Count - 1].GetATagValue().Replace("(", "kdxx").Replace(")", "xxdk").GetRegexBegEnd("kdxx", "xxdk");
pageInt = Convert.ToInt32(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
string typeId = ToolHtml.GetHtmlInputValue(html, "typeId");
string boardId = ToolHtml.GetHtmlInputValue(html, "boardId");
string totalRows = ToolHtml.GetHtmlInputValue(html, "totalRows");
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"typeId", "boardId", "totalRows", "pageNO"
}, new string[] {
typeId, boardId, totalRows, i.ToString()
});
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, nvc, Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "lefttable")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount - 1; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty,
infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "政策法规";
headName = tr.Columns[1].ToNodePlainString();
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
infoUrl = tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "context_div")));
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = MsgTypeCosnt.ZhongShanMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "中山市区", string.Empty, infoCtx, infoType);
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
else
{
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int img = 0; img < imgList.Count; img++)
{
ImageTag imgTag = imgList[img] as ImageTag;
BaseAttach baseInfo = ToolHtml.GetBaseAttachByUrl(imgTag.GetAttribute("src"), headName, info.Id);
if (baseInfo != null)
{
ToolDb.SaveEntity(baseInfo, string.Empty);
}
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList attachList = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (attachList != null && attachList.Count > 0)
{
for (int a = 0; a < attachList.Count; a++)
{
ATag aTag = attachList[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = ToolHtml.GetBaseAttachByUrl(aTag.Link, aTag.LinkText, info.Id);
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
int sqlCount = 0;
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8, ref cookiestr);
}
catch { return(null); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "ctl00_ContentPlaceHolder1_total_page_count")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToNodePlainString();
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"__EVENTTARGET",
"__EVENTARGUMENT",
"__VIEWSTATE",
"__EVENTVALIDATION",
"ctl00$ContentPlaceHolder1$CP"
}, new string[] {
"ctl00$ContentPlaceHolder1$DownPage",
"",
viewState,
eventValidation,
i.ToString()
});
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc, Encoding.UTF8, ref cookiestr);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "secondrightlistbox")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
INode node = listNode[j];
ATag aTag = node.GetATag();
if (aTag == null)
{
continue;
}
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
prjName = aTag.GetAttribute("title");
beginDate = node.ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.swggzy.cn/" + aTag.Link.GetReplace("amp;");
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "contentbox2")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidType = prjName.GetInviteBidType();
msgType = "汕尾市公共资源交易中心";
specType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("广东省", "汕尾市区", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate, this.ExistsHtlCtx))
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ImageTag img = aNode[a] as ImageTag;
try
{
BaseAttach attach = ToolHtml.GetBaseAttach(img.GetAttribute("src"), prjName, info.Id, "SiteManage\\Files\\InviteAttach\\");
if (attach != null)
{
ToolDb.SaveEntity(attach, "SourceID,AttachServerPath");
}
}
catch { }
}
}
}
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
}
}
}
}
return(null);
}