protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "p_bar")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("/", "页");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.cqzb.gov.cn/class-5-45(" + i + ").aspx");
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "ztb_list_right")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
INode node = listNode[j];
ATag aTag = node.GetATag();
prjName = aTag.GetAttribute("title");
beginDate = node.ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.cqzb.gov.cn/" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "ztb_zbxx1")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.ToCtxString();
parser = new Parser(new Lexer(HtmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
if (tableNode != null && tableNode.Count > 0)
{
TableTag table = tableNode[0] as TableTag;
string ctx = string.Empty;
for (int r = 0; r < table.RowCount; r++)
{
for (int c = 0; c < table.Rows[r].ColumnCount; c++)
{
string temp = table.Rows[r].Columns[c].ToNodePlainString();
if ((c + 1) % 2 == 0)
{
ctx += temp + "\r\n";
}
else
{
ctx += temp + "：";
}
}
}
bidUnit = ctx.GetRegex("拟中标人");
if (bidUnit.Contains("/"))
{
bidUnit = ctx.GetBidRegex();
}
if (bidUnit.Contains("/"))
{
bidUnit = ctx.GetRegex("中标人");
}
bidMoney = ctx.GetMoneyRegex();
buildUnit = ctx.GetBuildRegex();
prjAddress = ctx.GetAddressRegex();
prjMgr = ctx.GetMgrRegex();
}
try
{
bidType = prjName.GetInviteBidType();
}
catch { }
specType = "建设工程";
msgType = "重庆市招标投标综合网";
BidInfo info = ToolDb.GenBidInfo("重庆市", "重庆市及区县", area, string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.cqzb.gov.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "mtop pages")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("1/", "页");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.gsei.com.cn/index.php/cms/item-list-category-1336-page-" + i + ".shtml", Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "label_ul_b")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
ATag aTag = listNode[j].GetATag();
if (aTag == null)
{
continue;
}
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
prjName = aTag.GetAttribute("title");
beginDate = listNode[j].ToPlainTextString().GetDateRegex();
InfoUrl = aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "p8_content_show")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址"));
}
if (buildUnit.Contains("工程地点") || buildUnit.Contains("武警"))
{
buildUnit = "";
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "甘肃省信息中心";
specType = "政府采购";
inviteType = "建设工程";
InviteInfo info = ToolDb.GenInviteInfo("甘肃省", "甘肃省及地市", area, string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.gsei.com.cn/" + a.Link.GetReplace("../,./");
}
if (Encoding.Default.GetByteCount(link) > 500)
{
continue;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch { }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "huifont")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToNodePlainString().GetReplace("1/");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl + "?Paging=" + i, Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "98%")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
TableRow tr = table.Rows[j];
ATag aTag = tr.GetATag();
if (aTag == null)
{
continue;
}
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
prjName = aTag.GetAttribute("title");
beginDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
InfoUrl = "http://ggzy.xjbt.gov.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "tblInfo")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址")) + "地址";
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "新疆生产建设兵团公共资源交易中心";
specType = "政府采购";
inviteType = "建设工程";
InviteInfo info = ToolDb.GenInviteInfo("新疆维吾尔自治区", "新疆维吾尔自治区及地市", area, string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://ggzy.xjbt.gov.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
//取得页码
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagination")));
if (tdNodes != null && tdNodes.Count > 0)
{
string pageTemp = tdNodes.AsString().Replace(" ", "");
try
{
pageInt = int.Parse(pageTemp.GetRegexBegEnd("/", "页"));
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://new.sztc.com/bidBulletin/index_" + i + ".jhtml");
}
catch
{
continue;
}
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "lb-link")), true), new TagNameFilter("li")));
if (nodeList != null && nodeList.Count > 0)
{
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
ATag aTag = nodeList[j].GetATag();
prjName = aTag.LinkText.ToNodeString().Replace(" ", "");
beginDate = prjName.GetDateRegex();
if (!string.IsNullOrEmpty(prjName))
{
prjName = prjName.Replace(beginDate, "");
}
InfoUrl = aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "ninfo-con"), new TagNameFilter("div")));
if (dtnode != null && dtnode.Count > 0)
{
HtmlTxt = dtnode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
code = inviteCtx.GetCodeRegex().GetCodeDel();
buildUnit = inviteCtx.GetBuildRegex();
prjAddress = inviteCtx.GetAddressRegex();
specType = "政府采购";
msgType = "深圳市国际招标有限公司";
inviteType = prjName.GetInviteBidType();
InviteInfo info = ToolDb.GenInviteInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
dtlparser = new Parser(new Lexer(HtmlTxt));
NodeList FileTag = dtlparser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (FileTag != null && FileTag.Count > 0)
{
for (int f = 0; f < FileTag.Count; f++)
{
ATag file = FileTag[f] as ATag;
if (file.IsAtagAttach())
{
string link = string.Empty;
if (file.Link.ToLower().Contains("http"))
{
link = file.Link;
}
else
{
link = "http://new.sztc.com/" + file.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(file.ToPlainTextString(), info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch { }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "ctl00_ContentPlaceHolder1_Repeater1_ctl16_lblpc")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToNodePlainString();
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl + "&page=" + (i - 1).ToString(), Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "slist")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
INode node = listNode[j];
ATag aTag = node.GetATag();
prjName = aTag.GetAttribute("title");
beginDate = DateTime.Now.Year + "-" + node.GetSpan().StringText.ToNodeString().GetReplace(" ");
area = node.ToNodePlainString().GetReplace("[", "【").GetReplace("]", "】").GetRegexBegEnd("【", "】");
InfoUrl = "http://www.xjztb.net/Homepage/" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "print1")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址")) + "地址";
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "新疆维吾尔自治区建设工程招标投标监督管理办公室";
specType = "建设工程";
inviteType = "建设工程";
InviteInfo info = ToolDb.GenInviteInfo("新疆维吾尔自治区", "新疆维吾尔自治区及地市", area, string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.xjztb.net/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
//取得页码
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.cobo91.com/project/affiche.aspx?typeId=3", Encoding.Default, ref cookiestr);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "PageDataList")));
if (tdNodes != null && tdNodes.Count > 0)
{
string pageTemp = tdNodes.AsString().Replace(" ", "").Replace(" ", "").Trim();
Regex regpage = new Regex(@"共[0-9]+条");
try
{
int pageCount = int.Parse(regpage.Match(pageTemp).Value.Replace("共", "").Replace("条", "").Trim());
if (pageCount % 15 > 0)
{
pageInt = (pageCount / 15) + 1;
}
else
{
pageInt = pageCount / 15;
}
}
catch (Exception ex) { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"cataId",
"find_yn",
"key_word",
"typeId",
"__EVENTARGUMENT",
"__EVENTTARGET",
"__EVENTVALIDATION",
"__VIEWSTATE"
}, new string[] {
"1,2,3,4,5,6,7,8,",
string.Empty,
string.Empty,
"1,2,3,4,5,6,7,8,",
string.Empty,
"PageDataList$ctl12$LinkButton1",
eventValidation,
viewState
});
try { html = this.ToolWebSite.GetHtmlByUrl("http://www.cobo91.com/project/affiche.aspx?typeId=3", nvc, Encoding.Default, ref cookiestr); }
catch (Exception ex) { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("id", "StaffList"), new TagNameFilter("table")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount; j++)
{
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[j];
code = tr.Columns[0].ToPlainTextString().Trim();
prjName = tr.Columns[1].ToPlainTextString().Trim();
beginDate = tr.Columns[2].ToPlainTextString().Trim();
ATag aTag = tr.Columns[1].SearchFor(typeof(ATag), true)[0] as ATag;
InfoUrl = "http://www.cobo91.com/project/" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).Replace(" ", "").Trim();
Parser dtlparserHTML = new Parser(new Lexer(htmldetail));
NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("bgcolor", "#ffffff"), new TagNameFilter("table")));
HtmlTxt = dtnodeHTML.AsHtml();
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).ToLower().Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n").Replace("<p>", "\r\n").Replace("</p>", "\r\n");
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmldetail = regexHtml.Replace(htmldetail, "");
}
catch (Exception ex) { continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("bgcolor", "#ffffff"), new TagNameFilter("table")));
bidCtx = System.Web.HttpUtility.HtmlDecode(dtnode.AsString().Replace("【打印本页】", "").Replace("【关闭窗口】", "").Replace("版权所有：中邦国际招标&邦迪工程顾问", ""));
Regex regBidUnit = new Regex(@"单位(：|:)[^\r\n]+\r\n");
bidUnit = regBidUnit.Match(bidCtx).Value.Replace("单位", "").Replace("：", "").Replace(":", "").Trim();
if (bidUnit == "" || bidUnit == null)
{
bidUnit = "";
}
if (Encoding.Default.GetByteCount(bidUnit) > 150)
{
bidUnit = bidUnit.Substring(0, 150);
}
Regex regBidMoneystr = new Regex(@"(金额|价格)(：|:)[^\r\n]+\r\n");
string monerystr = regBidMoneystr.Match(bidCtx).Value.Replace("金额", "").Replace("价格", "").Replace("：", "").Replace(":", "").Replace(",", "").Replace("，", "").Trim();
Regex regBidMoney = new Regex(@"[0-9]+[.]{0,1}[0-9]+");
if (!string.IsNullOrEmpty(regBidMoney.Match(monerystr).Value))
{
if ((monerystr.Contains("万元") || monerystr.Contains("万美元")))
{
bidMoney = regBidMoney.Match(monerystr).Value;
}
else
{
try
{
bidMoney = (decimal.Parse(regBidMoney.Match(monerystr).Value) / 10000).ToString();
if (decimal.Parse(bidMoney) < decimal.Parse("0.1"))
{
bidMoney = "0";
}
}
catch (Exception)
{
bidMoney = "0";
}
}
}
if (string.IsNullOrEmpty(buildUnit))
{
buildUnit = "深圳市中邦国际招标有限公司";
}
specType = "其他";
msgType = "中邦国际招标&邦迪工程顾问";
prjName = ToolDb.GetPrjName(prjName);
bidType = ToolHtml.GetInviteTypes(prjName);
BidInfo info = ToolDb.GenBidInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int sqlCount = 0;
string html = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl + this.MaxCount);
}
catch { return(null); }
int startIndex = html.IndexOf("{");
int endIndex = html.LastIndexOf("}");
html = html.Substring(startIndex, (endIndex + 1) - startIndex);
JavaScriptSerializer serializer = new JavaScriptSerializer();
Dictionary <string, object> smsTypeJson = (Dictionary <string, object>)serializer.DeserializeObject(html);
object[] objvalues = smsTypeJson["rows"] as object[];
foreach (object objValue in objvalues)
{
Dictionary <string, object> dic = (Dictionary <string, object>)objValue;
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
string ziGeDengJi = string.Empty, ziGeZhengShu = string.Empty, zbFangShi = string.Empty;
code = Convert.ToString(dic["bdBH"]);
prjName = Convert.ToString(dic["bdName"]);
string sel = Convert.ToString(dic["zbgsStartTime"]);
try
{
beginDate = ToolHtml.GetDateTimeByLong(Convert.ToInt64(sel)).ToString();
}
catch { }
string end = Convert.ToString(dic["zbgsEndTime"]);
try
{
endDate = ToolHtml.GetDateTimeByLong(Convert.ToInt64(end)).ToString();
}
catch { }
try
{
bidType = Convert.ToString(dic["gcLeiXing2"]);
}
catch (Exception ex)
{
bidType = ToolHtml.GetInviteTypes(prjName);
}
InfoUrl = Convert.ToString(dic["detailUrl"]);
bool isJson = false;
string urll = string.Empty;
List <Dictionary <string, object> > listAttachs = new List <Dictionary <string, object> >();
try
{
urll = "https://www.szjsjy.com.cn:8001/jyw-ba/jyxx/queryOldOTDataDetail.do?type=4&id=" + dic["dbZhongBiaoJieGuoGuid"];
HtmlTxt = this.ToolWebSite.GetHtmlByUrl(urll).GetJsString().GetReplace("\\t,\\r,\\n,\"");
}
catch (Exception ex)
{
Logger.Error(prjName);
continue;
}
try
{
if (string.IsNullOrWhiteSpace(HtmlTxt))
{
string htmldtl = string.Empty;
isJson = true;
try
{
urll = "https://www.szjsjy.com.cn:8001/jyw-ba/jyxx/queryZbgs.do?guid=" + dic["dbZhongBiaoJieGuoGuid"] + "&ggGuid=&bdGuid=";
htmldtl = this.ToolWebSite.GetHtmlByUrl(urll);
}
catch (Exception ex)
{
Logger.Error(prjName);
continue;
}
Dictionary <string, object> dtlJsons = (Dictionary <string, object>)serializer.DeserializeObject(htmldtl);
buildUnit = Convert.ToString(dtlJsons["zbrAndLht"]);
bidUnit = Convert.ToString(dtlJsons["tbrName"]);
bidMoney = Convert.ToString(dtlJsons["zhongBiaoJE"]);
try
{
bidMoney = (decimal.Parse(bidMoney) / 1000000).ToString();
}
catch { }
prjMgr = Convert.ToString(dtlJsons["xiangMuJiLi"]);
Dictionary <string, object> gg = null;
try
{
gg = dtlJsons["gg"] as Dictionary <string, object>;
}
catch { }
Dictionary <string, object> bd = null;
Dictionary <string, object> gc = null;
Dictionary <string, object> xm = null;
try
{
bd = dtlJsons["bd"] as Dictionary <string, object>;
}
catch { }
try
{
gc = bd["gc"] as Dictionary <string, object>;
}
catch { }
try
{
xm = bd["xm"] as Dictionary <string, object>;
}
catch { }
try
{
if (prjMgr.Contains("----;"))
{
prjMgr = null;
}
ziGeDengJi = Convert.ToString(dtlJsons["ziGeDengJi"]);
if (ziGeDengJi.Contains("----;"))
{
ziGeDengJi = null;
}
ziGeZhengShu = Convert.ToString(dtlJsons["ziGeZhengShu"]);
if (ziGeZhengShu.Contains("----;"))
{
ziGeZhengShu = null;
}
zbFangShi = Convert.ToString(gc["zbFangShi"]);
if (zbFangShi.Contains("1"))
{
zbFangShi = "公开招标";
}
else if (zbFangShi.Contains("2"))
{
zbFangShi = "邀请招标";
}
else if (zbFangShi.Contains("4"))
{
zbFangShi = "单一来源";
}
else if (zbFangShi.Contains("5"))
{
zbFangShi = "预选招标子工程";
}
}
catch { }
string htl = this.ToolWebSite.GetHtmlByUrl(InfoUrl);
Parser parser = new Parser(new Lexer(htl));
NodeList nodelist = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "right_bg")));
if (nodelist != null && nodelist.Count > 0)
{
HtmlTxt = nodelist.AsHtml();
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"gcBH\"></span>", "<span id=\"gcBH\">" + code + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"gcName\"></span>", "<span id=\"gcBH\">" + gc["gcName"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"bdName\"></span>", "<span id=\"bdName\">" + prjName + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"xmBH\"></span>", "<span id=\"xmBH\">" + xm["xm_BH"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"xmName\"></span>", "<span id=\"xmName\">" + xm["xm_Name"] + "</span>");
}
catch { }
try
{
long zbgsStartTime = Convert.ToInt64(dtlJsons["zbgsStartTime"]);
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zbgsStartTime\"></span>", "<span id=\"zbgsStartTime\">" + ToolHtml.GetDateTimeByLong(zbgsStartTime) + "</span>");
}
catch { }
try
{
long zbgsEndTime = Convert.ToInt64(dtlJsons["zbgsEndTime"]);
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zbgsEndTime\"></span>", "<span id=\"zbgsEndTime\">" + ToolHtml.GetDateTimeByLong(zbgsEndTime) + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zbRName\"></span>", "<span id=\"zbRName\">" + gc["zbRName"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zbdlJG\"></span>", "<span id=\"zbdlJG\">" + gc["creatorName"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zbFangShi\"></span>", "<span id=\"zbFangShi\">" + zbFangShi + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"tbrName\"></span>", "<span id=\"tbrName\">" + dtlJsons["tbrName"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zhongBiaoJE\"></span>", "<span id=\"zhongBiaoJE\">" + bidMoney + "万元</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"zhongBiaoGQ\"></span>", "<span id=\"zhongBiaoGQ\">" + dtlJsons["zhongBiaoGQ"] + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"xiangMuJiLi\"></span>", "<span id=\"xiangMuJiLi\">" + prjMgr + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"ziGeDengJi\"></span>", "<span id=\"ziGeDengJi\">" + ziGeDengJi + "</span>");
}
catch { }
try
{
HtmlTxt = HtmlTxt.GetReplace("<span id=\"ziGeZhengShu\"></span>", "<span id=\"ziGeZhengShu\">" + ziGeZhengShu + "</span>");
}
catch { }
try
{
string zanding = string.IsNullOrWhiteSpace(Convert.ToString(dtlJsons["isZanDingJinE"])) ? "否" : "是";
HtmlTxt = HtmlTxt.GetReplace("<span id=\"isZanDingJinE\"></span>", "<span id=\"isZanDingJinE\">" + zanding + "</span>");
}
catch { }
}
try
{
string fileUrl = "https://www.szjsjy.com.cn:8001/jyw-ba/jyxx/filegroup/queryByGroupGuidZS.do?groupGuid=" + dtlJsons["ztbFileGroupGuid"];
string fileJson = this.ToolWebSite.GetHtmlByUrl(fileUrl);
Dictionary <string, object> fileDic = (Dictionary <string, object>)serializer.DeserializeObject(fileJson);
object[] objFile = fileDic["rows"] as object[];
foreach (object file in objFile)
{
Dictionary <string, object> attach = file as Dictionary <string, object>;
listAttachs.Add(attach);
}
}
catch { }
}
}
catch { continue; }
bidCtx = HtmlTxt.Replace("<br />", "\r\n").Replace("<BR>", "\r\n").Replace("</P>", "\r\n").ToCtxString();
if (!isJson)
{
prjAddress = bidCtx.GetAddressRegex();
buildUnit = bidCtx.GetBuildRegex();
bidUnit = bidCtx.GetBidRegex();
bidMoney = bidCtx.GetMoneyRegex();
prjMgr = bidCtx.GetMgrRegex();
if (string.IsNullOrEmpty(code))
{
code = bidCtx.GetCodeRegex();
}
}
msgType = "深圳市建设工程交易中心宝安分中心";
specType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("广东省", "深圳宝安区工程", "宝安区", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate, this.ExistsHtlCtx))
{
if (!isJson)
{
Parser parser = new Parser(new Lexer(HtmlTxt));
NodeList fileNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (fileNode != null && fileNode.Count > 0)
{
for (int f = 0; f < fileNode.Count; f++)
{
ATag tag = fileNode[f] as ATag;
if (tag.IsAtagAttach() || tag.Link.ToLower().Contains("downloadfile"))
{
try
{
BaseAttach attach = null;
string link = string.Empty;
if (tag.Link.ToLower().Contains("http"))
{
link = tag.Link;
if (link.StartsWith("\\"))
{
link = link.Substring(link.IndexOf("\\"), link.Length - link.IndexOf("\\"));
}
if (link.EndsWith("//"))
{
link = link.Remove(link.LastIndexOf("//"));
}
link = link.GetReplace("\\", "");
}
else
{
link = "https://www.szjsjy.com.cn:8001/" + tag.Link;
}
attach = ToolHtml.GetBaseAttachByUrl(link, tag.LinkText, info.Id, "SiteManage\\Files\\InviteAttach\\");
if (attach != null)
{
ToolDb.SaveEntity(attach, "SourceID,AttachServerPath");
}
}
catch { continue; }
}
}
}
}
else if (listAttachs.Count > 0)
{
foreach (Dictionary <string, object> attach in listAttachs)
{
BaseAttach attachBase = null;
try
{
string attachName = Convert.ToString(attach["attachName"]);
string attachId = Convert.ToString(attach["attachGuid"]);
string link = "https://www.szjsjy.com.cn:8001/file/downloadFile?fileId=" + attachId;
attachBase = ToolHtml.GetBaseAttach(link, attachName, info.Id, "SiteManage\\Files\\InviteAttach\\");
if (attachBase != null)
{
ToolDb.SaveEntity(attachBase, "SourceID,AttachServerPath");
}
}
catch { }
}
}
}
//if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate, this.ExistsHtlCtx))
//{
// Parser parser = new Parser(new Lexer(HtmlTxt));
// NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
// if (aNode != null && aNode.Count > 0)
// {
// for (int k = 0; k < aNode.Count; k++)
// {
// ATag a = aNode[k] as ATag;
// if (a.IsAtagAttach())
// {
// string link = string.Empty;
// if (a.Link.ToLower().Contains("http"))
// {
// link = a.Link.Replace("\\", "");
// BaseAttach attach = null;
// try
// {
// attach = ToolHtml.GetBaseAttach(link, a.LinkText, info.Id, "SiteManage\\Files\\InviteAttach\\");
// }
// catch { }
// if (attach != null)
// ToolDb.SaveEntity(attach, "");
// }
// }
// }
// }
//}
//if (!crawlAll && sqlCount >= this.MaxCount) return list;
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8);
}
catch { return(null); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "huifont")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString();
temp = temp.Substring(temp.IndexOf("/") + 1, temp.Length - temp.IndexOf("/") - 1);
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.hebggzy.cn/024/024002/" + i + ".html", Encoding.UTF8);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("li"), new HasAttributeFilter("class", "right-text-li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
INode node = listNode[j];
ATag aTag = node.GetATag();
infoType = "通知公告";
headName = aTag.GetAttribute("title");
releaseTime = node.ToPlainTextString().GetDateRegex();
infoUrl = "http://www.hebggzy.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "article-main")));
if (dtlNode != null && dtlNode.Count > 0)
{
ctxHtml = dtlNode.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = "河北省公共资源交易中心";
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "河北省", "河北省及地市", "", infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields, this.ExistsUpdate))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.Link.ToLower().Contains("download") || a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.hebggzy.cn/" + a.Link.GetReplace("../,./");
}
if (Encoding.Default.GetByteCount(link) > 500)
{
continue;
}
try
{
BaseAttach attach = ToolHtml.GetBaseAttachByUrl(link, a.LinkText, info.Id);
if (attach != null)
{
ToolDb.SaveEntity(attach, "SourceID,AttachServerPath");
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
//取得页码
string cookie = string.Empty;
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl), Encoding.UTF8, ref cookie);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("style", "float:left;")));
if (tdNodes != null && tdNodes.Count > 0)
{
try
{
string pageTemp = tdNodes[0].ToPlainTextString().Replace(" ", "").Trim();
Regex regpage = new Regex(@"共\d+页");
//string ss = regpage.Match(pageTemp).Value.Replace("页", "").Replace("共", "").Trim();
pageInt = int.Parse(regpage.Match(pageTemp).Value.Replace("共", "").Replace("页", "").Trim());
}
catch (Exception ex) { }
string cookiestr = string.Empty;
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl + "&page=" + i.ToString()), Encoding.UTF8, ref cookie);
}
catch (Exception ex)
{
continue;
}
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "tb_list")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[j];
beginDate = tr.Columns[4].ToPlainTextString().Trim();
endDate = tr.Columns[5].ToPlainTextString().Trim();
prjName = tr.Columns[1].ToPlainTextString().Replace(" ", "");
ATag aTag = tr.Columns[1].SearchFor(typeof(ATag), true)[0] as ATag;
InfoUrl = "http://zb.cmc.com.cn/TenderNotice/" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = ToolHtml.GetHtmlByUrl(SiteUrl, InfoUrl);
Parser dtlparserHTML = new Parser(new Lexer(htmldetail));
NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "ct"), new TagNameFilter("div")));
HtmlTxt = dtnodeHTML.AsHtml();
htmldetail = ToolHtml.GetHtmlByUrl(SiteUrl, InfoUrl).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n");
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmldetail = regexHtml.Replace(htmldetail, "");
}
catch (Exception ex)
{ continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "ct"), new TagNameFilter("div")));
inviteCtx = dtnode.AsString().Trim().Replace(" ", "");
Regex regCtx = new Regex(@"([\r\n]+)|([\t]+)|(\[该信息共被浏览了[0-9]+次\]\[关闭\])");
inviteCtx = regCtx.Replace(inviteCtx, "\r\n");
Regex regcode = new Regex(@"编号(：|:)[^\r\n]+\r\n");
code = regcode.Match(inviteCtx).Value.Replace("编号", "").Replace("：", "").Replace(":", "").Trim();
if (Encoding.Default.GetByteCount(code) > 50)
{
code = "";
}
specType = "其他";
msgType = "中国机械进出口（集团）有限公司";
inviteType = ToolHtml.GetInviteTypes(prjName);
InviteInfo info = ToolDb.GenInviteInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
public void DealHtml(IList list, string html, bool crawlAll)
{
Parser parserDtl = new Parser(new Lexer(html));
NodeList aNodes = parserDtl.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "lefttable")));
if (aNodes != null && aNodes.Count > 0)
{
Type typs = typeof(ATag);
TableTag table = aNodes[0] as TableTag;
for (int t = 1; t < table.RowCount - 1; t++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, bidType = string.Empty,
inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, ctx = string.Empty, CreateTime = string.Empty, FbTime = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[t] as TableRow;
ATag aTag = tr.SearchFor(typeof(ATag), true)[0] as ATag;
InfoUrl = aTag.Link;
prjName = table.Rows[t].Columns[1].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
endDate = table.Rows[t].Columns[2].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
string htmlDtl = string.Empty;
try
{
htmlDtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default);
}
catch (Exception ex)
{
continue;
}
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmlDtl = regexHtml.Replace(htmlDtl, "");
Parser parserCtx = new Parser(new Lexer(htmlDtl));
NodeList ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "printTb lefttable")));
if (ctxNode != null && ctxNode.Count > 0)
{
Parser parserdiv = new Parser(new Lexer(htmlDtl));
NodeList aNodesdiv = parserdiv.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "biuuu_button")));
HtmlTxt = ctxNode.AsHtml().Replace(aNodesdiv.AsHtml(), "").Trim();
Type tp = typeof(ATag);
TableTag tabTag = ctxNode[0] as TableTag;
string startTime = tabTag.Rows[1].Columns[0].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
Regex regex = new Regex(@"时间：\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}");
Match math = regex.Match(startTime);
beginDate = math.Value.Replace("时间：", "").Trim();
Regex regexcode = new Regex("(工程编号|项目编号)：[^\r\n]+[\r\n]{1}");
Match match = regexcode.Match(tabTag.ToPlainTextString());
if (match.Value.Length > 0)
{
code = match.Value.Substring(match.Value.IndexOf("：") + 1).Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
}
Regex regexBuildUnit = new Regex("(中标人|中标单位|中 标 人)：[^\r\n]+[\r\n]{1}");
Match matchBuildUnit = regexBuildUnit.Match(tabTag.ToPlainTextString());
if (matchBuildUnit.Value.Length > 0)
{
buildUnit = matchBuildUnit.Value.Substring(matchBuildUnit.Value.IndexOf("：") + 1).Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
}
Regex regexbidUnit = new Regex("(招标人|建设单位|招 标 人)：[^\r\n]+[\r\n]{1}");
Match matchbidUnit = regexbidUnit.Match(tabTag.ToPlainTextString());
if (matchbidUnit.Value.Length > 0)
{
bidUnit = matchbidUnit.Value.Replace("招 标 人：", "").Replace("招标人：", "").Replace("建设单位：", "").Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
}
Regex regexMoney = new Regex("(中标价|其中标价为|中标价格|中 标 价)：[^\r\n]+[\r\n]{1}");
Match matchMoney = regexMoney.Match(tabTag.ToPlainTextString());
if (matchMoney.Value.Length > 0)
{
bidMoney = matchMoney.Value.Replace("中 标 价：", "").Replace("中标价：", "").Replace("其中标价为：", "").Replace("中标价格：", "").Replace("\r", "");
}
Regex regBidMoney = new Regex(@"[0-9]+[.]{0,1}[0-9]+");
if (bidMoney.Contains("万"))
{
bidMoney = bidMoney.Remove(bidMoney.IndexOf("万")).Trim();
bidMoney = regBidMoney.Match(bidMoney).Value;
}
else
{
try
{
bidMoney = (decimal.Parse(regBidMoney.Match(bidMoney).Value) / 10000).ToString();
if (decimal.Parse(bidMoney) < decimal.Parse("0.1"))
{
bidMoney = "0";
}
}
catch (Exception)
{
bidMoney = "0";
}
}
if (Encoding.Default.GetByteCount(code) > 50)
{
code = "";
}
if (buildUnit == "" || buildUnit == null)
{
buildUnit = "";
}
if (Encoding.Default.GetByteCount(buildUnit) > 150)
{
buildUnit = buildUnit.Substring(0, 150);
}
if (bidUnit == "" || bidUnit == null)
{
bidUnit = "";
}
if (Encoding.Default.GetByteCount(bidUnit) > 150)
{
bidUnit = bidUnit.Substring(0, 150);
}
ctx = tabTag.Rows[2].Columns[0].ToPlainTextString().Replace(" ", " ").Replace("\r\n\r\n\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n");
if (ctx.Length > 0)
{
Regex regexCtx = new Regex("<!--[^<]+-->");
ctx = regexCtx.Replace(ctx, "");
}
}
parserCtx.Reset();
ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "toptd_bai")));
Regex regDate = new Regex(@"\d{4}-\d{1,2}-\d{1,2}");
beginDate = regDate.Match(ctxNode.AsString()).Value.Trim();
if (ctx.Contains("公示开始时间"))
{
beginDate = ctx.Substring(ctx.IndexOf("公示开始时间")).ToString();
Regex regBeDate = new Regex(@"\d{4}年\d{1,2}月\d{1,2}日");
beginDate = regBeDate.Match(beginDate).Value.Trim();
}
if (beginDate == "")
{
beginDate = regDate.Match(ctxNode.AsString()).Value.Trim();
}
if (beginDate == "")
{
beginDate = string.Empty;
}
prjName = ToolDb.GetPrjName(prjName);
bidType = ToolHtml.GetInviteTypes(prjName);
BidInfo info = ToolDb.GenBidInfo("广东省", "惠州市区", "惠东县", string.Empty, code, prjName, bidUnit, beginDate, buildUnit, beginDate, endDate, ctx, string.Empty, "惠州市建设工程交易中心", bidType, "建设工程", string.Empty, bidMoney, InfoUrl, string.Empty, HtmlTxt);
list.Add(info);
ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("a"), new HasAttributeFilter("target", "_blank")));
NodeList aTagNodes = ctxNode.SearchFor(typeof(ATag), true);
for (int a = 0; a < aTagNodes.Count; a++)
{
ATag fileTage = aTagNodes[a] as ATag;
if (fileTage.Link.Contains("http://www.ebc.huizhou.gov.cn/index/loadNewsFile"))
{
string downloadURL = fileTage.Link;
BaseAttach attach = ToolDb.GenBaseAttach(fileTage.ToPlainTextString(), info.Id, downloadURL);
base.AttachList.Add(attach);
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return;
}
}
}
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
string html = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int pageInt = 1;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8, ref cookiestr);
}
catch
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "zbggmore2_Pager")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("总页数：", "当前");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"__VIEWSTATE",
"__EVENTTARGET",
"__EVENTARGUMENT",
"__EVENTVALIDATION"
}, new string[] {
viewState,
"zbggmore2$Pager",
i.ToString(),
eventValidation
});
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc, Encoding.UTF8, ref cookiestr);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "zbggmore2_DataGrid1")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
TableRow tr = table.Rows[j];
ATag aTag = tr.Columns[1].GetATag();
prjName = aTag.GetAttribute("title");
area = tr.ToNodePlainString().GetRegexBegEnd("【", "】");
endDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.lnzb.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "tblInfo")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToLower().GetReplace("</p>,<br/>,<br>", "\r\n").ToCtxString();
beginDate = inviteCtx.GetRegex("发布时间").GetDateRegex("yyyy/MM/dd");
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址"));
}
msgType = "辽宁省建设厅招标投标管理处";
specType = "建设工程";
inviteType = "施工";
InviteInfo info = ToolDb.GenInviteInfo("辽宁省", "辽宁省及地市", area, string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "scott")), true), new TagNameFilter("a")));
if (pageList != null && pageList.Count > 0)
{
try
{
string temp = pageList[pageList.Count - 1].GetATagValue().Replace("(", "kdxx").Replace(")", "xxdk").GetRegexBegEnd("kdxx", "xxdk");
pageInt = Convert.ToInt32(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "lefttable")));
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 1; j < table.RowCount - 1; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty,
infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "办事指南";
headName = tr.Columns[1].ToNodePlainString();
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
infoUrl = tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString().Replace("<?xml:namespace prefix = o ns = \"urn:schemas-microsoft-com:office:office\" />", "");
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "context_div")));
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = MsgTypeCosnt.ZhongShanMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "中山市区", string.Empty, infoCtx, infoType);
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
else
{
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int img = 0; img < imgList.Count; img++)
{
ImageTag imgTag = imgList[img] as ImageTag;
BaseAttach baseInfo = ToolHtml.GetBaseAttachByUrl(imgTag.GetAttribute("src"), headName, info.Id);
if (baseInfo != null)
{
ToolDb.SaveEntity(baseInfo, string.Empty);
}
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList attachList = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (attachList != null && attachList.Count > 0)
{
for (int a = 0; a < attachList.Count; a++)
{
ATag aTag = attachList[a] as ATag;
if (aTag.IsAtagAttach())
{
BaseAttach obj = ToolHtml.GetBaseAttachByUrl(aTag.Link, aTag.LinkText, info.Id);
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8, ref cookiestr);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "ID_ucBiddingList_ucPager1_lbPage")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().Replace("1/", "");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"__EVENTTARGET",
"__EVENTARGUMENT",
"__VIEWSTATE",
"ID_ucBiddingList$txtTitle",
"ID_ucBiddingList$ucPager1$listPage"
}, new string[] {
"ID_ucBiddingList$ucPager1$btnNext",
"",
viewState,
"",
(i - 1).ToString()
});
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc, Encoding.UTF8, ref cookiestr);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("cellPadding", "0")));
if (listNode != null && listNode.Count > 2)
{
TableTag table = listNode[2] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
TableRow tr = table.Rows[j];
ATag aTag = tr.GetATag();
prjName = aTag.GetAttribute("title");
beginDate = tr.Columns[0].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.hnztb.org" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "Table3")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址")) + "地址";
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "湖南省住房和城乡建设厅";
specType = "建设工程";
inviteType = prjName.GetInviteBidType();
buildUnit = buildUnit.Replace(" ", "");
InviteInfo info = ToolDb.GenInviteInfo("湖南省", "湖南省及地市", area, string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default, ref cookiestr);
}
catch { return(null); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "TblOSInfoList1_Pager")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToNodePlainString().GetRegexBegEnd("总页数：", "当前");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"__VIEWSTATE",
"TblOSInfoList1:KeyWord",
"__EVENTTARGET",
"__EVENTARGUMENT"
}, new string[] {
viewState,
"",
"TblOSInfoList1:Pager",
i.ToString()
});
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc, Encoding.Default, ref cookiestr);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "TblOSInfoList1_DataGrid1")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
TableRow tr = table.Rows[j];
ATag aTag = tr.Columns[1].GetATag();
prjName = aTag.GetAttribute("title").GetReplace(";");
code = tr.Columns[1].ToNodePlainString().GetReplace("[", "【").GetReplace("]", "】").GetRegexBegEnd("【", "】");
beginDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
if (!aTag.Link.Contains("http"))
{
string f = aTag.Link;
InfoUrl = "http://www.zmctc.com/zjgcjy/Notice/" + aTag.Link;
}
else
{
InfoUrl = aTag.Link;
}
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
if (htmldtl.Contains("�"))
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "news")));
if (dtlNode == null || dtlNode.Count < 1)
{
parser.Reset();
dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "olds")));
}
if (dtlNode == null || dtlNode.Count < 1)
{
parser.Reset();
dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("id", "TDContent")));
}
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.ToCtxString();
parser = new Parser(new Lexer(HtmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "Pbgginfodetailnew1_company")));
if (tableNode == null || tableNode.Count < 1)
{
parser.Reset();
tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "PbggInfoDetail1_company")));
}
if (tableNode == null || tableNode.Count < 1)
{
parser.Reset();
tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("rules", "all")));
}
if (tableNode != null && tableNode.Count > 0)
{
TableTag tag = tableNode[0] as TableTag;
string ctx = string.Empty;
for (int r = 0; r < tag.Rows[0].ColumnCount; r++)
{
try
{
ctx += tag.Rows[0].Columns[r].ToNodePlainString() + "：";
ctx += tag.Rows[1].Columns[r].ToNodePlainString() + "\r\n";
}
catch { }
}
if (!ctx.Contains("否决投标"))
{
bidUnit = ctx.GetBidRegex();
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = ctx.GetRegex("中标候选人");
}
bidMoney = ctx.GetMoneyRegex();
prjMgr = ctx.GetMgrRegex();
if (string.IsNullOrEmpty(prjMgr))
{
prjMgr = ctx.GetRegex("项目经理/总监");
}
}
}
msgType = "浙江省公共资源交易中心";
specType = "政府采购";
bidType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("浙江省", "浙江省及地市", area, string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://downc.zmctc.com/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <NoticeInfo>();
int pageInt = 37;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
string url = "http://www.gsggzyjy.cn/ajax/Controls_InfoList,App_Web_rzplwhmc.ashx?_method=getCurrentData&_session=rw";
try
{
this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8, ref cookiestr);
string post = "currentPage=1\r\nQuery=";
html = ToolHtml.GetHtmlByUrlPost(url, post, Encoding.UTF8, ref cookiestr);
}
catch { return(null); }
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
string post = "currentPage=" + i + "\r\nQuery=";
html = ToolHtml.GetHtmlByUrlPost(url, post, Encoding.UTF8, ref cookiestr);
}
catch { continue; }
}
Parser parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("li"));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string InfoTitle = string.Empty, InfoType = string.Empty, PublistTime = string.Empty, InfoCtx = string.Empty, InfoUrl = string.Empty, prjCode = string.Empty, buildUnit = string.Empty, htmlTxt = string.Empty, area = string.Empty;
INode node = listNode[j];
ATag aTag = node.GetATag();
InfoTitle = aTag.GetAttribute("title");
InfoType = "控制价公示";
PublistTime = node.GetSpan().StringText;
InfoUrl = "http://www.gsggzyjy.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "ContentPlaceHolder1_InfoHtml")));
if (dtlNode != null && dtlNode.Count > 0)
{
htmlTxt = dtlNode.AsHtml();
InfoCtx = htmlTxt.ToCtxString();
parser = new Parser(new Lexer(htmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
if (tableNode != null && tableNode.Count > 0)
{
string ctx = string.Empty;
TableTag table = tableNode[0] as TableTag;
for (int r = 0; r < table.RowCount; r++)
{
for (int c = 0; c < table.Rows[r].ColumnCount; c++)
{
string temp = table.Rows[r].Columns[c].ToNodePlainString();
if ((c + 1) % 2 == 0)
{
ctx += temp.GetReplace(":,：") + "\r\n";
}
else
{
ctx += temp.GetReplace(":,：") + "：";
}
}
}
buildUnit = ctx.GetBuildRegex();
}
NoticeInfo info = ToolDb.GenNoticeInfo("甘肃省", "甘肃省及地市", area, string.Empty, InfoTitle, InfoType, InfoCtx, PublistTime, string.Empty, "甘肃省公共资源交易中心", InfoUrl, prjCode, buildUnit, string.Empty, string.Empty, "房建市政工程", string.Empty, htmlTxt);
parser = new Parser(new Lexer(htmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k].GetATag();
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.gsggzyjy.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
string str = System.Web.HttpUtility.UrlDecode("appid=1&webid=1&path=%2F&columnid=808&sourceContentType=1&unitid=620&webname=%E6%B5%99%E6%B1%9F%E7%9C%81%E5%8F%91%E5%B1%95%E5%92%8C%E6%94%B9%E9%9D%A9%E5%A7%94%E5%91%98%E4%BC%9A&permissiontype=0");
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"col",
"appid",
"webid",
"path",
"columnid",
"sourceContentType",
"unitid",
"webname",
"permissiontype"
},
new string[] {
"1",
"1",
"1",
"/",
"148",
"1",
"363",
"江苏政务服务网",
"0"
});
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc);
}
catch { return(null); }
try
{
string temp = html.GetRegexBegEnd("<totalpage>", "</totalpage>");
pageInt = int.Parse(temp);
}
catch { }
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"col",
"appid",
"webid",
"path",
"columnid",
"sourceContentType",
"unitid",
"webname",
"permissiontype"
},
new string[] {
"1",
"1",
"1",
"/",
"148",
"1",
"363",
"江苏政务服务网",
"0"
});
try
{
int endrecord = i * 45;
int startrecord = 45 * i - 44;
html = this.ToolWebSite.GetHtmlByUrl("http://www.jszwfw.gov.cn/module/jslib/jquery/jpage/dataproxy.jsp?perpage=15&endrecord=" + endrecord + "&startrecord=" + startrecord, nvc);
}
catch { continue; }
}
Parser parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "99%")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
TableRow tr = (listNode[j] as TableTag).Rows[0];
ATag aTag = tr.Columns[1].GetATag();
prjName = aTag.GetAttribute("title");
if (prjName.Contains(" "))
{
string[] str = prjName.Split(' ');
code = str[0];
prjName = str[1];
}
else
{
string str = prjName.GetNotChina();
if (str.Length > 2 && prjName.IsNumber())
{
try
{
int index = prjName.IndexOf(str.Substring(0, 2));
code = prjName.Substring(0, index);
prjName = prjName.Substring(index, prjName.Length - index);
}
catch { }
}
}
beginDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.jszwfw.gov.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "zoom")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.ToLower().GetReplace("</p>,</br>", "\r\n").ToCtxString();
buildUnit = bidCtx.GetBuildRegex();
if (string.IsNullOrEmpty(code))
{
code = bidCtx.GetCodeRegex().GetCodeDel();
}
bidUnit = bidCtx.GetBidRegex().GetReplace("名称");
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = bidCtx.GetRegex("第一中标候选单位为,第一名,中标（成交）候选人名称").GetReplace("名称");
}
bidMoney = bidCtx.GetMoneyRegex();
if (bidMoney == "0" || string.IsNullOrEmpty(bidMoney))
{
bidMoney = bidCtx.GetMoneyRegex(null, true);
}
prjMgr = bidCtx.GetMgrRegex();
try
{
if (decimal.Parse(bidMoney) > 100000)
{
bidMoney = (decimal.Parse(bidMoney) / 10000).ToString();
}
}
catch { }
msgType = "江苏省政务服务管理办公室";
specType = "政府采购";
bidType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("江苏省", "江苏省及地市", area, string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.jszwfw.gov.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
//取得页码
int pageInt = 1, sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default).GetJsString();
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "95%")));
if (nodeList != null && nodeList.Count > 0)
{
List <INode> list = new List <INode>();
list.Add(nodeList[10]);
list.Add(nodeList[4]);
list.Add(nodeList[2]);
foreach (INode t in list)
{
TableTag table = t as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty,
infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "政策法规";
headName = tr.Columns[0].ToNodePlainString();
releaseTime = tr.Columns[1].ToPlainTextString().GetDateRegex();
infoUrl = "http://market.meizhou.gov.cn" + tr.Columns[0].GetATagValue("onclick").GetRegexBegEnd(",'", "',");
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch
{
continue;
}
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("align", "center")));
if (dtlList != null && dtlList.Count > 0)
{
if (dtlList.Count > 1)
{
ctxHtml = dtlList[1].ToHtml();
}
else
{
ctxHtml = dtlList.ToHtml();
}
infoCtx = ctxHtml.ToCtxString().Replace(">", "");
msgType = MsgTypeCosnt.MeiZhouMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "梅州市区", string.Empty, infoCtx, infoType);
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
else
{
sqlCount++;
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(htldtl));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach baseInfo = ToolHtml.GetBaseAttach("http://market.meizhou.gov.cn" + aTag.Link, aTag.LinkText, info.Id);
if (baseInfo != null)
{
ToolDb.SaveEntity(baseInfo, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
string html = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int pageInt = 1;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default);
}
catch (Exception)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("li"), new HasAttributeFilter("class", "wb-page-li")), true), new TagNameFilter("a")));
if (pageList != null && pageList.Count > 0)
{
try
{
ATag atag = pageList[pageList.Count - 2] as ATag;
string temp = atag.LinkText.GetReplace("1/", "");
pageInt = int.Parse(temp);
}
catch
{ pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.mzggzy.com/TPFront/jsgc/004001/?pageing=" + i.ToString(), Encoding.Default);
}
catch
{
continue;
}
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "ewb-data-items ewb-pt6")), true), new TagNameFilter("li")));
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
ATag aTag = nodeList[j].GetATag();
prjName = aTag.ChildrenHTML.GetReplace("\r\n", "").ToRegString();
beginDate = nodeList[j].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.mzggzy.com" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "infodetail")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (string.IsNullOrWhiteSpace(buildUnit))
{
buildUnit = inviteCtx.GetRegex("招标人（盖章）");
if (string.IsNullOrWhiteSpace(buildUnit))
{
buildUnit = inviteCtx.GetRegex("招标人（盖单位章）");
}
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "梅州市公共资源交易中心";
specType = "建设工程";
InviteInfo info = ToolDb.GenInviteInfo("广东省", "梅州市区", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl), Encoding.Default);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList sNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("noWrap", "true")));
if (sNode != null && sNode.Count > 0)
{
try
{
string temp = sNode.AsString().ToNodeString();
Regex reg = new Regex(@"/[^页]+页");
string page = reg.Match(temp).Value.Replace("/", "").Replace("页", "");
pageInt = Convert.ToInt32(page);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(
new string[]
{ "__VIEWSTATE",
"__EVENTTARGET",
"__EVENTARGUMENT",
"__EVENTVALIDATION",
"tbSearchText" },
new string[] {
viewState,
"pager",
i.ToString(),
eventValidation,
""
});
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, nvc, Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "ShowListMiddleContent")));
if (nodeList != null && nodeList.Count > 0)
{
parser = new Parser(new Lexer(nodeList.ToHtml()));
NodeList viewList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "ShowListTitle")));
parser = new Parser(new Lexer(nodeList.ToHtml()));
NodeList timeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "ShowListTime")));
if (viewList != null && viewList.Count > 0 && timeList != null && timeList.Count > 0 && timeList.Count == viewList.Count)
{
for (int j = 0; j < viewList.Count; j++)
{
string prjName = string.Empty,
buildUnit = string.Empty, bidUnit = string.Empty,
bidMoney = string.Empty, code = string.Empty,
bidDate = string.Empty,
beginDate = string.Empty,
endDate = string.Empty, bidType = string.Empty,
specType = string.Empty, InfoUrl = string.Empty,
msgType = string.Empty, bidCtx = string.Empty,
prjAddress = string.Empty, remark = string.Empty,
prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
prjName = viewList[j].GetATagValue("title");
bidType = prjName.GetInviteBidType();
beginDate = timeList[j].ToNodePlainString().GetDateRegex();
InfoUrl = "http://www.gdlonghu.gov.cn/Gb/Home/" + viewList[j].GetATagHref();
string htlDtl = string.Empty;
try
{
htlDtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default);
htlDtl = htlDtl.GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htlDtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "lblContent")));
if (dtlList != null && dtlList.Count > 0)
{
HtmlTxt = dtlList.ToHtml();
bidCtx = HtmlTxt.ToCtxString();
bidUnit = bidCtx.GetBidRegex();
bidMoney = bidCtx.GetMoneyRegex();
if (string.IsNullOrEmpty(bidUnit))
{
string ctx = string.Empty;
parser = new Parser(new Lexer(HtmlTxt));
NodeList tabList = parser.ExtractAllNodesThatMatch(new TagNameFilter("table"));
if (tabList != null && tabList.Count > 0)
{
try
{
TableTag tab = tabList[0] as TableTag;
if (tab.RowCount > 2 && tab.Rows[0].ColumnCount > 0)
{
for (int d = 0; d < tab.Rows[0].ColumnCount; d++)
{
ctx += tab.Rows[0].Columns[d].ToNodePlainString().Replace("价格得分", "得分").Replace("<?xml:namespaceprefix=ons=\"urn:schemas-microsoft-com:office:office\"/>", "") + "：";
ctx += tab.Rows[2].Columns[d].ToNodePlainString().Replace("\r", "").Replace("\n", "") + "\r\n";
}
}
}
catch { }
}
bidUnit = ctx.GetBidRegex();
bidMoney = ctx.GetMoneyRegex();
if (bidUnit.Contains("商务及技术"))
{
ctx = string.Empty;
if (tabList != null && tabList.Count > 0)
{
try
{
TableTag tab = tabList[0] as TableTag;
if (tab.RowCount > 0 && tab.Rows[0].ColumnCount > 1)
{
ctx += tab.Rows[0].Columns[0].ToNodePlainString() + "：";
ctx += tab.Rows[0].Columns[1].ToNodePlainString() + "\r\n";
}
}
catch { }
}
bidUnit = ctx.GetBidRegex();
}
}
if (string.IsNullOrEmpty(bidMoney) || bidMoney == "0")
{
bidMoney.GetMoneyRegex();
}
prjAddress = bidCtx.GetAddressRegex();
code = bidCtx.GetCodeRegex();
buildUnit = bidCtx.GetBuildRegex();
msgType = "汕头龙湖区政府";
specType = "建设工程";
BidInfo info = ToolDb.GenBidInfo("广东省", "汕头市区", "龙湖区", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType,
bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aList = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aList != null && aList.Count > 0)
{
for (int c = 0; c < aList.Count; c++)
{
ATag a = aList[c] as ATag;
if (a.Link.IsAtagAttach())
{
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText.Replace(" ", "").Replace(";", "").Replace("；", ""), info.Id, "http://www.gdlonghu.gov.cn" + a.Link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
int sqlCount = 0;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, Encoding.Default).GetJsString();
}
catch (Exception ex)
{
return(null);
}
Parser parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "font9grey1")));
if (nodeList != null && nodeList.Count > 1)
{
for (int i = 0; i < 2; i++)
{
TableTag table = nodeList[i] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string headName = string.Empty, releaseTime = string.Empty, infoScorce = string.Empty, msgType = string.Empty, infoUrl = string.Empty, ctxHtml = string.Empty, infoCtx = string.Empty, infoType = string.Empty;
TableRow tr = table.Rows[j];
infoType = "办事指南";
headName = tr.Columns[1].ToNodePlainString();
releaseTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
infoUrl = "http://www.gzzb.gd.cn" + tr.Columns[1].GetATagHref();
string htldtl = string.Empty;
try
{
htldtl = this.ToolWebSite.GetHtmlByUrl(infoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htldtl));
NodeList dtlList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "contentDiv")));
if (dtlList != null && dtlList.Count > 0)
{
ctxHtml = dtlList.AsHtml();
infoCtx = ctxHtml.ToCtxString();
msgType = MsgTypeCosnt.GuangZhouMsgType;
NotifyInfo info = ToolDb.GenNotifyInfo(headName, releaseTime, infoScorce, msgType, infoUrl, ctxHtml, "广东省", "广州市区", string.Empty, infoCtx, infoType);
sqlCount++;
if (!crawlAll && sqlCount >= this.MaxCount)
{
return(null);
}
if (ToolDb.SaveEntity(info, this.ExistCompareFields))
{
parser = new Parser(new Lexer(ctxHtml));
NodeList imgList = parser.ExtractAllNodesThatMatch(new TagNameFilter("img"));
if (imgList != null && imgList.Count > 0)
{
for (int img = 0; img < imgList.Count; img++)
{
ImageTag imgTag = imgList[img] as ImageTag;
try
{
BaseAttach obj = null;
if (imgTag.GetAttribute("src").Contains("http"))
{
obj = ToolHtml.GetBaseAttach(imgTag.GetAttribute("src"), headName, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.gzzb.gd.cn" + imgTag.GetAttribute("src"), headName, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
parser = new Parser(new Lexer(ctxHtml));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag aTag = aNode[a] as ATag;
if (aTag.IsAtagAttach())
{
try
{
BaseAttach obj = null;
if (aTag.Link.Contains("http"))
{
obj = ToolHtml.GetBaseAttach(aTag.Link, aTag.LinkText, info.Id);
}
else
{
obj = ToolHtml.GetBaseAttach("http://www.gzzb.gd.cn" + aTag.Link, aTag.LinkText, info.Id);
}
if (obj != null)
{
ToolDb.SaveEntity(obj, string.Empty);
}
}
catch { }
}
}
}
}
}
}
}
}
return(null);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <NoticeInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("nowrap", "true")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("总页数", "当前页").Replace("：", "");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl + "?Paging=" + i, Encoding.UTF8);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "99%")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 0; j < table.RowCount - 1; j++)
{
string InfoTitle = string.Empty, InfoType = string.Empty, PublistTime = string.Empty, InfoCtx = string.Empty, InfoUrl = string.Empty, prjCode = string.Empty, buildUnit = string.Empty, htmlTxt = string.Empty, area = string.Empty;
TableRow tr = table.Rows[j];
ATag aTag = tr.Columns[1].GetATag();
InfoTitle = aTag.GetAttribute("title");
PublistTime = tr.Columns[2].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.gxzbtb.cn" + aTag.Link;
InfoType = "澄清变更";
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("id", "TDContent")));
if (dtlNode != null && dtlNode.Count > 0)
{
htmlTxt = dtlNode.AsHtml().GetJsString();
InfoCtx = htmlTxt.ToCtxString();
buildUnit = InfoCtx.GetBuildRegex();
NoticeInfo info = ToolDb.GenNoticeInfo("广西壮族自治区", "广西壮族自治区及地市", area, string.Empty, InfoTitle, InfoType, InfoCtx, PublistTime, string.Empty, "广西壮族自治区公共资源交易中心", InfoUrl, prjCode, buildUnit, string.Empty, string.Empty, "政府采购", string.Empty, htmlTxt);
parser = new Parser(new Lexer(htmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k].GetATag();
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.gxzbtb.cn" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "page")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToPlainTextString().GetRegexBegEnd("共", "页");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://ggzy.zhuhai.gov.cn//zbjj/index_" + i + ".htm", Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "news")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
INode node = listNode[j];
ATag aTag = node.GetATag();
if (aTag == null)
{
continue;
}
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, HtmlTxt = string.Empty;
prjName = aTag.GetAttribute("title");
beginDate = node.ToPlainTextString().GetDateRegex();
InfoUrl = aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "m_r m_r_g")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.ToCtxString();
parser = new Parser(new Lexer(HtmlTxt.ToLower().Replace("th", "td")));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "bordertb")));
if (tableNode != null && tableNode.Count > 0)
{
string ctx = string.Empty;
TableTag table = tableNode[0] as TableTag;
for (int r = 1; r < table.RowCount; r++)
{
for (int c = 0; c < table.Rows[r].ColumnCount; c++)
{
string temp = table.Rows[r].Columns[c].ToNodePlainString();
if (c % 2 == 0)
{
ctx += temp + "：";
}
else
{
ctx += temp + "\r\n";
}
}
}
buildUnit = ctx.GetBuildRegex();
code = ctx.GetCodeRegex().GetCodeDel();
bidUnit = ctx.GetBidRegex();
bidMoney = ctx.GetMoneyRegex();
prjMgr = ctx.GetMgrRegex();
}
msgType = "珠海市公共资源交易中心";
specType = "建设工程";
bidType = prjName.GetInviteBidType();
BidInfo info = ToolDb.GenBidInfo("广东省", "珠海市区", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, string.Empty, bidMoney, InfoUrl, prjMgr, HtmlTxt);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://ggzy.zhuhai.gov.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default, ref cookiestr);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("span"), new HasAttributeFilter("id", "lblPageCount")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[0].ToNodePlainString();
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"__EVENTTARGET",
"__EVENTARGUMENT",
"__LASTFOCUS",
"__VIEWSTATE",
"__VIEWSTATEGENERATOR",
"__EVENTVALIDATION",
"gcbh_Text_Box",
"gcmc_TextBox",
"num_TextBox",
"ImageButton3.x",
"ImageButton3.y"
}, new string[] {
"", "", "",
viewState,
"B0108473",
eventValidation,
"", "", "",
"5", "12"
});
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, nvc, Encoding.Default, ref cookiestr);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "DataGrid1")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
TableRow tr = table.Rows[j];
ATag aTag = tr.Columns[1].GetATag();
if (aTag == null)
{
continue;
}
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
code = tr.Columns[0].ToNodePlainString();
prjName = aTag.LinkText.GetReplace(" ");
buildUnit = tr.Columns[3].ToNodePlainString();
endDate = tr.Columns[4].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.bcactc.com/home/gcxx/" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "hei_text")));
if (dtlNode != null && dtlNode.Count > 0)
{
TableTag dtlTable = dtlNode[0] as TableTag;
HtmlTxt = dtlTable.ToHtml();
for (int r = 0; r < dtlTable.RowCount; r++)
{
for (int c = 0; c < dtlTable.Rows[r].ColumnCount; c++)
{
if ((c + 1) % 2 == 0)
{
inviteCtx += dtlTable.Rows[r].Columns[c].ToNodePlainString() + "\r\n";
}
else
{
inviteCtx += dtlTable.Rows[r].Columns[c].ToNodePlainString().GetReplace(":,：") + "：";
}
}
}
if (code.Contains(".."))
{
code = inviteCtx.GetCodeRegex();
}
prjAddress = inviteCtx.GetAddressRegex().GetCodeDel();
beginDate = inviteCtx.GetRegex("发布日期,发布时间").GetDateRegex();
if (string.IsNullOrEmpty(beginDate))
{
beginDate = inviteCtx.GetDateRegex();
}
if (string.IsNullOrEmpty(beginDate))
{
beginDate = inviteCtx.GetDateRegex("yyyy/MM/dd");
}
if (string.IsNullOrEmpty(beginDate))
{
beginDate = inviteCtx.GetDateRegex("yyyy年MM月dd日");
}
msgType = "北京市建设工程发包承包交易中心";
specType = "建设工程";
inviteType = "施工";
InviteInfo info = ToolDb.GenInviteInfo("北京市", "北京市区", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
string html = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int pageInt = 1;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.Default);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagination page-mar")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("/共", "页");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl + "&page=" + i, Encoding.Default);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "wsbs-table")));
if (listNode != null && listNode.Count > 0)
{
TableTag table = listNode[0] as TableTag;
for (int j = 1; j < table.RowCount; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[j];
prjName = tr.Columns[1].ToNodePlainString();
if (prjName[prjName.Length - 1] == '）')
{
int staIndex = prjName.LastIndexOf("（");
int endIndex = prjName.LastIndexOf("）");
if (staIndex > 0 && endIndex > 0 && endIndex > staIndex)
{
code = prjName.Substring(staIndex + 1, endIndex - staIndex - 1);
if (!code.IsChina())
{
prjName = prjName.Remove(staIndex);
}
else
{
code = string.Empty;
}
}
}
beginDate = tr.Columns[2].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.gzggzy.cn" + tr.Columns[1].GetATagHref();
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "xx-text")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.ToHtml();
inviteCtx = HtmlTxt.Replace("</p>", "\r\n").Replace("<br />", "\r\n").Replace("<br/>", "\r\n").ToCtxString();
if (string.IsNullOrEmpty(code))
{
code = inviteCtx.GetCodeRegex();
}
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("联系"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("联系"));
}
prjAddress = inviteCtx.GetAddressRegex();
msgType = "广州公共资源交易中心";
specType = "政府采购";
inviteType = inviteCtx.GetRegex("项目类别", true, 50);
if (string.IsNullOrEmpty(inviteType))
{
inviteType = prjName.GetInviteBidType();
}
InviteInfo info = ToolDb.GenInviteInfo("广东省", "广州政府采购", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
parser = new Parser(new Lexer(HtmlTxt));
NodeList fileNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (fileNode != null && fileNode.Count > 0)
{
for (int k = 0; k < fileNode.Count; k++)
{
ATag fileAtag = fileNode[k].GetATag();
if (fileAtag.IsAtagAttach())
{
string fileName = fileAtag.LinkText.ToNodeString().Replace(" ", "");
string fileLink = fileAtag.Link;
if (!fileLink.ToLower().Contains("http"))
{
fileLink = "http://www.gzggzy.cn" + fileAtag.Link;
}
base.AttachList.Add(ToolDb.GenBaseAttach(fileName, info.Id, fileLink));
}
}
}
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
public void DealHtml(IList list, string html, bool crawlAll)
{
Parser parserDtl = new Parser(new Lexer(html));
NodeList aNodes = parserDtl.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "lefttable")));
if (aNodes != null && aNodes.Count > 0)
{
Type typs = typeof(ATag);
TableTag table = aNodes[0] as TableTag;
for (int t = 1; t < table.RowCount - 1; t++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty,
inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, ctx = string.Empty, CreateTime = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[t] as TableRow;
ATag aTag = tr.SearchFor(typeof(ATag), true)[0] as ATag;
InfoUrl = aTag.Link;
prjName = table.Rows[t].Columns[1].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
endDate = table.Rows[t].Columns[2].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
string htmlDtl = string.Empty;
try
{
htmlDtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default);
}
catch (Exception ex)
{
continue;
}
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmlDtl = regexHtml.Replace(htmlDtl, "");
Parser parserCtx = new Parser(new Lexer(htmlDtl));
NodeList ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "printTb lefttable")));
if (ctxNode != null && ctxNode.Count > 0)
{
Parser parserdiv = new Parser(new Lexer(htmlDtl));
NodeList aNodesdiv = parserdiv.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "biuuu_button")));
HtmlTxt = ctxNode.AsHtml().Replace(aNodesdiv.AsHtml(), "").Trim();
Type tp = typeof(ATag);
TableTag tabTag = ctxNode[0] as TableTag;
string startTime = tabTag.Rows[1].Columns[0].ToPlainTextString().Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
Regex regex = new Regex(@"时间：\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}");
Match math = regex.Match(startTime);
beginDate = math.Value.Replace("时间：", "");
Regex regexcode = new Regex("(工程编号|项目编号|招标编号)：[^\r\n]+[\r\n]{1}");
Match match = regexcode.Match(tabTag.ToPlainTextString());
code = match.Value.Substring(match.Value.IndexOf("：") + 1).Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
Regex regexBuildUnit = new Regex("(招标人|建设单位|招标采购单位)：[^\r\n]+[\r\n]{1}");
Match matchBuildUnit = regexBuildUnit.Match(tabTag.ToPlainTextString());
buildUnit = matchBuildUnit.Value.Substring(matchBuildUnit.Value.IndexOf("：") + 1).Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
Regex regexAddress = new Regex("(建设地点|项目地点|工程地点)：[^\r\n]+[\r\n]{1}");
Match matchAddress = regexAddress.Match(tabTag.ToPlainTextString());
prjAddress = matchAddress.Value.Substring(matchAddress.Value.IndexOf("：") + 1).Replace("\r\n", "").Replace("\t", "").Replace(" ", " ").Trim();
if (Encoding.Default.GetByteCount(code) > 50)
{
code = "";
}
if (buildUnit == "" || buildUnit == null)
{
buildUnit = "";
}
if (Encoding.Default.GetByteCount(buildUnit) > 150)
{
buildUnit = buildUnit.Substring(0, 150);
}
if (Encoding.Default.GetByteCount(prjAddress) > 200)
{
prjAddress = "见招标公告内容";
}
ctx = tabTag.Rows[2].Columns[0].ToPlainTextString().Replace(" ", " ").Replace("\r\n\r\n\r\n\r\n", "\r\n").Replace("\r\n\r\n", "\r\n");
if (ctx.Length > 0)
{
Regex regexCtx = new Regex("<!--[^<]+-->");
ctx = regexCtx.Replace(ctx, "");
}
if (beginDate.Length > 0 && endDate.Length > 0)
{
DateTime begin = new DateTime();
DateTime end = new DateTime();
try
{
begin = DateTime.Parse(beginDate);
end = DateTime.Parse(endDate);
}
catch (Exception)
{
}
if (begin > end)
{
endDate = string.Empty;
}
}
}
parserCtx.Reset();
ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("td"), new HasAttributeFilter("class", "toptd_bai")));
Regex regDate = new Regex(@"\d{4}-\d{1,2}-\d{1,2}");
beginDate = regDate.Match(ctxNode.AsString()).Value.Trim();
if (beginDate == "")
{
beginDate = string.Empty;
}
specType = "建设工程";
inviteType = ToolHtml.GetInviteTypes(prjName);
InviteInfo info = ToolDb.GenInviteInfo("广东省", "惠州市区", "大亚湾区", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, ctx, remark, "惠州市建设工程交易中心", inviteType, "建设工程", string.Empty, InfoUrl, HtmlTxt);
list.Add(info);
ctxNode = parserCtx.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("a"), new HasAttributeFilter("target", "_blank")));
NodeList aTagNodes = ctxNode.SearchFor(typeof(ATag), true);
for (int a = 0; a < aTagNodes.Count; a++)
{
ATag fileTage = aTagNodes[a] as ATag;
if (fileTage.Link.Contains("http://www.ebc.huizhou.gov.cn/index/loadNewsFile"))
{
string downloadURL = fileTage.Link;
BaseAttach attach = ToolDb.GenBaseAttach(fileTage.ToPlainTextString(), info.Id, downloadURL);
base.AttachList.Add(attach);
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return;
}
}
}
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <BidInfo>();
string html = string.Empty;
int pageInt = 1;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl, Encoding.UTF8);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("ul"), new HasAttributeFilter("class", "pagination")), true), new TagNameFilter("a")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode[pageNode.Count - 2].ToNodePlainString();
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.cfcpn.com/plist/jieguo?pageNo=" + i + "&kflag=0&keyword=&keywordType=&province=&city=&typeOne=", Encoding.UTF8);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "cfcpn_list_content text-left")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
INode node = listNode[j];
ATag aTag = node.GetATag();
beginDate = node.ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.cfcpn.com" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList telNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("p"), new HasAttributeFilter("class", "cfcpn_news_title")));
if (telNode != null && telNode.Count > 0)
{
prjName = telNode.AsHtml();
prjName = prjName.ToCtxString();
}
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "news_content")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
bidCtx = HtmlTxt.GetReplace("<br/>,</p>,<br>,<br />,</div>", "\r\n").ToCtxString().GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：").GetReplace("一包：\r\n", "一包：");
buildUnit = bidCtx.GetBuildRegex();
prjAddress = bidCtx.GetAddressRegex();
code = bidCtx.GetCodeRegex().GetCodeDel();
bidUnit = bidCtx.GetBidRegex();
if (string.IsNullOrEmpty(bidUnit))
{
bidUnit = bidCtx.GetRegex("成交候选人,第一中标候选人名称,一包").GetReplace("名称");
}
bidMoney = bidCtx.GetMoneyRegex();
prjMgr = bidCtx.GetMgrRegex();
if (bidUnit.Contains("废标") || bidCtx.Contains("废除原因") || bidCtx.Contains("废止原因") || bidCtx.Contains("废标"))
{
bidUnit = "废标";
prjMgr = string.Empty;
bidMoney = "0";
}
if (string.IsNullOrEmpty(bidUnit))
{
parser = new Parser(new Lexer(HtmlTxt));
NodeList tableNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("border", "1")));
if (tableNode != null && tableNode.Count > 0)
{
TableTag dtlTable = tableNode[0] as TableTag;
string ctx = string.Empty;
if (dtlTable.RowCount > 1)
{
try
{
for (int r = 0; r < dtlTable.Rows[0].ColumnCount; r++)
{
ctx += dtlTable.Rows[0].Columns[r].ToNodePlainString() + "：";
ctx += dtlTable.Rows[1].Columns[r].ToNodePlainString() + "\r\n";
}
}
catch { }
bidUnit = ctx.GetBidRegex();
if (bidMoney == "0")
{
bidMoney = ctx.GetMoneyRegex();
}
if (string.IsNullOrEmpty(prjMgr))
{
prjMgr = ctx.GetMgrRegex();
}
}
}
}
bidUnit = bidUnit.GetReplace("名称, ", "");
buildUnit = buildUnit.GetReplace(" ");
prjAddress = prjAddress.GetReplace(" ");
prjName = prjName.GetReplace(" ");
code = code.GetReplace(" ");
prjMgr = prjMgr.GetReplace(" ");
specType = "政府采购";
bidType = prjName.GetInviteBidType();
msgType = "中国金融集中采购网";
BidInfo info = ToolDb.GenBidInfo("全国", "金融专项采购", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int a = 0; a < aNode.Count; a++)
{
ATag file = aNode[a].GetATag();
if (file.IsAtagAttach())
{
string link = file.Link;
if (!link.ToLower().Contains("http"))
{
link = "http://www.cfcpn.com/" + file.Link;
}
base.AttachList.Add(ToolDb.GenBaseAttach(file.LinkText, info.Id, link));
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
string htl = string.Empty;
string cookiestr = string.Empty;
string viewState = string.Empty;
int page = 3;
string eventValidation = string.Empty;
try
{
htl = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl), Encoding.UTF8, ref cookiestr);
}
catch (Exception ex)
{
return list;
}
for (int i = 1; i <= page; i++)
{
if (i > 1)
{
try
{
htl = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl + "&page.number=" + i.ToString()), Encoding.UTF8);
}
catch (Exception ex) { continue; }
}
Parser parser = new Parser(new Lexer(htl));
parser = new Parser(new Lexer(htl));
NodeList tableNodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("class", "table_title3")));
if (tableNodeList.Count > 0)
{
TableTag table = (TableTag)tableNodeList[0];
for (int j = 1; j < table.RowCount; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[j];
prjName = tr.Columns[4].ToPlainTextString().Trim();
ATag aTag = tr.Columns[4].SearchFor(typeof(ATag), true)[0] as ATag;
beginDate = tr.Columns[6].ToPlainTextString().Trim();
endDate = tr.Columns[8].ToPlainTextString().Trim();
InfoUrl = "http://bidding.cnpec.com.cn/member/" + aTag.Link.Replace("amp;", "").Trim();
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "");
}
catch (Exception)
{
continue;
}
Parser parserdetail = new Parser(new Lexer(htmldetail));
NodeList dtnode = parserdetail.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("width", "100%")));
if (dtnode.Count > 0)
{
TableTag tableNode = (TableTag)dtnode[0];
HtmlTxt = dtnode.AsHtml();
for (int k = 1; k < tableNode.RowCount; k++)
{
TableRow trow = tableNode.Rows[k];
for (int c = 0; c < trow.ColumnCount; c++)
{
string tr1 = string.Empty;
tr1 = trow.Columns[c].ToPlainTextString().Trim();
inviteCtx += "\r\n" + tr1;
}
}
inviteCtx += "\r\n";
inviteCtx = inviteCtx.Replace("&", "").Replace("nbsp;", "").Replace("”", "").Replace("“", "").Trim();
Regex regCode = new Regex(@"招标编号(：|:)[^\r\n]+\r\n");
code = regCode.Match(inviteCtx).Value.Replace("招标编号：", "").Trim();
Regex regBuidUnit = new Regex(@"(招标人|建设单位)(：|:)[^\r\n]+\r\n");
buildUnit = regBuidUnit.Match(inviteCtx).Value.Replace("招标人：", "").Replace("建设单位:", "").Trim();
msgType = "中广核工程有限公司";
specType = "建设工程";
if (buildUnit == "")
{
buildUnit = "";
}
inviteType = ToolHtml.GetInviteTypes(prjName);
prjAddress = "见招标信息";
remark = " 国际招标";
InviteInfo info = ToolDb.GenInviteInfo("广东省", "深圳社会招标", "",
string.Empty, code, prjName, prjAddress, buildUnit,
beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount) return list;
}
}
}
}
return list;
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(SiteUrl);
}
catch
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList sNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "paging")));
if (sNode != null && sNode.Count > 0)
{
try
{
string temp = sNode.AsString().GetReplace(" ").GetRegexBegEnd(",共", "页");
pageInt = int.Parse(temp);
}
catch { pageInt = 1; }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://wanjiang.dg.gov.cn/zbgs-" + i + ".html", Encoding.UTF8);
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList viewList = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "Party_news")), true), new TagNameFilter("p")));
if (viewList != null && viewList.Count > 0)
{
for (int j = 0; j < viewList.Count; j++)
{
ATag aTag = viewList[j].GetATag();
string beginDate = viewList[j].ToPlainTextString().GetDateRegex();
string prjName = aTag.LinkText.GetReplace("[" + beginDate + "]");
string InfoUrl = "http://wanjiang.dg.gov.cn/" + aTag.Link.GetReplace("./");
string htmDtl = string.Empty;
try
{
htmDtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8);
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>|<style[^<]*</style>|<xml[^<]*</xml>");
htmDtl = regexHtml.Replace(htmDtl, "");
}
catch { continue; }
parser = new Parser(new Lexer(htmDtl));
NodeList dtl = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "about-nn")));
if (dtl != null && dtl.Count > 0)
{
string code = string.Empty, buildUnit = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
HtmlTxt = dtl.AsHtml().ToLower();
inviteCtx = HtmlTxt.ToLower().GetReplace("</p>,</br>,<br>", "\r\n").ToCtxString();
buildUnit = inviteCtx.GetBuildRegex();
prjAddress = inviteCtx.GetAddressRegex();
code = inviteCtx.GetCodeRegex().GetCodeDel();
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址"));
}
if (buildUnit.Contains("招标代理"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("招标代理"));
}
specType = "政府采购";
inviteType = prjName.GetInviteBidType();
msgType = "东莞市万江区办事处办公室";
InviteInfo info = ToolDb.GenInviteInfo("广东省", "东莞市区", "万江区", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k].GetATag();
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://wanjiang.dg.gov.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
if (list.Count % 20 == 0)
{
Thread.Sleep(1000 * 500);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new List <InviteInfo>();
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.SiteUrl);
}
catch { return(list); }
Parser parser = new Parser(new Lexer(html));
NodeList pageNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("style", "padding-top:15px;")));
if (pageNode != null && pageNode.Count > 0)
{
try
{
string temp = pageNode.AsString().GetRegexBegEnd("共", "页").Replace("：", "");
pageInt = int.Parse(temp);
}
catch { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.huajiezn.cn/html/zbgg/index_" + i + ".html");
}
catch { continue; }
}
parser = new Parser(new Lexer(html));
NodeList listNode = parser.ExtractAllNodesThatMatch(new AndFilter(new HasParentFilter(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "mainnavb")), true), new TagNameFilter("li")));
if (listNode != null && listNode.Count > 0)
{
for (int j = 0; j < listNode.Count; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty,
prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty,
specType = string.Empty, beginDate = string.Empty, endDate = string.Empty,
remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty,
CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty, area = string.Empty;
ATag aTag = listNode[j].GetATag();
prjName = aTag.LinkText.ToNodeString();
beginDate = listNode[j].ToPlainTextString().GetDateRegex();
InfoUrl = "http://www.huajiezn.cn" + aTag.Link;
string htmldtl = string.Empty;
try
{
htmldtl = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).GetJsString();
}
catch { continue; }
parser = new Parser(new Lexer(htmldtl));
NodeList dtlNode = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "Zoom")));
if (dtlNode != null && dtlNode.Count > 0)
{
HtmlTxt = dtlNode.AsHtml();
parser.Reset();
NodeList h3Node = parser.ExtractAllNodesThatMatch(new TagNameFilter("h3"));
if (h3Node != null && h3Node.Count > 0)
{
prjName = h3Node[0].ToNodePlainString();
}
inviteCtx = HtmlTxt.ToCtxString();
prjAddress = inviteCtx.GetAddressRegex();
buildUnit = inviteCtx.GetBuildRegex();
if (buildUnit.Contains("公司"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("公司")) + "公司";
}
if (buildUnit.Contains("地址"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("地址"));
}
if (buildUnit.Contains("招标代理"))
{
buildUnit = buildUnit.Remove(buildUnit.IndexOf("招标代理"));
}
code = inviteCtx.GetCodeRegex().GetCodeDel();
msgType = "华杰工程咨询有限公司中南分公司";
specType = "建设工程";
inviteType = prjName.GetInviteBidType();
buildUnit = buildUnit.Replace(" ", "");
InviteInfo info = ToolDb.GenInviteInfo("湖北省", "湖北省及地市", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
parser = new Parser(new Lexer(HtmlTxt));
NodeList aNode = parser.ExtractAllNodesThatMatch(new TagNameFilter("a"));
if (aNode != null && aNode.Count > 0)
{
for (int k = 0; k < aNode.Count; k++)
{
ATag a = aNode[k] as ATag;
if (a.IsAtagAttach())
{
string link = string.Empty;
if (a.Link.ToLower().Contains("http"))
{
link = a.Link;
}
else
{
link = "http://www.huajiezn.cn/" + a.Link;
}
BaseAttach attach = ToolDb.GenBaseAttach(a.LinkText, info.Id, link);
base.AttachList.Add(attach);
}
}
}
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
return(list);
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
//取得页码
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode(SiteUrl), Encoding.Default);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("id", "anp1")));
if (tdNodes != null)
{
string pageTemp = tdNodes.AsString().Replace(" ", "").Trim();
Regex regpage = new Regex(@"当前第[^页]+页");
try
{
pageInt = int.Parse(regpage.Match(pageTemp).Value.Split('/')[1].Replace("页", "").Trim());
}
catch (Exception ex) { }
string cookiestr = string.Empty;
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
viewState = this.ToolWebSite.GetAspNetViewState(html);
eventValidation = this.ToolWebSite.GetAspNetEventValidation(html);
NameValueCollection nvc = this.ToolWebSite.GetNameValueCollection(new string[] {
"anp1_input",
"Columnlist2_DepartmentTreeView_CheckedList",
"Columnlist2_DepartmentTreeView_EditEvents",
"Columnlist2_DepartmentTreeView_ExpandedList",
"Columnlist2_DepartmentTreeView_MoveEvents",
"Columnlist2_DepartmentTreeView_MultipleSelectedList",
"Columnlist2_DepartmentTreeView_ScrollData",
"Columnlist2_DepartmentTreeView_SelectedNode",
"Columnlist2_DepartmentTreeView_ValueChangeEvents",
"Login2:txtPassword",
"Login2:txtUserName",
"__EVENTARGUMENT",
"__EVENTTARGET",
"__VIEWSTATE"
}, new string[] {
(i - 1).ToString(),
string.Empty,
string.Empty,
string.Empty,
string.Empty,
string.Empty,
"0,0",
"p_379",
string.Empty,
string.Empty,
string.Empty,
i.ToString(),
"anp1",
viewState
});
try { html = this.ToolWebSite.GetHtmlByUrl(SiteUrl, nvc, Encoding.Default, ref cookiestr); }
catch (Exception ex) { continue; }
}
Regex regHTML1 = new Regex(@"<td>[^<]+<td>");
Regex regHTML2 = new Regex(@"</td>[^<]+</td>");
html = regHTML2.Replace(regHTML1.Replace(html, "<td>"), "</td>");
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("table"), new HasAttributeFilter("id", "dlstNews")));
if (nodeList != null && nodeList.Count > 0)
{
if (nodeList != null && nodeList.Count > 0)
{
TableTag table = nodeList[0] as TableTag;
for (int j = 0; j < table.RowCount; j++)
{
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
TableRow tr = table.Rows[j];
beginDate = tr.Columns[1].ToPlainTextString().Trim();
ATag aTag = tr.Columns[0].SearchFor(typeof(ATag), true)[0] as ATag;
InfoUrl = "http://www.szgxzb.com/zbgg/" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).Replace(" ", "").Trim();
Parser dtlparserHTML = new Parser(new Lexer(htmldetail));
NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("id", "newsContent"), new TagNameFilter("div")));
HtmlTxt = dtnodeHTML.AsHtml();
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.Default).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n");
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmldetail = regexHtml.Replace(htmldetail, "");
}
catch (Exception ex) { continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList prjNameNode = dtlparser.ExtractAllNodesThatMatch(new TagNameFilter("title"));
string prj = prjNameNode.AsString().Replace("国信招标--", "");
dtlparser.Reset();
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("id", "newsContent"), new TagNameFilter("div")));
bidCtx = dtnode.AsString();
Regex regcode = new Regex(@"（招标编号：[^）]+）");
code = regcode.Match(bidCtx).Value.Replace("招标编号", "").Replace("（", "").Replace("）", "").Replace("：", "").Trim();
if (Encoding.Default.GetByteCount(code) > 50)
{
code = "";
}
Regex regBidUnit = new Regex(@"单位：[^\r\n]+\r\n");
bidUnit = regBidUnit.Match(bidCtx).Value.Replace("单位", "").Replace("：", "").Trim();
if (Encoding.Default.GetByteCount(bidUnit) > 150)
{
bidUnit = "";
}
Regex regBidMoneystr = new Regex(@"金额：[^\r\n]+\r\n");
string monerystr = regBidMoneystr.Match(bidCtx).Value.Replace("金额", "").Replace("：", "").Replace(",", "").Replace("，", "").Trim();
Regex regBidMoney = new Regex(@"[0-9]+[.]{0,1}[0-9]+");
Regex regName = new Regex(@"名称：[^\r\n]+\r\n");
prjName = regName.Match(bidCtx).Value.Replace("名称", "").Replace("：", "").Replace(":", "").Replace("\r\n", "");
if (string.IsNullOrEmpty(prjName))
{
prjName = prj;
}
if (!string.IsNullOrEmpty(regBidMoney.Match(monerystr).Value))
{
if ((monerystr.Contains("万元") || monerystr.Contains("万美元")) && !monerystr.Contains("万元整"))
{
bidMoney = regBidMoney.Match(monerystr).Value;
}
else
{
try
{
bidMoney = (decimal.Parse(regBidMoney.Match(monerystr).Value) / 10000).ToString();
if (decimal.Parse(bidMoney) < decimal.Parse("0.1"))
{
bidMoney = "0";
}
}
catch (Exception)
{
bidMoney = "0";
}
}
}
specType = "其他";
msgType = "深圳市国信招标有限公司";
prjName = ToolDb.GetPrjName(prjName);
bidType = ToolHtml.GetInviteTypes(prjName);
BidInfo info = ToolDb.GenBidInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
}
}
return(list);
}
