private static void InitalizeDatabaseConfig(CompositeTag dbConfig)
{
foreach (CompositeTag tableConfig in dbConfig.SubTags)
{
InitalizeTableConfig(tableConfig, dbConfig.TryGetAttributeValueAt("id"));
}
}
private void ReadCompositeTag(CompositeTag htmlNodes, int level = 0)
{
string padding = "";
for (int i = 0; i < level; i++)
{
padding += "--";
}
for (int i = 0; i < htmlNodes.ChildCount; i++)
{
var node = htmlNodes.GetChild(i);
Console.WriteLine(padding + " " + i + " " + node.GetType());
var nodeList = node as NodeList;
if (nodeList != null)
{
ReadNodeList(nodeList, level + 1);
}
var composite = node as CompositeTag;
if (composite != null)
{
ReadCompositeTag(composite, level + 1);
}
}
}
private static string GetTagValue(CompositeTag Tags, string key)
{
String tempValue = null;
TagElement tempTag = null;
tempTag = Tags.FindTag(key);
if (tempTag == null)
{
return(null);
}
else
{
return(tempTag.TryGetAttributeValueAt("value"));
}
}
private static void InitalizeTableConfig(CompositeTag tableConfig, string dsName)
{
List <string> pKeys = new List <string>();
foreach (TagElement columnConfig in tableConfig.SubTags)
{
string isKey = columnConfig.TryGetAttributeValueAt("isKey");
isKey = isKey.ToLower();
if (isKey != null && isKey.Equals("yes"))
{
pKeys.Add(columnConfig.TryGetAttributeValueAt("id"));
}
}
TablesWithSchema.SetPrimaryKey(dsName, tableConfig.TryGetAttributeValueAt("id"), pKeys.ToArray());
}
private void ValidateStyle(CompositeTag ct)
{
if (ct.Container == null)
{
ct.Folder.Foldout.Style = GetStyle(ct.Behavior, null);
}
else if (IsInExecutionSequence(ct.Container))
{
ct.Folder.Foldout.Style = GetStyle(ct.Behavior, ct.Container);
}
else
{
ct.Folder.Foldout.Style = null;
}
}
public static void GetDataSource(CompositeTag dataSourceTag)
{
foreach (CompositeTag childConfig in dataSourceTag.SubTags)
{
DataSourceConfig dsConfig = new DataSourceConfig();
String id = childConfig.TryGetAttributeValueAt("id");
dsConfig.dataSourceName = id;
dsConfig.url = GetTagValue(childConfig, "connectionString");
dsConfig.dialectClass = GetTagValue(childConfig, "dialectClass");
dsConfig.providerName = GetTagValue(childConfig, "providerName");
String tempValue = null;
tempValue = GetTagValue(childConfig, "minSize");
if (tempValue != null)
{
dsConfig.minSize = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "maxSize");
if (tempValue != null)
{
dsConfig.maxSize = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "timeOut");
if (tempValue != null)
{
dsConfig.timeOut = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "usePool");
if (tempValue != null)
{
dsConfig.pooled = Convert.ToBoolean(tempValue);
}
if (DataSourceList.ContainsKey(id))
{
DataSourceList[id] = dsConfig;
}
else
{
DataSourceList.Add(id, dsConfig);
}
}
}
public static void GetDataSource(CompositeTag dataSourceTag)
{
foreach (CompositeTag childConfig in dataSourceTag.SubTags)
{
DataSourceConfig dsConfig = new DataSourceConfig();
String id = childConfig.TryGetAttributeValueAt("id");
dsConfig.dataSourceName = id;
dsConfig.url = GetTagValue(childConfig, "connectionString");
dsConfig.dialectClass = GetTagValue(childConfig, "dialectClass");
dsConfig.providerName = GetTagValue(childConfig, "providerName");
String tempValue = null;
tempValue = GetTagValue(childConfig, "minSize");
if (tempValue != null)
{
dsConfig.minSize = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "maxSize");
if (tempValue != null)
{
dsConfig.maxSize = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "timeOut");
if (tempValue != null)
{
dsConfig.timeOut = Convert.ToInt32(tempValue);
}
tempValue = GetTagValue(childConfig, "usePool");
if (tempValue != null)
{
dsConfig.pooled = Convert.ToBoolean(tempValue);
}
if (DataSourceList.ContainsKey(id))
{
DataSourceList[id] = dsConfig;
}
else
{
DataSourceList.Add(id, dsConfig);
}
}
}
public static void InitialDatabaseConfigs(string dbConfigFile)
{
XmlDocument xmlDoc = new XmlDocument();
xmlDoc.Load("~/DBHelper.config");
IEbfConfig dbConfigs = ConfigFactory.GetConfigOf(dbConfigFile);
TagElement rootTag = dbConfigs.RootTag;
CompositeTag dataSourceTag = (CompositeTag)dbConfigs.FindTag("dblinks");
GetDataSource(dataSourceTag);
BrokerFactory.InitDataSource(DataSourceList);
if (rootTag is CompositeTag)
{
foreach (CompositeTag dbConfig in ((CompositeTag)rootTag).SubTags)
{
if (!dbConfig.Name.Equals("dslist"))
{
InitalizeDatabaseConfig(dbConfig);
}
}
}
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
//取得页码
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl("http://www.szsszx.com/tender/pager?key=caigou&pagenumber=20&pageindex=1", Encoding.UTF8);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagelist")));
if (tdNodes != null && tdNodes.Count > 0)
{
string pageTemp = tdNodes.AsString().Replace(" ", "");
Regex regpage = new Regex(@"1/\d+");
try
{
pageInt = int.Parse(regpage.Match(pageTemp).Value.Replace("1/", ""));
}
catch (Exception ex) { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=caigou&pagenumber=20&pageindex=" + i.ToString()), Encoding.UTF8);
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new TagNameFilter("li"));
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
CompositeTag obj = nodeList[j] as CompositeTag;
ATag aTag = obj.SearchFor(typeof(ATag), true)[0] as ATag;
Span dateSpan = obj.SearchFor(typeof(Span), true)[0] as Span;
prjName = aTag.GetAttribute("title");
beginDate = dateSpan.ToPlainTextString().Trim(new char[] { '[', ']' });
InfoUrl = "http://www.szsszx.com" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Trim();
Parser dtlparserHTML = new Parser(new Lexer(htmldetail));
NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div")));
HtmlTxt = dtnodeHTML.AsHtml();
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n");
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmldetail = regexHtml.Replace(htmldetail, "");
}
catch (Exception ex) { continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div")));
inviteCtx = dtnode.AsString().Replace(" ", "").Replace("[ifgtemso11]", "").Replace("[endif]", "").Replace("<!", "");
inviteCtx = Regex.Replace(inviteCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace("<", "").Replace(">", "").Replace("\n\n\n\t", "\r\n").Replace("\n\n", "\r\n");
code = inviteCtx.GetRegex("招标编号,编号", true, 50).Replace("开标时间安排如下", "");
Regex regbuildUnit = new Regex(@"(采购人|采购单位)(：|:)[^\r\n]+\r\n");
buildUnit = regbuildUnit.Match(inviteCtx).Value.Replace("采购人", "").Replace("采购单位", "").Replace("：", "").Replace(":", "").Trim();
if (string.IsNullOrEmpty(beginDate))
{
beginDate = DateTime.Now.ToString("yyyy-MM-dd");
}
Regex regprjAddress = new Regex(@"地点(：|:)[^\r\n]+\r\n");
prjAddress = regprjAddress.Match(inviteCtx).Value.Replace("地点", "").Replace("：", "").Replace(":", "").Trim();
specType = "其他";
msgType = "深圳市深水水务咨询有限公司";
inviteType = ToolHtml.GetInviteTypes(prjName);
InviteInfo info = ToolDb.GenInviteInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
return(list);
}
private static string GetTagValue(CompositeTag Tags, string key)
{
String tempValue = null;
TagElement tempTag = null;
tempTag = Tags.FindTag(key);
if (tempTag == null)
{
return null;
}
else
{
return tempTag.TryGetAttributeValueAt("value");
}
}
private static void InitalizeDatabaseConfig(CompositeTag dbConfig)
{
foreach (CompositeTag tableConfig in dbConfig.SubTags)
{
InitalizeTableConfig(tableConfig, dbConfig.TryGetAttributeValueAt("id"));
}
}
private static void InitalizeTableConfig(CompositeTag tableConfig, string dsName)
{
List<string> pKeys = new List<string>();
foreach (TagElement columnConfig in tableConfig.SubTags)
{
string isKey = columnConfig.TryGetAttributeValueAt("isKey");
isKey = isKey.ToLower();
if (isKey != null && isKey.Equals("yes"))
{
pKeys.Add(columnConfig.TryGetAttributeValueAt("id"));
}
}
TablesWithSchema.SetPrimaryKey(dsName, tableConfig.TryGetAttributeValueAt("id"), pKeys.ToArray());
}
protected override IList ExecuteCrawl(bool crawlAll)
{
IList list = new ArrayList();
//取得页码
int pageInt = 1;
string html = string.Empty;
string viewState = string.Empty;
string eventValidation = string.Empty;
string cookiestr = string.Empty;
try
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=zhongbiao&pagenumber=20&pageindex=1"), Encoding.UTF8);
}
catch (Exception ex)
{
return(list);
}
Parser parser = new Parser(new Lexer(html));
NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagelist")));
if (tdNodes != null && tdNodes.Count > 0)
{
string pageTemp = tdNodes.AsString().Replace(" ", "");
Regex regpage = new Regex(@"1/\d+");
try
{
pageInt = int.Parse(regpage.Match(pageTemp).Value.Replace("1/", ""));
}
catch (Exception ex) { }
}
for (int i = 1; i <= pageInt; i++)
{
if (i > 1)
{
html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=zhongbiao&pagenumber=20&pageindex=" + i.ToString()), Encoding.UTF8);
}
parser = new Parser(new Lexer(html));
NodeList nodeList = parser.ExtractAllNodesThatMatch(new TagNameFilter("li"));
if (nodeList != null && nodeList.Count > 0)
{
for (int j = 0; j < nodeList.Count; j++)
{
string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty;
CompositeTag obj = nodeList[j] as CompositeTag;
ATag aTag = obj.SearchFor(typeof(ATag), true)[0] as ATag;
Span dateSpan = obj.SearchFor(typeof(Span), true)[0] as Span;
prjName = aTag.GetAttribute("title");
beginDate = dateSpan.ToPlainTextString().Trim(new char[] { '[', ']' });
InfoUrl = "http://www.szsszx.com" + aTag.Link;
string htmldetail = string.Empty;
try
{
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Trim();
Parser dtlparserHTML = new Parser(new Lexer(htmldetail));
NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div")));
HtmlTxt = dtnodeHTML.AsHtml();
htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n");
Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>");
htmldetail = regexHtml.Replace(htmldetail, "");
}
catch (Exception ex) { continue; }
Parser dtlparser = new Parser(new Lexer(htmldetail));
NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div")));
bidCtx = dtnode.ToHtml().ToCtxString();
bidCtx = Regex.Replace(bidCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace("<", "").Replace(">", "").Replace("\n\n\n\t", "\r\n").Replace("\n\n", "\r\n");
Regex regCode = new Regex(@"招标编号(：|:)[^\r\n]+\r\n");
code = regCode.Match(bidCtx).Value.Replace("招标编号", "").Replace("：", "").Replace(":", "").Trim();
if (Encoding.Default.GetByteCount(code) > 50)
{
code = "";
}
Regex regbuildUnit = new Regex(@"(采购人|采购单位|采购代理机构)(：|:)[^\r\n]+\r\n");
buildUnit = regbuildUnit.Match(bidCtx).Value.Replace("采购人", "").Replace("采购单位", "").Replace("采购代理机构", "").Replace("：", "").Replace(":", "").Trim();
prjAddress = bidCtx.GetAddressRegex();
bidUnit = bidCtx.GetBidRegex();// regBidUnit.Match(bidCtx).Value.Replace("中标单位", "").Replace("：", "").Replace(":", "").Trim();
Regex regBidMoneystr = new Regex(@"(中标价|价格|金额)(：|:)[^\r\n]+\r\n");
string monerystr = regBidMoneystr.Match(bidCtx).Value.Replace("中标价", "").Replace("价格", "").Replace("金额", "").Replace("万元整", "").Replace("：", "").Replace(":", "").Replace(",", "").Replace("，", "").Trim();
Regex regBidMoney = new Regex(@"[0-9]+[.]{0,1}[0-9]+");
if (!string.IsNullOrEmpty(regBidMoney.Match(monerystr).Value))
{
if ((monerystr.Contains("万元") || monerystr.Contains("万美元")) && !monerystr.Contains("万元整"))
{
bidMoney = regBidMoney.Match(monerystr).Value;
}
else
{
try
{
bidMoney = (decimal.Parse(regBidMoney.Match(monerystr).Value) / 10000).ToString();
if (decimal.Parse(bidMoney) < decimal.Parse("0.1"))
{
bidMoney = "0";
}
}
catch (Exception)
{
bidMoney = "0";
}
}
}
specType = "其他";
msgType = "深圳市深水水务咨询有限公司";
prjName = ToolDb.GetPrjName(prjName);
bidType = ToolHtml.GetInviteTypes(prjName);
BidInfo info = ToolDb.GenBidInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt);
list.Add(info);
if (!crawlAll && list.Count >= this.MaxCount)
{
return(list);
}
}
}
}
return(list);
}
public void setParent(CompositeTag tag)
{
delegateNode.setParent(tag);
}
public static Span GetSpan(this CompositeTag value, int i = 0)
{
return(ToolHtml.GetHtmlSpan(value.ToHtml(), null, i));
}
public void setParent(CompositeTag tag)
{
}
public static TableTag GetTableTag(this CompositeTag value, int i = 0)
{
return(ToolHtml.GetHtmlTableTag(value.ToHtml(), null, i));
}
/// <summary>
/// 获取Html里A标签href
/// </summary>
/// <param name="value"></param>
/// <param name="i"></param>
/// <returns></returns>
public static string GetATagHref(this CompositeTag value, int i = 0)
{
return(ToolHtml.GetHtmlAtag(value.ToHtml(), null, i).Link);
}
private BaseControl CreateNode(Skill.Framework.AI.Behavior behavior, Skill.Framework.AI.BehaviorContainer container)
{
BaseControl control = null;
switch (behavior.Type)
{
case Skill.Framework.AI.BehaviorType.Action:
case Skill.Framework.AI.BehaviorType.Condition:
case Skill.Framework.AI.BehaviorType.ChangeState:
Label lbl = new Label();
lbl.Style = _FailurStyle;
lbl.Content.text = behavior.Name;
BehaviorTag bt = new BehaviorTag(behavior, container, lbl);
_Behaviors.Add(bt);
control = lbl;
if (behavior.Type == Framework.AI.BehaviorType.Action)
{
lbl.Content.image = Resources.UITextures.BTree.Action;
}
else if (behavior.Type == Framework.AI.BehaviorType.ChangeState)
{
lbl.Content.image = Resources.UITextures.BTree.ChangeState;
}
else if (behavior.Type == Framework.AI.BehaviorType.Condition)
{
lbl.Content.image = Resources.UITextures.BTree.Condition;
}
break;
case Skill.Framework.AI.BehaviorType.Decorator:
case Skill.Framework.AI.BehaviorType.Composite:
Skill.Editor.UI.FolderView fw = new UI.FolderView();
fw.Foldout.Style = _FailurStyle;
fw.Foldout.Content.text = behavior.Name;
fw.Foldout.IsOpen = true;
CompositeTag ct = new CompositeTag(behavior, container, fw);
_Composites.Add(ct);
control = fw;
if (behavior.Type == Framework.AI.BehaviorType.Composite)
{
Skill.Framework.AI.Composite composite = (Skill.Framework.AI.Composite)behavior;
switch (composite.CompositeType)
{
case Skill.Framework.AI.CompositeType.Sequence:
fw.Foldout.Content.image = Resources.UITextures.BTree.Sequence;
break;
case Skill.Framework.AI.CompositeType.Concurrent:
fw.Foldout.Content.image = Resources.UITextures.BTree.Concurrent;
break;
case Skill.Framework.AI.CompositeType.Random:
fw.Foldout.Content.image = Resources.UITextures.BTree.Random;
break;
case Skill.Framework.AI.CompositeType.Priority:
case Skill.Framework.AI.CompositeType.State:
fw.Foldout.Content.image = Resources.UITextures.BTree.Priority;
break;
case Skill.Framework.AI.CompositeType.Loop:
fw.Foldout.Content.image = Resources.UITextures.BTree.Loop;
break;
}
}
else if (behavior.Type == Framework.AI.BehaviorType.Decorator)
{
Skill.Framework.AI.Decorator decorator = (Skill.Framework.AI.Decorator)behavior;
switch (decorator.DecoratorType)
{
case Skill.Framework.AI.DecoratorType.Default:
fw.Foldout.Content.image = Resources.UITextures.BTree.Decorator;
break;
case Skill.Framework.AI.DecoratorType.AccessLimit:
fw.Foldout.Content.image = Resources.UITextures.BTree.AccessLimitDecorator;
break;
}
}
break;
}
return(control);
}
/// <summary>
/// 获取ATag属性值
/// </summary>
/// <param name="value"></param>
/// <param name="strName">ATag属性</param>
/// <param name="i"></param>
/// <returns></returns>
public static string GetATagValue(this CompositeTag value, string strName = "href", int i = 0)
{
return(ToolHtml.GetHtmlAtagValue(strName, value.ToHtml(), null, i));
}
/// <summary>
/// CompositeTag转换字符串
/// </summary>
/// <param name="value"></param>
/// <returns></returns>
public static string ToNodePlainString(this CompositeTag value)
{
return(value.ToPlainTextString().ToNodeString().Replace(" ", ""));
}