private static void InitalizeDatabaseConfig(CompositeTag dbConfig) { foreach (CompositeTag tableConfig in dbConfig.SubTags) { InitalizeTableConfig(tableConfig, dbConfig.TryGetAttributeValueAt("id")); } }
private void ReadCompositeTag(CompositeTag htmlNodes, int level = 0) { string padding = ""; for (int i = 0; i < level; i++) { padding += "--"; } for (int i = 0; i < htmlNodes.ChildCount; i++) { var node = htmlNodes.GetChild(i); Console.WriteLine(padding + " " + i + " " + node.GetType()); var nodeList = node as NodeList; if (nodeList != null) { ReadNodeList(nodeList, level + 1); } var composite = node as CompositeTag; if (composite != null) { ReadCompositeTag(composite, level + 1); } } }
private static string GetTagValue(CompositeTag Tags, string key) { String tempValue = null; TagElement tempTag = null; tempTag = Tags.FindTag(key); if (tempTag == null) { return(null); } else { return(tempTag.TryGetAttributeValueAt("value")); } }
private static void InitalizeTableConfig(CompositeTag tableConfig, string dsName) { List <string> pKeys = new List <string>(); foreach (TagElement columnConfig in tableConfig.SubTags) { string isKey = columnConfig.TryGetAttributeValueAt("isKey"); isKey = isKey.ToLower(); if (isKey != null && isKey.Equals("yes")) { pKeys.Add(columnConfig.TryGetAttributeValueAt("id")); } } TablesWithSchema.SetPrimaryKey(dsName, tableConfig.TryGetAttributeValueAt("id"), pKeys.ToArray()); }
private void ValidateStyle(CompositeTag ct) { if (ct.Container == null) { ct.Folder.Foldout.Style = GetStyle(ct.Behavior, null); } else if (IsInExecutionSequence(ct.Container)) { ct.Folder.Foldout.Style = GetStyle(ct.Behavior, ct.Container); } else { ct.Folder.Foldout.Style = null; } }
public static void GetDataSource(CompositeTag dataSourceTag) { foreach (CompositeTag childConfig in dataSourceTag.SubTags) { DataSourceConfig dsConfig = new DataSourceConfig(); String id = childConfig.TryGetAttributeValueAt("id"); dsConfig.dataSourceName = id; dsConfig.url = GetTagValue(childConfig, "connectionString"); dsConfig.dialectClass = GetTagValue(childConfig, "dialectClass"); dsConfig.providerName = GetTagValue(childConfig, "providerName"); String tempValue = null; tempValue = GetTagValue(childConfig, "minSize"); if (tempValue != null) { dsConfig.minSize = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "maxSize"); if (tempValue != null) { dsConfig.maxSize = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "timeOut"); if (tempValue != null) { dsConfig.timeOut = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "usePool"); if (tempValue != null) { dsConfig.pooled = Convert.ToBoolean(tempValue); } if (DataSourceList.ContainsKey(id)) { DataSourceList[id] = dsConfig; } else { DataSourceList.Add(id, dsConfig); } } }
public static void GetDataSource(CompositeTag dataSourceTag) { foreach (CompositeTag childConfig in dataSourceTag.SubTags) { DataSourceConfig dsConfig = new DataSourceConfig(); String id = childConfig.TryGetAttributeValueAt("id"); dsConfig.dataSourceName = id; dsConfig.url = GetTagValue(childConfig, "connectionString"); dsConfig.dialectClass = GetTagValue(childConfig, "dialectClass"); dsConfig.providerName = GetTagValue(childConfig, "providerName"); String tempValue = null; tempValue = GetTagValue(childConfig, "minSize"); if (tempValue != null) { dsConfig.minSize = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "maxSize"); if (tempValue != null) { dsConfig.maxSize = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "timeOut"); if (tempValue != null) { dsConfig.timeOut = Convert.ToInt32(tempValue); } tempValue = GetTagValue(childConfig, "usePool"); if (tempValue != null) { dsConfig.pooled = Convert.ToBoolean(tempValue); } if (DataSourceList.ContainsKey(id)) { DataSourceList[id] = dsConfig; } else { DataSourceList.Add(id, dsConfig); } } }
public static void InitialDatabaseConfigs(string dbConfigFile) { XmlDocument xmlDoc = new XmlDocument(); xmlDoc.Load("~/DBHelper.config"); IEbfConfig dbConfigs = ConfigFactory.GetConfigOf(dbConfigFile); TagElement rootTag = dbConfigs.RootTag; CompositeTag dataSourceTag = (CompositeTag)dbConfigs.FindTag("dblinks"); GetDataSource(dataSourceTag); BrokerFactory.InitDataSource(DataSourceList); if (rootTag is CompositeTag) { foreach (CompositeTag dbConfig in ((CompositeTag)rootTag).SubTags) { if (!dbConfig.Name.Equals("dslist")) { InitalizeDatabaseConfig(dbConfig); } } } }
protected override IList ExecuteCrawl(bool crawlAll) { IList list = new ArrayList(); //取得页码 int pageInt = 1; string html = string.Empty; string viewState = string.Empty; string eventValidation = string.Empty; string cookiestr = string.Empty; try { html = this.ToolWebSite.GetHtmlByUrl("http://www.szsszx.com/tender/pager?key=caigou&pagenumber=20&pageindex=1", Encoding.UTF8); } catch (Exception ex) { return(list); } Parser parser = new Parser(new Lexer(html)); NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagelist"))); if (tdNodes != null && tdNodes.Count > 0) { string pageTemp = tdNodes.AsString().Replace(" ", ""); Regex regpage = new Regex(@"1/\d+"); try { pageInt = int.Parse(regpage.Match(pageTemp).Value.Replace("1/", "")); } catch (Exception ex) { } } for (int i = 1; i <= pageInt; i++) { if (i > 1) { html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=caigou&pagenumber=20&pageindex=" + i.ToString()), Encoding.UTF8); } parser = new Parser(new Lexer(html)); NodeList nodeList = parser.ExtractAllNodesThatMatch(new TagNameFilter("li")); if (nodeList != null && nodeList.Count > 0) { for (int j = 0; j < nodeList.Count; j++) { string code = string.Empty, buildUnit = string.Empty, prjName = string.Empty, prjAddress = string.Empty, inviteCtx = string.Empty, inviteType = string.Empty, specType = string.Empty, beginDate = string.Empty, endDate = string.Empty, remark = string.Empty, inviteCon = string.Empty, InfoUrl = string.Empty, CreateTime = string.Empty, msgType = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty; CompositeTag obj = nodeList[j] as CompositeTag; ATag aTag = obj.SearchFor(typeof(ATag), true)[0] as ATag; Span dateSpan = obj.SearchFor(typeof(Span), true)[0] as Span; prjName = aTag.GetAttribute("title"); beginDate = dateSpan.ToPlainTextString().Trim(new char[] { '[', ']' }); InfoUrl = "http://www.szsszx.com" + aTag.Link; string htmldetail = string.Empty; try { htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Trim(); Parser dtlparserHTML = new Parser(new Lexer(htmldetail)); NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div"))); HtmlTxt = dtnodeHTML.AsHtml(); htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n"); Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>"); htmldetail = regexHtml.Replace(htmldetail, ""); } catch (Exception ex) { continue; } Parser dtlparser = new Parser(new Lexer(htmldetail)); NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div"))); inviteCtx = dtnode.AsString().Replace(" ", "").Replace("[ifgtemso11]", "").Replace("[endif]", "").Replace("<!", ""); inviteCtx = Regex.Replace(inviteCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace("<", "").Replace(">", "").Replace("\n\n\n\t", "\r\n").Replace("\n\n", "\r\n"); code = inviteCtx.GetRegex("招标编号,编号", true, 50).Replace("开标时间安排如下", ""); Regex regbuildUnit = new Regex(@"(采购人|采购单位)(:|:)[^\r\n]+\r\n"); buildUnit = regbuildUnit.Match(inviteCtx).Value.Replace("采购人", "").Replace("采购单位", "").Replace(":", "").Replace(":", "").Trim(); if (string.IsNullOrEmpty(beginDate)) { beginDate = DateTime.Now.ToString("yyyy-MM-dd"); } Regex regprjAddress = new Regex(@"地点(:|:)[^\r\n]+\r\n"); prjAddress = regprjAddress.Match(inviteCtx).Value.Replace("地点", "").Replace(":", "").Replace(":", "").Trim(); specType = "其他"; msgType = "深圳市深水水务咨询有限公司"; inviteType = ToolHtml.GetInviteTypes(prjName); InviteInfo info = ToolDb.GenInviteInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, prjAddress, buildUnit, beginDate, endDate, inviteCtx, remark, msgType, inviteType, specType, otherType, InfoUrl, HtmlTxt); list.Add(info); if (!crawlAll && list.Count >= this.MaxCount) { return(list); } } } } return(list); }
private static string GetTagValue(CompositeTag Tags, string key) { String tempValue = null; TagElement tempTag = null; tempTag = Tags.FindTag(key); if (tempTag == null) { return null; } else { return tempTag.TryGetAttributeValueAt("value"); } }
private static void InitalizeDatabaseConfig(CompositeTag dbConfig) { foreach (CompositeTag tableConfig in dbConfig.SubTags) { InitalizeTableConfig(tableConfig, dbConfig.TryGetAttributeValueAt("id")); } }
private static void InitalizeTableConfig(CompositeTag tableConfig, string dsName) { List<string> pKeys = new List<string>(); foreach (TagElement columnConfig in tableConfig.SubTags) { string isKey = columnConfig.TryGetAttributeValueAt("isKey"); isKey = isKey.ToLower(); if (isKey != null && isKey.Equals("yes")) { pKeys.Add(columnConfig.TryGetAttributeValueAt("id")); } } TablesWithSchema.SetPrimaryKey(dsName, tableConfig.TryGetAttributeValueAt("id"), pKeys.ToArray()); }
protected override IList ExecuteCrawl(bool crawlAll) { IList list = new ArrayList(); //取得页码 int pageInt = 1; string html = string.Empty; string viewState = string.Empty; string eventValidation = string.Empty; string cookiestr = string.Empty; try { html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=zhongbiao&pagenumber=20&pageindex=1"), Encoding.UTF8); } catch (Exception ex) { return(list); } Parser parser = new Parser(new Lexer(html)); NodeList tdNodes = parser.ExtractAllNodesThatMatch(new AndFilter(new TagNameFilter("div"), new HasAttributeFilter("class", "pagelist"))); if (tdNodes != null && tdNodes.Count > 0) { string pageTemp = tdNodes.AsString().Replace(" ", ""); Regex regpage = new Regex(@"1/\d+"); try { pageInt = int.Parse(regpage.Match(pageTemp).Value.Replace("1/", "")); } catch (Exception ex) { } } for (int i = 1; i <= pageInt; i++) { if (i > 1) { html = this.ToolWebSite.GetHtmlByUrl(this.ToolWebSite.UrlEncode("http://www.szsszx.com/tender/pager?key=zhongbiao&pagenumber=20&pageindex=" + i.ToString()), Encoding.UTF8); } parser = new Parser(new Lexer(html)); NodeList nodeList = parser.ExtractAllNodesThatMatch(new TagNameFilter("li")); if (nodeList != null && nodeList.Count > 0) { for (int j = 0; j < nodeList.Count; j++) { string prjName = string.Empty, buildUnit = string.Empty, bidUnit = string.Empty, bidMoney = string.Empty, code = string.Empty, bidDate = string.Empty, beginDate = string.Empty, endDate = string.Empty, bidType = string.Empty, specType = string.Empty, InfoUrl = string.Empty, msgType = string.Empty, bidCtx = string.Empty, prjAddress = string.Empty, remark = string.Empty, prjMgr = string.Empty, otherType = string.Empty, HtmlTxt = string.Empty; CompositeTag obj = nodeList[j] as CompositeTag; ATag aTag = obj.SearchFor(typeof(ATag), true)[0] as ATag; Span dateSpan = obj.SearchFor(typeof(Span), true)[0] as Span; prjName = aTag.GetAttribute("title"); beginDate = dateSpan.ToPlainTextString().Trim(new char[] { '[', ']' }); InfoUrl = "http://www.szsszx.com" + aTag.Link; string htmldetail = string.Empty; try { htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Trim(); Parser dtlparserHTML = new Parser(new Lexer(htmldetail)); NodeList dtnodeHTML = dtlparserHTML.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div"))); HtmlTxt = dtnodeHTML.AsHtml(); htmldetail = this.ToolWebSite.GetHtmlByUrl(InfoUrl, Encoding.UTF8).Replace(" ", "").Replace("</br>", "\r\n").Replace("<br>", "\r\n"); Regex regexHtml = new Regex(@"<script[^<]*</script>|<\?xml[^/]*/>"); htmldetail = regexHtml.Replace(htmldetail, ""); } catch (Exception ex) { continue; } Parser dtlparser = new Parser(new Lexer(htmldetail)); NodeList dtnode = dtlparser.ExtractAllNodesThatMatch(new AndFilter(new HasAttributeFilter("class", "news-content"), new TagNameFilter("div"))); bidCtx = dtnode.ToHtml().ToCtxString(); bidCtx = Regex.Replace(bidCtx, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase).Replace("<", "").Replace(">", "").Replace("\n\n\n\t", "\r\n").Replace("\n\n", "\r\n"); Regex regCode = new Regex(@"招标编号(:|:)[^\r\n]+\r\n"); code = regCode.Match(bidCtx).Value.Replace("招标编号", "").Replace(":", "").Replace(":", "").Trim(); if (Encoding.Default.GetByteCount(code) > 50) { code = ""; } Regex regbuildUnit = new Regex(@"(采购人|采购单位|采购代理机构)(:|:)[^\r\n]+\r\n"); buildUnit = regbuildUnit.Match(bidCtx).Value.Replace("采购人", "").Replace("采购单位", "").Replace("采购代理机构", "").Replace(":", "").Replace(":", "").Trim(); prjAddress = bidCtx.GetAddressRegex(); bidUnit = bidCtx.GetBidRegex();// regBidUnit.Match(bidCtx).Value.Replace("中标单位", "").Replace(":", "").Replace(":", "").Trim(); Regex regBidMoneystr = new Regex(@"(中标价|价格|金额)(:|:)[^\r\n]+\r\n"); string monerystr = regBidMoneystr.Match(bidCtx).Value.Replace("中标价", "").Replace("价格", "").Replace("金额", "").Replace("万元整", "").Replace(":", "").Replace(":", "").Replace(",", "").Replace(",", "").Trim(); Regex regBidMoney = new Regex(@"[0-9]+[.]{0,1}[0-9]+"); if (!string.IsNullOrEmpty(regBidMoney.Match(monerystr).Value)) { if ((monerystr.Contains("万元") || monerystr.Contains("万美元")) && !monerystr.Contains("万元整")) { bidMoney = regBidMoney.Match(monerystr).Value; } else { try { bidMoney = (decimal.Parse(regBidMoney.Match(monerystr).Value) / 10000).ToString(); if (decimal.Parse(bidMoney) < decimal.Parse("0.1")) { bidMoney = "0"; } } catch (Exception) { bidMoney = "0"; } } } specType = "其他"; msgType = "深圳市深水水务咨询有限公司"; prjName = ToolDb.GetPrjName(prjName); bidType = ToolHtml.GetInviteTypes(prjName); BidInfo info = ToolDb.GenBidInfo("广东省", "深圳社会招标", "", string.Empty, code, prjName, buildUnit, beginDate, bidUnit, beginDate, endDate, bidCtx, string.Empty, msgType, bidType, specType, otherType, bidMoney, InfoUrl, prjMgr, HtmlTxt); list.Add(info); if (!crawlAll && list.Count >= this.MaxCount) { return(list); } } } } return(list); }
public void setParent(CompositeTag tag) { delegateNode.setParent(tag); }
public static Span GetSpan(this CompositeTag value, int i = 0) { return(ToolHtml.GetHtmlSpan(value.ToHtml(), null, i)); }
public void setParent(CompositeTag tag) { }
public static TableTag GetTableTag(this CompositeTag value, int i = 0) { return(ToolHtml.GetHtmlTableTag(value.ToHtml(), null, i)); }
/// <summary> /// 获取Html里A标签href /// </summary> /// <param name="value"></param> /// <param name="i"></param> /// <returns></returns> public static string GetATagHref(this CompositeTag value, int i = 0) { return(ToolHtml.GetHtmlAtag(value.ToHtml(), null, i).Link); }
private BaseControl CreateNode(Skill.Framework.AI.Behavior behavior, Skill.Framework.AI.BehaviorContainer container) { BaseControl control = null; switch (behavior.Type) { case Skill.Framework.AI.BehaviorType.Action: case Skill.Framework.AI.BehaviorType.Condition: case Skill.Framework.AI.BehaviorType.ChangeState: Label lbl = new Label(); lbl.Style = _FailurStyle; lbl.Content.text = behavior.Name; BehaviorTag bt = new BehaviorTag(behavior, container, lbl); _Behaviors.Add(bt); control = lbl; if (behavior.Type == Framework.AI.BehaviorType.Action) { lbl.Content.image = Resources.UITextures.BTree.Action; } else if (behavior.Type == Framework.AI.BehaviorType.ChangeState) { lbl.Content.image = Resources.UITextures.BTree.ChangeState; } else if (behavior.Type == Framework.AI.BehaviorType.Condition) { lbl.Content.image = Resources.UITextures.BTree.Condition; } break; case Skill.Framework.AI.BehaviorType.Decorator: case Skill.Framework.AI.BehaviorType.Composite: Skill.Editor.UI.FolderView fw = new UI.FolderView(); fw.Foldout.Style = _FailurStyle; fw.Foldout.Content.text = behavior.Name; fw.Foldout.IsOpen = true; CompositeTag ct = new CompositeTag(behavior, container, fw); _Composites.Add(ct); control = fw; if (behavior.Type == Framework.AI.BehaviorType.Composite) { Skill.Framework.AI.Composite composite = (Skill.Framework.AI.Composite)behavior; switch (composite.CompositeType) { case Skill.Framework.AI.CompositeType.Sequence: fw.Foldout.Content.image = Resources.UITextures.BTree.Sequence; break; case Skill.Framework.AI.CompositeType.Concurrent: fw.Foldout.Content.image = Resources.UITextures.BTree.Concurrent; break; case Skill.Framework.AI.CompositeType.Random: fw.Foldout.Content.image = Resources.UITextures.BTree.Random; break; case Skill.Framework.AI.CompositeType.Priority: case Skill.Framework.AI.CompositeType.State: fw.Foldout.Content.image = Resources.UITextures.BTree.Priority; break; case Skill.Framework.AI.CompositeType.Loop: fw.Foldout.Content.image = Resources.UITextures.BTree.Loop; break; } } else if (behavior.Type == Framework.AI.BehaviorType.Decorator) { Skill.Framework.AI.Decorator decorator = (Skill.Framework.AI.Decorator)behavior; switch (decorator.DecoratorType) { case Skill.Framework.AI.DecoratorType.Default: fw.Foldout.Content.image = Resources.UITextures.BTree.Decorator; break; case Skill.Framework.AI.DecoratorType.AccessLimit: fw.Foldout.Content.image = Resources.UITextures.BTree.AccessLimitDecorator; break; } } break; } return(control); }
/// <summary> /// 获取ATag属性值 /// </summary> /// <param name="value"></param> /// <param name="strName">ATag属性</param> /// <param name="i"></param> /// <returns></returns> public static string GetATagValue(this CompositeTag value, string strName = "href", int i = 0) { return(ToolHtml.GetHtmlAtagValue(strName, value.ToHtml(), null, i)); }
/// <summary> /// CompositeTag转换字符串 /// </summary> /// <param name="value"></param> /// <returns></returns> public static string ToNodePlainString(this CompositeTag value) { return(value.ToPlainTextString().ToNodeString().Replace(" ", "")); }