protected void Page_Load(object sender, EventArgs e) { if (!IsPostBack) { ddlModel.DataSource = this.bll.GetList(); ddlModel.DataBind(); DataTable dt = nodeBll.GetNodeListContainXML(0); Node_L.Text = "<select id='node_dp' name='node_dp' class='form-control text_300'>" + nodeBll.CreateDP(dt) + "</select>"; if (ItemID > 0) { M_CollectionItem mc = bc.GetSelect(ItemID); txtItemName.Text = mc.ItemName; txtSiteName.Text = mc.SiteName; Proto_DP.SelectedValue = mc.CollUrl.Contains("http:") ? "http://" : "https://"; txtUrl.Text = mc.PureUrl; txtNum.Text = mc.CollNum.ToString(); ddlModel.SelectedValue = mc.ModeID.ToString(); rblCoding.SelectedValue = mc.CodinChoice.ToString(); txtContext.Text = mc.CollContext; ddlModel.Enabled = false; function.Script(this, "setNodeDP('" + mc.NodeID + "');"); title = "<span style='color:red;'>采集项目设置</span> >> <a title=\"列表页采集设置\" href=\"CollectionStep2.aspx?Action=Modify&ItemId=" + ItemID + "\">列表页采集设置</a> >> <a title=\"内容页采集设置\" href=\"CollectionStep3.aspx?Action=Modify&ItemId=" + ItemID + "\">内容页采集设置</a>"; } } Call.SetBreadCrumb(Master, "<li><a href='" + CustomerPageAction.customPath2 + "I/Main.aspx'>工作台</a></li><li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li class='active'><a href='" + Request.RawUrl + "'>采集管理</a></li>"); }
protected void Page_Load(object sender, EventArgs e) { if (!IsPostBack) { M_CollectionItem mc = bc.GetSelect(ItemID); Url = mc.CollUrl; lblLink.Text = "<a href='" + mc.CollUrl + "' target=\"_blank\" class='btn btn-info' style='color:white;'>查看原页面</a>"; SourceHtml_Hid.Value = htmlHelper.GetHtmlFromSite(Url); if (!string.IsNullOrEmpty(mc.ListSettings)) { lfMod = JsonConvert.DeserializeObject <M_Collect_ListFilter>(mc.ListSettings); ListStart_T.Text = lfMod.ListStart; ListEnd_T.Text = lfMod.ListEnd; CharContain_T.Text = lfMod.CharContain; CharRegex_T.Text = lfMod.CharRegex; FillStart_T.Text = lfMod.FillStart; FillEnd_T.Text = lfMod.FillEnd; } if (!string.IsNullOrEmpty(mc.LinkList)) { txtHtml.Text = mc.LinkList; } else { txtHtml.Text = SourceHtml_Hid.Value; } Call.SetBreadCrumb(Master, "<li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li class='active'>当前:" + mc.ItemName + "</li>"); } }
//将内容写入指定的节点 private static void CollectByUrl(string url, M_CollectionItem item, M_ModelInfo modMod, DataTable fieldDT) { string html = htmlHelp.GetHtmlFromSite(StrHelper.UrlDeal(url)); if (string.IsNullOrEmpty(html)) { WriteLog("项目[" + item.ItemName + "]采集到的目标网址[" + url + "]的值为空", ERROR); return; } M_CommonData conMod = new M_CommonData(); conMod.ModelID = item.ModeID; conMod.NodeID = DataConvert.CLng(item.NodeID); conMod.TableName = modMod.TableName; conMod.Status = (int)ZLEnum.ConStatus.Audited; conMod.FirstNodeID = 0; DataTable table = new DataTable(); table.Columns.Add(new DataColumn("FieldName", typeof(string))); table.Columns.Add(new DataColumn("FieldType", typeof(string))); table.Columns.Add(new DataColumn("FieldValue", typeof(string))); List <M_Coll_FieldFilter> filterList = JsonConvert.DeserializeObject <List <M_Coll_FieldFilter> >(item.InfoPageSettings); //系统字段处理 conMod.Title = GetValueFromHtml(html, "Title", filterList); conMod.TopImg = GetValueFromHtml(html, "TopImg", filterList); conMod.Inputer = GetValueFromHtml(html, "Inputer", filterList); conMod.CreateTime = DataConvert.CDate(GetValueFromHtml(html, "CreateTime", filterList)); conMod.UpDateTime = DataConvert.CDate(GetValueFromHtml(html, "UpDateTime", filterList)); conMod.Hits = DataConvert.CLng(GetValueFromHtml(html, "Hits", filterList)); conMod.EliteLevel = DataConvert.CLng(GetValueFromHtml(html, "EliteLevel", filterList)); //扩展字段处理 foreach (DataRow field in fieldDT.Rows) { DataRow dr = table.NewRow(); dr["FieldName"] = field["FieldName"]; dr["FieldType"] = field["FieldType"]; dr["FieldValue"] = GetValueFromHtml(html, field["FieldName"].ToString(), filterList); //对值再处理,避免取错值 switch (dr["FieldType"].ToString()) { case "DateType": dr["FieldValue"] = DataConvert.CDate(dr["FieldValue"]); break; } table.Rows.Add(dr); } try { conMod.GeneralID = conBll.AddContent(table, conMod); WriteLog("项目[" + item.ItemName + "]对[" + url + "]的采集完成,ID=" + conMod.GeneralID); } catch (Exception ex) { WriteLog("项目[" + item.ItemName + "]对[" + url + "]的采集,在写入数据库时出错,原因:" + ex.Message, ERROR); } //---------- work_count++; }
protected void Next_Btn_Click(object sender, EventArgs e) { M_CollectionItem mc = bc.GetSelect(ItemID); mc.ListSettings = JsonConvert.SerializeObject(FillListFilter()); mc.LinkList = txtHtml.Text; bc.GetUpdate(mc); Response.Redirect("CollectionStep3.aspx?ItemId=" + ItemID); }
protected void Save_Btn_Click(object sender, EventArgs e) { M_CollectionItem mc = bc.GetSelect(ItemID); mc.Switch = 2; mc.InfoPageSettings = InfoPage_Hid.Value; bc.GetUpdate(mc); function.WriteSuccessMsg("采集项目已配置成功", "CollectionManage.aspx"); }
public static void AddWork(M_CollectionItem item) { if (workList.FirstOrDefault(p => p.CItem_ID == item.CItem_ID) != null) { } else { workList.Add(item); } }
/// <summary> /// 采集(暂时未做定时采集) /// </summary> /// <param name="mailinfo"></param> /// <returns></returns> private static StringBuilder GetCollectXml(M_CollectionItem mailinfo) { StringBuilder str = new StringBuilder(); str.AppendLine("<XmlType>"); str.AppendLine("<CollectTime>"); str.AppendLine(mailinfo.LastTime.ToString()); str.AppendLine("</CollectTime>"); str.AppendLine("</XmlType>"); return(str); }
public bool InsertUpdate(M_CollectionItem model) { if (model.CItem_ID > 0) { UpdateByID(model); } else { insert(model); } return(true); }
protected void Page_Load(object sender, EventArgs e) { if (!IsPostBack) { M_CollectionItem mc = bc.GetSelect(ItemID); //lblItemName.Text = mc.ItemName; SYS_RPT.DataSource = GetCollFieldList(); SYS_RPT.DataBind(); DataTable dt = bfield.SelByModelID(mc.ModeID); RPT.DataSource = dt; RPT.DataBind(); InfoPage_Hid.Value = JsonConvert.SerializeObject(mc.InfoPageSettings); Call.SetBreadCrumb(Master, "<li><a href='" + CustomerPageAction.customPath2 + "I/Main.aspx'>工作台</a></li><li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li><a href='CollectionManage.aspx'>项目管理</a></li><li class='active'>添加设置采集第三步</li>"); } }
//开始采集 protected void btnCollAll_Click(object sender, EventArgs e) { if (!string.IsNullOrEmpty(Request.Form["idchk"])) { string[] ids = Request.Form["idchk"].Split(new char[] { ',' }); foreach (string id in ids) { M_CollectionItem mc = bc.GetSelect(Convert.ToInt32(id)); mc.Switch = 1; mc.LastTime = DateTime.Now.ToString(); bc.GetUpdate(mc); B_Coll_Worker.AddWork(mc); } B_Coll_Worker.Start(); Response.Redirect("CollectionStatus.aspx"); } }
protected void Button1_Click(object sender, EventArgs e)//点击开始采集 { M_CollectionItem mc = new M_CollectionItem(); mc.CollContext = txtContext.Text; mc.CodinChoice = DataConverter.CLng(rblCoding.SelectedValue);//采集字符编码 mc.CollNum = (txtNum.Text == "") ? 0 : DataConverter.CLng(txtNum.Text); mc.CollUrl = StrHelper.UrlDeal(txtUrl.Text, Proto_DP.SelectedValue); mc.ItemName = txtItemName.Text; mc.SiteName = txtSiteName.Text; mc.ModeID = DataConverter.CLng(ddlModel.SelectedValue); mc.NodeID = Request.Form["node_dp"]; if (DataConverter.CLng(mc.NodeID) < 1 || mc.ModeID < 1) { function.WriteErrMsg("未指定节点或模型"); } mc.CodinChoice = Convert.ToInt32(rblCoding.SelectedValue); if (ItemID > 0) { M_CollectionItem mci = bc.GetSelect(ItemID); mc.CItem_ID = ItemID; mc.AddTime = mci.AddTime; mc.InfoPageSettings = mci.InfoPageSettings; mc.ListSettings = mci.ListSettings; mc.PageSettings = mci.PageSettings; mc.LinkList = mci.LinkList; mc.LastTime = mci.LastTime; mc.Switch = mci.Switch; mc.State = mci.State; bc.GetUpdate(mc); Response.Redirect("CollectionStep2.aspx?Action=Modify&ItemId=" + ItemID); } else { mc.AddTime = DateTime.Now; int newid = bc.GetInsert(mc); Response.Redirect("CollectionStep2.aspx?Action=Insert&ItemId=" + newid); } }
public void MyBind() { M_CollectionItem mc = bc.GetSelect(ItemID); Link_DP.DataSource = mc.LinkList.Split("\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries); Link_DP.DataBind(); Url = StrHelper.UrlDeal(Link_DP.Items[0].Value); txtHtml.Text = htmlHelper.GetHtmlFromSite(Url); CurLink_L.Text = "获取成功:" + Url; //---------------- try { if (!string.IsNullOrWhiteSpace(mc.InfoPageSettings)) { FilterModel model = JsonConvert.DeserializeObject <FilterModel>(mc.InfoPageSettings); EType_T.Text = model.EType; ID_T.Text = model.ID; Class_T.Text = model.CSS; Script_Chk.Checked = model.AllowScript; } } catch { } }
public bool GetUpdate(M_CollectionItem model) { return(UpdateByID(model)); }
public int GetInsert(M_CollectionItem model) { return(insert(model)); }
public int insert(M_CollectionItem model) { return(DBCenter.Insert(model)); }
public bool UpdateByID(M_CollectionItem model) { return(DBCenter.UpdateByID(model, model.CItem_ID)); }
/// <summary> /// 采集 /// </summary> /// <param name="mailinfo"></param> public void AddCollectio(M_CollectionItem Collectioinfo) { StringBuilder str = GetCollectXml(Collectioinfo); FileSystemObject.WriteFile(Collectpath, str.ToString()); }