protected void Page_Load(object sender, EventArgs e)
{
if (!IsPostBack)
{
ddlModel.DataSource = this.bll.GetList();
ddlModel.DataBind();
DataTable dt = nodeBll.GetNodeListContainXML(0);
Node_L.Text = "<select id='node_dp' name='node_dp' class='form-control text_300'>" + nodeBll.CreateDP(dt) + "</select>";
if (ItemID > 0)
{
M_CollectionItem mc = bc.GetSelect(ItemID);
txtItemName.Text = mc.ItemName;
txtSiteName.Text = mc.SiteName;
Proto_DP.SelectedValue = mc.CollUrl.Contains("http:") ? "http://" : "https://";
txtUrl.Text = mc.PureUrl;
txtNum.Text = mc.CollNum.ToString();
ddlModel.SelectedValue = mc.ModeID.ToString();
rblCoding.SelectedValue = mc.CodinChoice.ToString();
txtContext.Text = mc.CollContext;
ddlModel.Enabled = false;
function.Script(this, "setNodeDP('" + mc.NodeID + "');");
title = "<span style='color:red;'>采集项目设置</span> >> <a title=\"列表页采集设置\" href=\"CollectionStep2.aspx?Action=Modify&ItemId=" + ItemID + "\">列表页采集设置</a> >> <a title=\"内容页采集设置\" href=\"CollectionStep3.aspx?Action=Modify&ItemId=" + ItemID + "\">内容页采集设置</a>";
}
}
Call.SetBreadCrumb(Master, "<li><a href='" + CustomerPageAction.customPath2 + "I/Main.aspx'>工作台</a></li><li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li class='active'><a href='" + Request.RawUrl + "'>采集管理</a></li>");
}
protected void Page_Load(object sender, EventArgs e)
{
if (!IsPostBack)
{
M_CollectionItem mc = bc.GetSelect(ItemID);
Url = mc.CollUrl;
lblLink.Text = "<a href='" + mc.CollUrl + "' target=\"_blank\" class='btn btn-info' style='color:white;'>查看原页面</a>";
SourceHtml_Hid.Value = htmlHelper.GetHtmlFromSite(Url);
if (!string.IsNullOrEmpty(mc.ListSettings))
{
lfMod = JsonConvert.DeserializeObject <M_Collect_ListFilter>(mc.ListSettings);
ListStart_T.Text = lfMod.ListStart;
ListEnd_T.Text = lfMod.ListEnd;
CharContain_T.Text = lfMod.CharContain;
CharRegex_T.Text = lfMod.CharRegex;
FillStart_T.Text = lfMod.FillStart;
FillEnd_T.Text = lfMod.FillEnd;
}
if (!string.IsNullOrEmpty(mc.LinkList))
{
txtHtml.Text = mc.LinkList;
}
else
{
txtHtml.Text = SourceHtml_Hid.Value;
}
Call.SetBreadCrumb(Master, "<li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li class='active'>当前:" + mc.ItemName + "</li>");
}
}
//将内容写入指定的节点
private static void CollectByUrl(string url, M_CollectionItem item, M_ModelInfo modMod, DataTable fieldDT)
{
string html = htmlHelp.GetHtmlFromSite(StrHelper.UrlDeal(url));
if (string.IsNullOrEmpty(html))
{
WriteLog("项目[" + item.ItemName + "]采集到的目标网址[" + url + "]的值为空", ERROR); return;
}
M_CommonData conMod = new M_CommonData();
conMod.ModelID = item.ModeID;
conMod.NodeID = DataConvert.CLng(item.NodeID);
conMod.TableName = modMod.TableName;
conMod.Status = (int)ZLEnum.ConStatus.Audited;
conMod.FirstNodeID = 0;
DataTable table = new DataTable();
table.Columns.Add(new DataColumn("FieldName", typeof(string)));
table.Columns.Add(new DataColumn("FieldType", typeof(string)));
table.Columns.Add(new DataColumn("FieldValue", typeof(string)));
List <M_Coll_FieldFilter> filterList = JsonConvert.DeserializeObject <List <M_Coll_FieldFilter> >(item.InfoPageSettings);
//系统字段处理
conMod.Title = GetValueFromHtml(html, "Title", filterList);
conMod.TopImg = GetValueFromHtml(html, "TopImg", filterList);
conMod.Inputer = GetValueFromHtml(html, "Inputer", filterList);
conMod.CreateTime = DataConvert.CDate(GetValueFromHtml(html, "CreateTime", filterList));
conMod.UpDateTime = DataConvert.CDate(GetValueFromHtml(html, "UpDateTime", filterList));
conMod.Hits = DataConvert.CLng(GetValueFromHtml(html, "Hits", filterList));
conMod.EliteLevel = DataConvert.CLng(GetValueFromHtml(html, "EliteLevel", filterList));
//扩展字段处理
foreach (DataRow field in fieldDT.Rows)
{
DataRow dr = table.NewRow();
dr["FieldName"] = field["FieldName"];
dr["FieldType"] = field["FieldType"];
dr["FieldValue"] = GetValueFromHtml(html, field["FieldName"].ToString(), filterList);
//对值再处理,避免取错值
switch (dr["FieldType"].ToString())
{
case "DateType":
dr["FieldValue"] = DataConvert.CDate(dr["FieldValue"]);
break;
}
table.Rows.Add(dr);
}
try
{
conMod.GeneralID = conBll.AddContent(table, conMod);
WriteLog("项目[" + item.ItemName + "]对[" + url + "]的采集完成,ID=" + conMod.GeneralID);
}
catch (Exception ex) { WriteLog("项目[" + item.ItemName + "]对[" + url + "]的采集,在写入数据库时出错,原因:" + ex.Message, ERROR); }
//----------
work_count++;
}
protected void Next_Btn_Click(object sender, EventArgs e)
{
M_CollectionItem mc = bc.GetSelect(ItemID);
mc.ListSettings = JsonConvert.SerializeObject(FillListFilter());
mc.LinkList = txtHtml.Text;
bc.GetUpdate(mc);
Response.Redirect("CollectionStep3.aspx?ItemId=" + ItemID);
}
protected void Save_Btn_Click(object sender, EventArgs e)
{
M_CollectionItem mc = bc.GetSelect(ItemID);
mc.Switch = 2;
mc.InfoPageSettings = InfoPage_Hid.Value;
bc.GetUpdate(mc);
function.WriteSuccessMsg("采集项目已配置成功", "CollectionManage.aspx");
}
public static void AddWork(M_CollectionItem item)
{
if (workList.FirstOrDefault(p => p.CItem_ID == item.CItem_ID) != null)
{
}
else
{
workList.Add(item);
}
}
/// <summary>
/// 采集(暂时未做定时采集)
/// </summary>
/// <param name="mailinfo"></param>
/// <returns></returns>
private static StringBuilder GetCollectXml(M_CollectionItem mailinfo)
{
StringBuilder str = new StringBuilder();
str.AppendLine("<XmlType>");
str.AppendLine("<CollectTime>");
str.AppendLine(mailinfo.LastTime.ToString());
str.AppendLine("</CollectTime>");
str.AppendLine("</XmlType>");
return(str);
}
public bool InsertUpdate(M_CollectionItem model)
{
if (model.CItem_ID > 0)
{
UpdateByID(model);
}
else
{
insert(model);
}
return(true);
}
protected void Page_Load(object sender, EventArgs e)
{
if (!IsPostBack)
{
M_CollectionItem mc = bc.GetSelect(ItemID);
//lblItemName.Text = mc.ItemName;
SYS_RPT.DataSource = GetCollFieldList();
SYS_RPT.DataBind();
DataTable dt = bfield.SelByModelID(mc.ModeID);
RPT.DataSource = dt;
RPT.DataBind();
InfoPage_Hid.Value = JsonConvert.SerializeObject(mc.InfoPageSettings);
Call.SetBreadCrumb(Master, "<li><a href='" + CustomerPageAction.customPath2 + "I/Main.aspx'>工作台</a></li><li><a href='ContentManage.aspx'>内容管理</a></li><li><a href='CollectionManage.aspx'>信息采集</a></li><li><a href='CollectionManage.aspx'>项目管理</a></li><li class='active'>添加设置采集第三步</li>");
}
}
//开始采集
protected void btnCollAll_Click(object sender, EventArgs e)
{
if (!string.IsNullOrEmpty(Request.Form["idchk"]))
{
string[] ids = Request.Form["idchk"].Split(new char[] { ',' });
foreach (string id in ids)
{
M_CollectionItem mc = bc.GetSelect(Convert.ToInt32(id));
mc.Switch = 1;
mc.LastTime = DateTime.Now.ToString();
bc.GetUpdate(mc);
B_Coll_Worker.AddWork(mc);
}
B_Coll_Worker.Start();
Response.Redirect("CollectionStatus.aspx");
}
}
protected void Button1_Click(object sender, EventArgs e)//点击开始采集
{
M_CollectionItem mc = new M_CollectionItem();
mc.CollContext = txtContext.Text;
mc.CodinChoice = DataConverter.CLng(rblCoding.SelectedValue);//采集字符编码
mc.CollNum = (txtNum.Text == "") ? 0 : DataConverter.CLng(txtNum.Text);
mc.CollUrl = StrHelper.UrlDeal(txtUrl.Text, Proto_DP.SelectedValue);
mc.ItemName = txtItemName.Text;
mc.SiteName = txtSiteName.Text;
mc.ModeID = DataConverter.CLng(ddlModel.SelectedValue);
mc.NodeID = Request.Form["node_dp"];
if (DataConverter.CLng(mc.NodeID) < 1 || mc.ModeID < 1)
{
function.WriteErrMsg("未指定节点或模型");
}
mc.CodinChoice = Convert.ToInt32(rblCoding.SelectedValue);
if (ItemID > 0)
{
M_CollectionItem mci = bc.GetSelect(ItemID);
mc.CItem_ID = ItemID;
mc.AddTime = mci.AddTime;
mc.InfoPageSettings = mci.InfoPageSettings;
mc.ListSettings = mci.ListSettings;
mc.PageSettings = mci.PageSettings;
mc.LinkList = mci.LinkList;
mc.LastTime = mci.LastTime;
mc.Switch = mci.Switch;
mc.State = mci.State;
bc.GetUpdate(mc);
Response.Redirect("CollectionStep2.aspx?Action=Modify&ItemId=" + ItemID);
}
else
{
mc.AddTime = DateTime.Now;
int newid = bc.GetInsert(mc);
Response.Redirect("CollectionStep2.aspx?Action=Insert&ItemId=" + newid);
}
}
public void MyBind()
{
M_CollectionItem mc = bc.GetSelect(ItemID);
Link_DP.DataSource = mc.LinkList.Split("\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
Link_DP.DataBind();
Url = StrHelper.UrlDeal(Link_DP.Items[0].Value);
txtHtml.Text = htmlHelper.GetHtmlFromSite(Url);
CurLink_L.Text = "获取成功：" + Url;
//----------------
try
{
if (!string.IsNullOrWhiteSpace(mc.InfoPageSettings))
{
FilterModel model = JsonConvert.DeserializeObject <FilterModel>(mc.InfoPageSettings);
EType_T.Text = model.EType;
ID_T.Text = model.ID;
Class_T.Text = model.CSS;
Script_Chk.Checked = model.AllowScript;
}
}
catch { }
}
public bool GetUpdate(M_CollectionItem model)
{
return(UpdateByID(model));
}
public int GetInsert(M_CollectionItem model)
{
return(insert(model));
}
public int insert(M_CollectionItem model)
{
return(DBCenter.Insert(model));
}
public bool UpdateByID(M_CollectionItem model)
{
return(DBCenter.UpdateByID(model, model.CItem_ID));
}
/// <summary>
/// 采集
/// </summary>
/// <param name="mailinfo"></param>
public void AddCollectio(M_CollectionItem Collectioinfo)
{
StringBuilder str = GetCollectXml(Collectioinfo);
FileSystemObject.WriteFile(Collectpath, str.ToString());
}
