private void 制任务ToolStripMenuItem_Click(object sender, EventArgs e) { int ID = Get_DataViewID(); DALTask dal = new DALTask(); int currentMaxId = dal.GetMaxId(); ModelTask model = dal.GetModelSingleTask(ID); string currentTaskName = model.TaskName + "复制" + DateTime.Now.Millisecond; model.ID = currentMaxId; model.TaskName = currentTaskName; dal.Add(model); DALTaskLabel dalLable = new DALTaskLabel(); DataTable dt = dalLable.GetList(" TaskId=" + ID).Tables[0]; if (dt != null && dt.Rows.Count > 0) { foreach (DataRow dr in dt.Rows) { ModelTaskLabel modelLabel = dalLable.GetModel(StringHelper.Instance.SetNumber(dr["Id"].ToString())); modelLabel.TaskID = currentMaxId; dalLable.Add(modelLabel); } } CreateDataFile(currentTaskName, currentMaxId); this.ClassID = model.TaskClassID.Value; Bind_DataList(); }
/// <summary> /// 测试采集内容 /// </summary> private void btnTestViewUrl_Click(object sender, EventArgs e) { this.btnTestViewUrl.Enabled = false; this.txtTestViewUrlShow.Clear(); var Test_ViewUrl = this.txtTextViewUrl.Text; var encode = ((ListItem)this.ddlItemEncode.SelectedItem).Value; DALTaskLabel dal = new DALTaskLabel(); List <ModelTaskLabel> Test_LabelList = new List <ModelTaskLabel>(); ListView.ListViewItemCollection s = this.listViewTaskLabel.Items; foreach (ListViewItem ss in s) { Test_LabelList.Add(dal.GetModel(ss.SubItems[0].Text, ID)); } var model = new ModelTask() { TestViewUrl = Test_ViewUrl, PageEncode = encode }; var task = new TaskFactory().StartNew(() => { var spiderViewHelper = new SpiderViewHelper(); spiderViewHelper.Model = model; spiderViewHelper.OutViewUrlContentHandler += (string content) => { this.txtTestViewUrlShow.Invoke(new MethodInvoker(delegate() { this.txtTestViewUrlShow.AppendText(content); this.btnTestViewUrl.Enabled = true; })); }; spiderViewHelper.TestAllLabel(Test_ViewUrl, Test_LabelList); }); }
/// <summary> /// 创建文件 /// </summary> /// <param name="taskName"></param> private void CreateDataFile(string taskName, int taskID) { string baseDir = AppDomain.CurrentDomain.BaseDirectory + "Data\\Collection\\"; string SQLiteName = baseDir + taskName + "\\SpiderResult.db"; string LocalSQLiteName = "Data\\Collection\\" + taskName + "\\SpiderResult.db"; string SQL = string.Empty; if (!Directory.Exists(baseDir + taskName + "\\")) { Directory.CreateDirectory(baseDir + taskName + "\\"); } if (!File.Exists(SQLiteName)) { DbHelper.CreateDataBase(SQLiteName); DALTask dal = new DALTask(); string createSQL = string.Empty; DataTable dt = new DALTaskLabel().GetList(" TaskID=" + taskID).Tables[0]; foreach (DataRow dr in dt.Rows) { createSQL += @" [" + dr["LabelName"] + @"] varchar,"; } if (createSQL.Trim() != "") { createSQL = createSQL.Remove(createSQL.Length - 1); } string ss = string.Empty; if (!string.IsNullOrEmpty(createSQL)) { ss = ","; } SQL = @" create table Content( ID integer primary key autoincrement, [HrefSource] varchar, [已采] int, [已发] int" + ss + @" " + createSQL + @" ); "; DbHelper.Execute(LocalSQLiteName, SQL); } else { DataTable dt = new DALTaskLabel().GetList(" TaskID=" + taskID).Tables[0]; foreach (DataRow dr in dt.Rows) { try { DbHelper.Execute(LocalSQLiteName, " ALTER TABLE Content ADD COLUMN [" + dr["LabelName"] + "] VARCHAR; "); } catch { continue; } } } }
private void Bind_contextMenuStrip_Label(ContextMenuStrip cms) { cms.Items.Clear(); cms.Items.Add("Guid"); cms.Items.Add("Url"); DALTaskLabel dal = new DALTaskLabel(); DataTable dt = dal.GetList(" TaskID=" + this.ID + " Order by OrderID Asc").Tables[0]; foreach (DataRow dr in dt.Rows) { cms.Items.Add(dr["LabelName"].ToString()); } }
/// <summary> /// 加载任务标签 /// </summary> private void Bind_TaskLabel(string strWhere) { this.listViewTaskLabel.Items.Clear(); DALTaskLabel dal = new DALTaskLabel(); DataTable dt = dal.GetList(strWhere + " Order by OrderID Asc").Tables[0]; foreach (DataRow dr in dt.Rows) { ListViewItem li = new ListViewItem(dr["LabelName"].ToString()); li.SubItems.Add(dr["LabelNameCutRegex"].ToString()); li.Tag = dr["ID"].ToString(); this.listViewTaskLabel.Items.Add(li); } }
/// <summary> /// 标签下降 /// </summary> private void btnLabelDown_Click(object sender, EventArgs e) { if (this.listViewTaskLabel.SelectedItems.Count > 0) { var sel = this.listViewTaskLabel.SelectedItems; if (sel != null && sel.Count == 1) { ListView.SelectedListViewItemCollection item = this.listViewTaskLabel.SelectedItems; int iID = int.Parse("0" + item[0].Tag); DALTaskLabel dal = new DALTaskLabel(); dal.UpdateOrder(this.ID, iID, 1); this.Bind_TaskLabel(" TaskID=" + this.ID); } } else { MessageBox.Show("请先选择标签再进行操作!", "警告!"); } }
private void btnTaskLabelCopy_Click(object sender, EventArgs e) { if (this.listViewTaskLabel.SelectedItems.Count > 0) { var sel = this.listViewTaskLabel.SelectedItems; if (sel != null && sel.Count > 0) { ListView.SelectedListViewItemCollection item = this.listViewTaskLabel.SelectedItems; int ID = int.Parse("0" + item[0].Tag.ToString()); DALTaskLabel dal = new DALTaskLabel(); dal.TaskLabelCopy(ID); this.Bind_TaskLabel(" TaskID=" + this.ID); } } else { MessageBox.Show("请先选择标签再进行操作!", "警告!"); } }
/// <summary> /// 删除标签 /// </summary> private void btnCutLabelDel_Click(object sender, EventArgs e) { if (this.listViewTaskLabel.SelectedItems.Count > 0) { var sel = this.listViewTaskLabel.SelectedItems; if (sel != null && sel.Count == 1) { if (MessageBox.Show("你确定要删除吗?删除不可恢复!", "警告!", MessageBoxButtons.OKCancel, MessageBoxIcon.Warning) == DialogResult.OK) { ListView.SelectedListViewItemCollection item = this.listViewTaskLabel.SelectedItems; DALTaskLabel dal = new DALTaskLabel(); dal.Delete(int.Parse("0" + item[0].Tag)); this.Bind_TaskLabel(" TaskID=" + this.ID); } } } else { MessageBox.Show("请先选择标签再进行删除!", "警告!"); } }
private void btnSubmit_Click(object sender, EventArgs e) { string WebCollectionModule = this.txtWebCollectionModule.Text; if (string.IsNullOrEmpty(WebCollectionModule)) { MessageBox.Show("采集模块不能为空!"); return; } var model = SerializeHelper.DeserializeObject <ModelTask>(WebCollectionModule); DALTask dal = new DALTask(); int currentMaxId = dal.GetMaxId(); model.ID = currentMaxId; dal.Add(model); foreach (ModelTaskLabel label in model.ListTaskLabel) { label.TaskID = currentMaxId; var d = new DALTaskLabel(); var m = d.GetMaxID(); label.ID = m; d.Add(label); } foreach (ModelWebPublishModule label in model.ListModelWebPublishModule) { label.TaskID = currentMaxId; var d = new DALWebPublishModule(); label.ID = 0; d.Add(label); } MessageBox.Show("导入成功"); RefreshParentForm?.Invoke(this, new EventArgs()); this.Hide(); this.Close(); }
/// <summary> /// ±à¼±êÇ© /// </summary> private void Bind_DataEdit() { DALTaskLabel dal = new DALTaskLabel(); ListView.SelectedListViewItemCollection item = (ListView.SelectedListViewItemCollection)EditItem; int ID = int.Parse("0" + item[0].Tag); ModelTaskLabel model = dal.GetModel(ID); this.txtID.Text = ID.ToString(); this.txtLabelName.Text = model.LabelName; this.txtLabelNameCutRegex.Text = model.LabelNameCutRegex; this.chkScript.Checked = false; this.chkhref.Checked = false; if (!string.IsNullOrEmpty(model.LabelHtmlRemove)) { string[] arr = model.LabelHtmlRemove.Split(new string[] { "||||" }, StringSplitOptions.RemoveEmptyEntries); foreach (string str in arr) { if (str == "all") { this.chkAllHtml.Checked = true; } else if (str == "table") { this.chkTable.Checked = true; } else if (str == "font<span>") { this.chkFont.Checked = true; } else if (str == "a") { this.chkhref.Checked = true; } else if (str == "script") { this.chkScript.Checked = true; } } } #region ÄÚÈÝÅųý if (!string.IsNullOrEmpty(model.LabelRemove)) { foreach (string str in model.LabelRemove.Split(new string[] { "$$$$" }, StringSplitOptions.RemoveEmptyEntries)) { string[] aa = str.Split(new string[] { "||" }, StringSplitOptions.None); ListViewItem lvi = new ListViewItem(aa[0]); lvi.SubItems.Add(aa[1]); this.listViewContentRemove.Items.Add(lvi); } } #endregion #region ÄÚÈÝÌæ»» if (!string.IsNullOrEmpty(model.LabelReplace)) { foreach (string str in model.LabelReplace.Split(new string[] { "$$$$" }, StringSplitOptions.RemoveEmptyEntries)) { string[] aa = str.Split(new string[] { "||" }, StringSplitOptions.None); ListViewItem lvi = new ListViewItem(aa[0]); lvi.SubItems.Add(aa[1]); this.listViewContentReplace.Items.Add(lvi); } } #endregion this.chkLabelIsLoop.Checked = model.IsLoop == 1 ? true : false; this.cmbSpiderPlugin.Text = model.SpiderLabelPlugin == string.Empty ? "²»Ê¹Óòå¼þ" : model.SpiderLabelPlugin; this.chkDownResource.Checked = model.IsDownResource == 1 ? true : false; this.txtDownResourceExt.Text = model.DownResourceExts; }
private void btnSubmit_Click(object sender, EventArgs e) { DALTask dal = new DALTask(); ModelTask model = new ModelTask(); errorProvider.Clear(); if (string.IsNullOrEmpty(this.cmbSiteClassID.Text)) { errorProvider.SetError(this.cmbSiteClassID, "所属站点不能为空!"); return; } if (string.IsNullOrEmpty(this.txtTaskName.Text)) { errorProvider.SetError(this.txtTaskName, "任务名称不能为空!"); return; } if (string.IsNullOrEmpty(this.txtDemoListUrl.Text)) { errorProvider.SetError(this.txtDemoListUrl, "测试网站列表地址不能为空!"); return; } if (string.IsNullOrEmpty(this.ddlItemEncode.Text)) { errorProvider.SetError(this.ddlItemEncode, "网页编码不能为空!"); return; } int ID = int.Parse(this.txtID.Text); int TaskClassID = int.Parse(this.cmbSiteClassID.SelectedValue.ToString()); string TaskName = this.txtTaskName.Text; string OldTaskName = this.txtOldTaskName.Text; int IsSpiderUrl = this.chkIsSpiderUrl.Checked ? 1 : 0; int IsSpiderContent = this.chkIsSpiderContent.Checked ? 1 : 0; int IsPublishContent = this.chkIsPublishContent.Checked ? 1 : 0; string PageEncode = ((ListItem)this.ddlItemEncode.SelectedItem).Value; string CollectionContent = string.Empty; foreach (string str in listBoxLinkUrl.Items) { CollectionContent += str + "$$$$"; } string LinkUrlMustIncludeStr = this.txtLinkUrlMustIncludeStr.Text; string LinkUrlNoMustIncludeStr = this.txtLinkUrlNoMustIncludeStr.Text; string LinkUrlCutAreaStart = this.txtLinkUrlCutAreaStart.Text.Replace("'", "''"); string LinkUrlCutAreaEnd = this.txtLinkUrlCutAreaEnd.Text.Replace("'", "''"); string TestViewUrl = this.txtTextViewUrl.Text; int IsWebOnlinePublish1 = this.chkPublish01.Checked ? 1 : 0; int IsSaveLocal2 = this.chkPublish02.Checked ? 1 : 0; string SaveFileFormat2 = this.ddlSaveFileFormat2.Text; string SaveDirectory2 = this.txtSaveDirectory2.Text; string SaveHtmlTemplate2 = this.txtSaveHtmlTemplate2.Text; int IsSaveDataBase3 = this.chkPublish03.Checked ? 1 : 0; int SaveDataType3 = 0; if (this.rbtnAccess.Checked) { SaveDataType3 = 1; } else if (this.rbtnMsSql.Checked) { SaveDataType3 = 2; } else if (this.rbtnSqlite.Checked) { SaveDataType3 = 3; } else if (this.rbtnMySql.Checked) { SaveDataType3 = 4; } else if (this.rbtnSqlite.Checked) { SaveDataType3 = 5; } string SaveDataUrl3 = this.txtSaveDataUrl3.Text; string SaveDataSQL3 = this.txtSaveDataSQL3.Text; int IsSaveSQL4 = this.chkPublish04.Checked ? 1 : 0; decimal CollectionContentThreadCount = this.nudCollectionContentThreadCount.Value; decimal CollectionContentStepTime = this.nudCollectionContentStepTime.Value; decimal PublishContentThreadCount = this.nudPublishContentThreadCount.Value; decimal PublishContentStepTimeMin = this.nudPublishContentStepTimeMin.Value; decimal PublishContentStepTimeMax = this.nudPublishContentStepTimeMax.Value; int IsHandGetUrl = this.chkIsHandGetUrl.Checked ? 1 : 0; string HandCollectionUrlRegex = this.txtHandCollectionUrlRegex.Text; int IsPlan = this.chkTaskSetStatus.Checked ? 1 : 0; string PlanFormat = this.txtHiddenPlanFormat.Text; model.ID = ID; model.TaskClassID = TaskClassID; model.TaskName = TaskName; model.IsSpiderUrl = IsSpiderUrl; model.IsSpiderContent = IsSpiderContent; model.IsPublishContent = IsPublishContent; model.PageEncode = PageEncode; model.CollectionContent = CollectionContent; model.LinkUrlMustIncludeStr = LinkUrlMustIncludeStr; model.LinkUrlNoMustIncludeStr = LinkUrlNoMustIncludeStr; model.LinkUrlCutAreaStart = LinkUrlCutAreaStart; model.LinkUrlCutAreaEnd = LinkUrlCutAreaEnd; model.TestViewUrl = TestViewUrl; model.IsWebOnlinePublish1 = IsWebOnlinePublish1; model.IsSaveLocal2 = IsSaveLocal2; model.SaveFileFormat2 = SaveFileFormat2; model.SaveDirectory2 = SaveDirectory2; model.SaveHtmlTemplate2 = SaveHtmlTemplate2; model.IsSaveDataBase3 = IsSaveDataBase3; model.SaveDataType3 = SaveDataType3; model.SaveDataUrl3 = SaveDataUrl3; model.SaveDataSQL3 = SaveDataSQL3; model.IsSaveSQL4 = IsSaveSQL4; model.PluginSpiderUrl = this.cmbSpiderUrlPlugins.Text; model.PluginSaveContent = this.cmbSaveConentPlugins.Text; model.PluginPublishContent = this.cmbPublishContentPlugins.Text; model.CollectionUrlStepTime = int.Parse(nudCollectionUrlStepTime.Value.ToString()); model.CollectionContentThreadCount = Convert.ToInt32(CollectionContentThreadCount); model.CollectionContentStepTime = Convert.ToInt32(CollectionContentStepTime); model.PublishContentThreadCount = Convert.ToInt32(PublishContentThreadCount); model.PublishContentStepTimeMin = Convert.ToInt32(PublishContentStepTimeMin); model.PublishContentStepTimeMax = Convert.ToInt32(PublishContentStepTimeMax); model.IsHandGetUrl = IsHandGetUrl; model.HandCollectionUrlRegex = HandCollectionUrlRegex; model.DemoListUrl = this.txtDemoListUrl.Text; model.IsPlan = IsPlan; model.PlanFormat = PlanFormat; model.Status = this.cmbStatus.SelectedIndex == 0 ? 1 : 0; model.IsSource = this.chkIsSource.Checked ? 1 : 0; model.SourceText = this.txtSourceText.Text; if (ID == 0) { string guid = Guid.NewGuid().ToString(); ID = dal.GetMaxId(); model.ID = ID; model.Guid = guid; dal.Add(model); DALTaskLabel dalTaskLabel = new DALTaskLabel(); dalTaskLabel.UpdateTaskLabelByTaskID(ID); if (TaskOpDelegate != null) { TaskOpEv.TaskIndex = TaskIndex; TaskOpDelegate(this, TaskOpEv); } } else if (ID > 0) { if (TaskName != OldTaskName) { string RootPath = AppDomain.CurrentDomain.BaseDirectory + "Data\\Collection\\"; string OldTaskPath = RootPath + OldTaskName; string NewTaskPath = RootPath + TaskName; if (Directory.Exists(NewTaskPath)) { MessageBox.Show("任务名称存在!请换个名称试试!"); return; } try { Directory.Move(OldTaskPath, NewTaskPath); } catch (Exception ex) { MessageBox.Show("修改异常!" + ex.Message); return; } } if (!dal.CheckTaskGuid(ID)) { model.Guid = Guid.NewGuid().ToString(); } dal.Update(model); if (TaskOpDelegate != null) { TaskOpEv.TaskIndex = TaskIndex; TaskOpEv.OpType = 1; TaskOpDelegate(this, TaskOpEv); } } CreateDataFile(TaskName, ID); #region 更新计划任务 if (model.IsPlan == 1) { PlanTaskHelper.PushJobDetail(ID); } #endregion this.CloseForm(); }
/// <summary> /// 创建文件 /// </summary> /// <param name="taskName"></param> private void CreateDataFile(string taskName, int taskID) { string baseDir = AppDomain.CurrentDomain.BaseDirectory + "Data\\Collection\\"; string SQLiteName = baseDir + taskName + "\\SpiderResult.db"; string LocalSQLiteName = "Data\\Collection\\" + taskName + "\\SpiderResult.db"; string SQL = string.Empty; if (!Directory.Exists(baseDir + taskName + "\\")) { Directory.CreateDirectory(baseDir + taskName + "\\"); } if (!File.Exists(SQLiteName)) { DbHelper.CreateDataBase(SQLiteName); DALTask dal = new DALTask(); string createSQL = string.Empty; DataTable dt = new DALTaskLabel().GetList(" TaskID=" + taskID).Tables[0]; foreach (DataRow dr in dt.Rows) { createSQL += @" " + dr["LabelName"] + @" varchar,"; } if (createSQL.Trim() != "") { createSQL = createSQL.Remove(createSQL.Length - 1); } SQL = @" create table Content( ID integer primary key autoincrement, [HrefSource] varchar, [已采] int, [已发] int, " + createSQL + @" ); "; DbHelper.Execute(LocalSQLiteName, SQL); } else { DataTable dt = new DALTaskLabel().GetList(" TaskID=" + taskID).Tables[0]; //20190522 更新原因,原来删除或更改字段时,本地数据库中原有列不能删除,或变更 var columns = "ID,[HrefSource],[已采],[已发],"; foreach (DataRow dr in dt.Rows) { try { //20190522 更新原因,原来删除或更改字段时,本地数据库中原有列不能删除,或变更 columns += "[" + dr["LabelName"] + "],"; DbHelper.Execute(LocalSQLiteName, " ALTER TABLE Content ADD COLUMN [" + dr["LabelName"] + "] VARCHAR; "); } catch { continue; } } //20190522 更新原因,原来删除或更改字段时,本地数据库中原有列不能删除,或变更 columns = columns.TrimEnd(','); if (!string.IsNullOrEmpty(columns)) { DbHelper.Execute(LocalSQLiteName, $"create table tempContent as select {columns} from Content "); DbHelper.Execute(LocalSQLiteName, $"drop table if exists Content "); DbHelper.Execute(LocalSQLiteName, $"alter table tempContent rename to Content "); } } }
/// <summary> /// 编辑标签 /// </summary> private void Bind_DataEdit() { DALTaskLabel dal = new DALTaskLabel(); ListView.SelectedListViewItemCollection item = (ListView.SelectedListViewItemCollection)EditItem; int ID = int.Parse("0" + item[0].Tag); ModelTaskLabel model = dal.GetModel(ID); this.txtID.Text = ID.ToString(); this.txtLabelName.Text = model.LabelName; if (model.LabelSource == 1) { this.radioBtnSource.Checked = true; } else { this.radioBtnUrl.Checked = true; } this.txtLabelNameCutRegex.Text = model.LabelNameCutRegex; this.chkScript.Checked = false; this.chkhref.Checked = false; if (!string.IsNullOrEmpty(model.LabelHtmlRemove)) { string[] arr = model.LabelHtmlRemove.Split(new string[] { "||||" }, StringSplitOptions.RemoveEmptyEntries); foreach (string str in arr) { if (str == "all") { this.chkAllHtml.Checked = true; } else if (str == "table") { this.chkTable.Checked = true; } else if (str == "font<span>") { this.chkFont.Checked = true; } else if (str == "a") { this.chkhref.Checked = true; } else if (str == "script") { this.chkScript.Checked = true; } } } if (!string.IsNullOrEmpty(model.LblHtmlRemove)) { string[] arr = model.LblHtmlRemove.Split(new string[] { "||||" }, StringSplitOptions.RemoveEmptyEntries); foreach (string str in arr) { if (str == "table") { this.chkTableLbl.Checked = true; } else if (str == "script") { this.chkScriptLbl.Checked = true; } else if (str == "a") { this.chkHrefLbl.Checked = true; } else if (str == "span") { this.chkSpanLbl.Checked = true; } else if (str == "p") { this.chkPLbl.Checked = true; } else if (str == "iframe") { this.chkIFrameLbl.Checked = true; } else if (str == "font") { this.chkFontLbl.Checked = true; } else if (str == "style") { this.chkStyleLbl.Checked = true; } else if (str == "div") { this.chkDivLbl.Checked = true; } else if (str == "remark") { this.chkRemarkLbl.Checked = true; } } } #region 内容排除 if (!string.IsNullOrEmpty(model.LabelRemove)) { foreach (string str in model.LabelRemove.Split(new string[] { "$$$$" }, StringSplitOptions.RemoveEmptyEntries)) { //ListViewItem lvi = new ListViewItem(str); //this.listViewContentRemove.Items.Add(lvi); string[] aa = str.Split(new string[] { "||" }, StringSplitOptions.None); ListViewItem lvi = new ListViewItem(aa[0]); lvi.SubItems.Add(aa[1]); this.listViewContentRemove.Items.Add(lvi); } } #endregion #region 内容替换 if (!string.IsNullOrEmpty(model.LabelReplace)) { foreach (string str in model.LabelReplace.Split(new string[] { "$$$$" }, StringSplitOptions.RemoveEmptyEntries)) { string[] aa = str.Split(new string[] { "||" }, StringSplitOptions.None); ListViewItem lvi = new ListViewItem(aa[0]); lvi.SubItems.Add(aa[1]); this.listViewContentReplace.Items.Add(lvi); } } #endregion this.chkLabelIsLoop.Checked = model.IsLoop == 1 ? true : false; this.chkLabelNoNull.Checked = model.IsNoNull == 1 ? true : false; this.chkLabelIsLinkUrl.Checked = model.IsLinkUrl == 1 ? true : false; this.chkLabelIsPager.Checked = model.IsPager == 1 ? true : false; this.txtLabelValueIsLinkUrlRegex.Text = model.LabelValueLinkUrlRegex; this.txtLabelValueIsPagerRegex.Text = model.LabelValuePagerRegex; this.cmbSpiderPlugin.Text = model.SpiderLabelPlugin == string.Empty ? "不使用插件" : model.SpiderLabelPlugin; //下载资源 this.chkDownResource.Checked = model.IsDownResource == 1 ? true : false; this.txtDownResourceExt.Text = model.DownResourceExts; }