public void cyPortalAnalysis(PageContentEntity entity) { try { string pContent = entity.PContent; Utilities util = new Utilities(); SqlBuild sqlBuild = new SqlBuild(); SqlPara sqlPara = new SqlPara(); ClsDB clsDB = new ClsDB(); RegFunc rf = new RegFunc(); ArrayList arrayList = rf.GetStrArr(pContent, "\"aid\":", ","); for (int i = 0; i < 1; i++) { string nexurl = "http://t.cjcyw.com:8081/ship/detail"; clsPageUrl.AddPageUrl(entity.ProgramName, entity.KeyWord, entity.PID, "cyDetail", entity.SiteUrl, entity.Url, nexurl, "POST", "aid=" + arrayList[i].ToString(), entity.EnCode, "aid=" + arrayList[i].ToString(), getuser().cookieContainer, entity.AContent, entity.TrySpiderTimes, entity.Depth + 1); } } catch (Exception ex) { ClsLog clsLog = new ClsLog(); clsLog.AddLog(DateTime.Now.ToString(), "分析数据失败" + ex.ToString()); clsLog.AddLog(DateTime.Now.ToString(), entity.SType + ";" + entity.Url + ";"); } }
public void spiderMain() { ClsLog clsLog = new ClsLog(); clsLog.AddLog(DateTime.Now.ToString(), "入口抓取开始"); bool flag = false; int CurrSpiderTimes = 1; ClsPageUrl clsPageUrl = new ClsPageUrl(); Control.CheckForIllegalCrossThreadCalls = false; clsPageUrl.AddPageUrl("ProgramName", "", "", "Portal", "", "", "http://cht.cjsyw.com:8080/ShipSource/listSS.aspx?pageno=1", "GET", "", "utf-8", "", null, "", 1, 1); }
public void spiderMain() { ClsLog clsLog = new ClsLog(); clsLog.AddLog(DateTime.Now.ToString(), "入口抓取开始"); Program.helper.OntxtviewCompleted(this, new EventControllerArgs() { IsSuccess = true, Msg = "入口抓取开始" }); bool flag = false; ClsPageUrl clsPageUrl = new ClsPageUrl(); Program.helper.OntxtviewCompleted(this, new EventControllerArgs() { IsSuccess = true, Msg = "开始登陆" }); foreach (user item in Program.userList) { CookieContainer cookie = new CookieContainer(); HttpClient httpClient = new HttpClient("", 0, false, cookie); Program.helper.OntxtviewCompleted(this, new EventControllerArgs() { IsSuccess = true, Msg = item.userName + "登陆" }); string content = httpClient.GetResponse("", "http://t.cjcyw.com:8081/login", "Post", "pwd=" + item.psw + "&userid=" + item.userName + ""); item.cookie = httpClient.Cookie; item.cookieContainer = httpClient.cookieContainer;; } Control.CheckForIllegalCrossThreadCalls = false; if (url_comb.Text == "全部") { //船源 clsPageUrl.AddPageUrl("ProgramName", "", "", "cyPortal", "", "", "http://t.cjcyw.com:8081/ship/list", "GET", "", "utf-8", "", null, "", 1, 1); //货源 clsPageUrl.AddPageUrl("ProgramName", "", "", "hyPortal", "", "", "http://t.cjcyw.com:8081/goods/list", "GET", "", "utf-8", "", null, "", 1, 1); for (int i = 1; i <= nmccda.Value; i++) { //船舶档案 clsPageUrl.AddPageUrl("ProgramName", "", "", "cydaPortal", "", "", "http://t.cjcyw.com:8081/Boat/BoatList.aspx?pageno=" + i + "&&", "GET", "", "utf-8", "", null, "", 1, 1); } } else if (url_comb.Text == "船源") { //船源 clsPageUrl.AddPageUrl("ProgramName", "", "", "cyPortal", "", "", "http://t.cjcyw.com:8081/ship/list", "GET", "", "utf-8", "", null, "", 1, 1); } else if (url_comb.Text == "货源") { //货源 clsPageUrl.AddPageUrl("ProgramName", "", "", "hyPortal", "", "", "http://t.cjcyw.com:8081/goods/list", "GET", "", "utf-8", "", null, "", 1, 1); } else if (url_comb.Text == "船舶档案") { for (int i = 1; i <= nmccda.Value; i++) { //船舶档案 clsPageUrl.AddPageUrl("ProgramName", "", "", "cydaPortal", "", "", "http://t.cjcyw.com:8081/Boat/BoatList.aspx?pageno=" + i + "&&", "GET", "", "utf-8", "", null, "", 1, 1); } } }