public void cyPortalAnalysis(PageContentEntity entity)
{
try
{
string pContent = entity.PContent;
Utilities util = new Utilities();
SqlBuild sqlBuild = new SqlBuild();
SqlPara sqlPara = new SqlPara();
ClsDB clsDB = new ClsDB();
RegFunc rf = new RegFunc();
ArrayList arrayList = rf.GetStrArr(pContent, "\"aid\":", ",");
for (int i = 0; i < 1; i++)
{
string nexurl = "http://t.cjcyw.com:8081/ship/detail";
clsPageUrl.AddPageUrl(entity.ProgramName, entity.KeyWord, entity.PID, "cyDetail", entity.SiteUrl, entity.Url, nexurl,
"POST", "aid=" + arrayList[i].ToString(), entity.EnCode, "aid=" + arrayList[i].ToString(), getuser().cookieContainer, entity.AContent, entity.TrySpiderTimes, entity.Depth + 1);
}
}
catch (Exception ex)
{
ClsLog clsLog = new ClsLog();
clsLog.AddLog(DateTime.Now.ToString(), "分析数据失败" + ex.ToString());
clsLog.AddLog(DateTime.Now.ToString(), entity.SType + ";" + entity.Url + ";");
}
}
public void spiderMain()
{
ClsLog clsLog = new ClsLog();
clsLog.AddLog(DateTime.Now.ToString(), "入口抓取开始");
bool flag = false;
int CurrSpiderTimes = 1;
ClsPageUrl clsPageUrl = new ClsPageUrl();
Control.CheckForIllegalCrossThreadCalls = false;
clsPageUrl.AddPageUrl("ProgramName", "", "", "Portal", "", "", "http://cht.cjsyw.com:8080/ShipSource/listSS.aspx?pageno=1",
"GET", "", "utf-8", "", null, "", 1, 1);
}
public void spiderMain()
{
ClsLog clsLog = new ClsLog();
clsLog.AddLog(DateTime.Now.ToString(), "入口抓取开始");
Program.helper.OntxtviewCompleted(this, new EventControllerArgs()
{
IsSuccess = true, Msg = "入口抓取开始"
});
bool flag = false;
ClsPageUrl clsPageUrl = new ClsPageUrl();
Program.helper.OntxtviewCompleted(this, new EventControllerArgs()
{
IsSuccess = true, Msg = "开始登陆"
});
foreach (user item in Program.userList)
{
CookieContainer cookie = new CookieContainer();
HttpClient httpClient = new HttpClient("", 0, false, cookie);
Program.helper.OntxtviewCompleted(this, new EventControllerArgs()
{
IsSuccess = true, Msg = item.userName + "登陆"
});
string content = httpClient.GetResponse("", "http://t.cjcyw.com:8081/login", "Post", "pwd=" + item.psw + "&userid=" + item.userName + "");
item.cookie = httpClient.Cookie;
item.cookieContainer = httpClient.cookieContainer;;
}
Control.CheckForIllegalCrossThreadCalls = false;
if (url_comb.Text == "全部")
{
//船源
clsPageUrl.AddPageUrl("ProgramName", "", "", "cyPortal", "", "", "http://t.cjcyw.com:8081/ship/list",
"GET", "", "utf-8", "", null, "", 1, 1);
//货源
clsPageUrl.AddPageUrl("ProgramName", "", "", "hyPortal", "", "", "http://t.cjcyw.com:8081/goods/list",
"GET", "", "utf-8", "", null, "", 1, 1);
for (int i = 1; i <= nmccda.Value; i++)
{
//船舶档案
clsPageUrl.AddPageUrl("ProgramName", "", "", "cydaPortal", "", "", "http://t.cjcyw.com:8081/Boat/BoatList.aspx?pageno=" + i + "&&",
"GET", "", "utf-8", "", null, "", 1, 1);
}
}
else if (url_comb.Text == "船源")
{
//船源
clsPageUrl.AddPageUrl("ProgramName", "", "", "cyPortal", "", "", "http://t.cjcyw.com:8081/ship/list",
"GET", "", "utf-8", "", null, "", 1, 1);
}
else if (url_comb.Text == "货源")
{
//货源
clsPageUrl.AddPageUrl("ProgramName", "", "", "hyPortal", "", "", "http://t.cjcyw.com:8081/goods/list",
"GET", "", "utf-8", "", null, "", 1, 1);
}
else if (url_comb.Text == "船舶档案")
{
for (int i = 1; i <= nmccda.Value; i++)
{
//船舶档案
clsPageUrl.AddPageUrl("ProgramName", "", "", "cydaPortal", "", "", "http://t.cjcyw.com:8081/Boat/BoatList.aspx?pageno=" + i + "&&",
"GET", "", "utf-8", "", null, "", 1, 1);
}
}
}
