public override bool AfterAllGrab(IListSheet listSheet)
{
this.GetSeedInfoFromListSheet(listSheet);
//下一步必须执行
bool isNewDo = false;
string localLogFileName = null;
List<string> allListPageUrls = null;
localLogFileName = this.LoginName + "_" + this.KeyWords + "_listPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce)
{
//如果是生产环境，那么直接爬取列表页
allListPageUrls = this.GetAllListPages(this.Parameters, this.SeedPageUrl);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
else
{
//读取历史爬取的列表页地址文件
allListPageUrls = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, "listPageUrl");
if (allListPageUrls == null)
{
allListPageUrls = this.GetAllListPages(this.Parameters, this.SeedPageUrl);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
}
List<Dictionary<string, string>> allPersonPageUrlInfos = null;
localLogFileName = this.LoginName + "_" + this.KeyWords + "_personPageUrlInfo";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接解析列表页
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
}
else
{
//读取历史解析获得的个人网页地址
allPersonPageUrlInfos = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, new string[] { "personUrl", "personName" });
if (allPersonPageUrlInfos == null)
{
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
isNewDo = true;
}
}
List<string> allPersonPageUrls = null;
localLogFileName = this.LoginName + "_" + this.KeyWords + "_personPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接爬取个人详情页
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.LoginName, this.LoginPassword);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
else
{
//读取历史生成的个人网页网址
allPersonPageUrls = this.RunPage.TryGetInfoFromMiddleFile(this.LoginName + "." + this.KeyWords + ".personPageUrl", "personUrl");
if (allPersonPageUrls == null)
{
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.LoginName, this.LoginPassword);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
}
List<Dictionary<string, string>> personInfoList = ProcessPersonPage.GetPersonInfoFromLocalPages(this.RunPage, allPersonPageUrls, false, null);
string personInfosFilePath = this.RunPage.GetFilePath("SearchResult_Linkedin2Linkedin_" + this.LoginName + "_" + this.KeyWords + ".xlsx", this.RunPage.GetExportDir());
ProcessPersonPage.SavePersonInfoToFile(this.RunPage, personInfoList, personInfosFilePath);
return true;
}
private void GetOneKeyWordsRelatedInfos(Dictionary <string, string> seedRow)
{
//下一步必须执行
bool isNewDo = false;
string localLogFileName = null;
string keyWords = this.GetKeyWords(seedRow);
List <string> allListPageUrls = null;
localLogFileName = "_" + this.GetLoginName(seedRow) + "_" + this.GetKeyWords(seedRow) + "_listPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce)
{
//如果是生产环境，那么直接爬取列表页
allListPageUrls = this.GetAllListPages(this.GetSeedPageUrl(seedRow), keyWords);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
else
{
//读取历史爬取的列表页地址文件
allListPageUrls = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, "listPageUrl");
if (allListPageUrls == null)
{
allListPageUrls = this.GetAllListPages(this.GetSeedPageUrl(seedRow), keyWords);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
}
List <Dictionary <string, string> > allPersonPageUrlInfos = null;
localLogFileName = "_" + this.GetLoginName(seedRow) + "_" + this.GetKeyWords(seedRow) + "_personPageUrlInfo";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接解析列表页
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
}
else
{
//读取历史解析获得的个人网页地址
allPersonPageUrlInfos = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, new string[] { "personUrl", "personName" });
if (allPersonPageUrlInfos == null)
{
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls);
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
isNewDo = true;
}
}
LoginLinkedin.LoginByRandomUser(this.RunPage, this.LinkedinLoginPageUrl, this.LinkedinLoginSucceedCheckUrl);
List <string> allPersonPageUrls = null;
localLogFileName = "_" + this.GetLoginName(seedRow) + "_" + this.GetKeyWords(seedRow) + "_personPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接爬取个人详情页
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.GetLoginName(seedRow), this.GetLoginPassword(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
else
{
//读取历史生成的个人网页网址
allPersonPageUrls = this.RunPage.TryGetInfoFromMiddleFile(this.GetLoginName(seedRow) + "." + this.GetKeyWords(seedRow) + ".personPageUrl", "personUrl");
if (allPersonPageUrls == null)
{
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.GetLoginName(seedRow), this.GetLoginPassword(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
}
List <Dictionary <string, string> > personInfoList = ProcessPersonPage.GetPersonInfoFromLocalPages(this.RunPage, allPersonPageUrls, true, this.GetKeyWords(seedRow));
string personInfosFilePath = this.RunPage.GetFilePath("SearchResult_GoogleHttpRequest2Linkedin_" + this.GetLoginName(seedRow) + "_" + this.GetKeyWords(seedRow) + ".xlsx", this.RunPage.GetExportDir());
ProcessPersonPage.SavePersonInfoToFile(this.RunPage, personInfoList, personInfosFilePath);
}
public override void AfterGrabOne(string pageUrl, Dictionary <string, string> seedRow, bool needReGrab, bool existLocalFile)
{
try
{
this.GetPageFromBaidu(pageUrl, this.GetKeyWords(seedRow));
//下一步必须执行
bool isNewDo = false;
string localLogFileName = null;
List <string> allListPageUrls = null;
localLogFileName = "_" + this.GetKeyWords(seedRow) + "_listPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce)
{
//如果是生产环境，那么直接爬取列表页
allListPageUrls = this.GetAllListPages(this.GetSeedPageUrl(seedRow), this.GetKeyWords(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
else
{
//读取历史爬取的列表页地址文件
allListPageUrls = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, "listPageUrl");
if (allListPageUrls == null)
{
allListPageUrls = this.GetAllListPages(this.GetSeedPageUrl(seedRow), this.GetKeyWords(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "listPageUrl", allListPageUrls);
isNewDo = true;
}
}
List <Dictionary <string, string> > allPersonPageUrlInfos = null;
localLogFileName = "_" + this.GetKeyWords(seedRow) + "_personPageUrlInfo";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接解析列表页
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls, this.GetKeyWords(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
}
else
{
//读取历史解析获得的个人网页地址
allPersonPageUrlInfos = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, new string[] { "personUrl", "personName" });
if (allPersonPageUrlInfos == null)
{
allPersonPageUrlInfos = this.GetPersonPageUrlsFromListPages(this.RunPage.GetDetailSourceFileDir(), allListPageUrls, this.GetKeyWords(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, new string[] { "personUrl", "personName" }, allPersonPageUrlInfos);
isNewDo = true;
}
}
LoginLinkedin.LoginByRandomUser(this.RunPage, this.LinkedinLoginPageUrl, this.LinkedinLoginSucceedCheckUrl);
List <string> allPersonPageUrls = null;
localLogFileName = "_" + this.GetKeyWords(seedRow) + "_personPageUrl";
if (SysConfig.SysExecuteType == SysExecuteType.Produce || isNewDo)
{
//如果是生产环境，那么直接爬取个人详情页
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.GetLoginName(seedRow), this.GetLoginPassword(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
else
{
//读取历史生成的个人网页网址
allPersonPageUrls = this.RunPage.TryGetInfoFromMiddleFile(localLogFileName, "personUrl");
if (allPersonPageUrls == null)
{
allPersonPageUrls = ProcessPersonPage.GetAllPersonPageUrls(this.RunPage, allPersonPageUrlInfos, this.GetLoginName(seedRow), this.GetLoginPassword(seedRow));
this.RunPage.SaveInfoToMiddleFile(localLogFileName, "personUrl", allPersonPageUrls);
isNewDo = true;
}
}
List <Dictionary <string, string> > personInfoList = ProcessPersonPage.GetPersonInfoFromLocalPages(this.RunPage, allPersonPageUrls, true, this.GetKeyWords(seedRow));
string personInfosResultFilePath = this.RunPage.GetFilePath("_SearchResult_Baidu2Linkedin_" + this.GetKeyWords(seedRow) + ".xlsx", this.RunPage.GetExportDir());
ProcessPersonPage.SavePersonInfoToFile(this.RunPage, personInfoList, personInfosResultFilePath);
}
catch (Exception ex)
{
throw ex;
}
}