private void BeginRestartOnce(string tabName)
{
//允许跳转到查询页面的次数，有时会出现跳转至登录页面的情况
const int allowGoToQueryPageCount = 10;
int goToQueryPageErrorCount = 0;
string currentUrl = "";
IeRunWebBrowser webBrowser = null;
while (currentUrl != _PageUrl && goToQueryPageErrorCount < allowGoToQueryPageCount)
{
//加载网页
webBrowser = this.ShowWebPage(_PageUrl, tabName);
currentUrl = webBrowser.Url.ToString();
}
if (currentUrl != _PageUrl)
{
throw new Exception("无法打开页面.");
}
else
{
this.RunPage.InvokeAppendLogText("准备重新拨号", LogLevelType.Normal, true);
InvokeRestartNow(webBrowser, this._User, this._Password, tabName);
}
}
private IeRunWebBrowser ShowWebPage(string url, string tabName)
{
IeRunWebBrowser webBrowser = (IeRunWebBrowser)this.RunPage.InvokeShowWebPage(url, tabName, WebBrowserType.IE);
int waitCount = 0;
while (!this.RunPage.CheckIsComplete(tabName))
{
if (SysConfig.WebPageRequestInterval * waitCount > WebRequestTimeout)
{
string errorInfo = "打开页面超时! PageUrl = " + url + ". 但是继续执行!";
this.RunPage.InvokeAppendLogText(errorInfo, Base.EnumTypes.LogLevelType.System, true);
break;
//超时
//throw new Exception("打开页面超时. PageUrl = " + url);
}
//等待
waitCount++;
Thread.Sleep(SysConfig.WebPageRequestInterval);
}
this.InvokeAddMyScript(webBrowser);
//再增加个等待，等待异步加载的数据
Thread.Sleep(1000);
return(webBrowser);
}
/// <summary>
/// 获取当前列表页及下一页地址
/// </summary>
/// <param name="listPageUrl"></param>
/// <returns></returns>
private string GetCurrentPageAndNextPageUrl(string listPageUrl, List<string> allListPageUrls)
{
string tabName = "ListPage";
IeRunWebBrowser webBrowser = (IeRunWebBrowser)this.RunPage.ShowWebPage(listPageUrl, tabName, SysConfig.WebPageRequestTimeout, false, WebBrowserType.Chromium);
string currentPageUrl = this.RunPage.InvokeGetWebBrowserPageUrl(webBrowser);
string listPageHtml = this.RunPage.InvokeGetPageHtml(tabName);
string localFilePath = this.RunPage.GetFilePath(currentPageUrl, this.RunPage.GetDetailSourceFileDir());
this.RunPage.SaveFile(listPageHtml, localFilePath, Encoding.UTF8);
allListPageUrls.Add(currentPageUrl);
string scriptMethodCode = "function myGetNextPageUrl(){"
+ "var nextLi = $('li.next');"
+ "if(nextLi.length == 0){"
+ "return '';"
+ "}"
+ "else{"
+ "return $(nextLi[0]).children('a').attr('href');"
+ "}"
+ "}";
this.RunPage.InvokeAddScriptMethod(webBrowser, scriptMethodCode);
string nextPageUrl = CommonUtil.UrlDecodeSymbolAnd((string)this.RunPage.InvokeDoScriptMethod(webBrowser, "myGetNextPageUrl", null));
if (nextPageUrl != null && nextPageUrl.Length > 0)
{
return nextPageUrl;
}
else
{
return null;
}
}
private void GoToManagmentPage(IeRunWebBrowser webBrowser)
{
HtmlElement btnElement = webBrowser.Document.GetElementById("sec_Management");
if (btnElement != null)
{
btnElement.InvokeMember("onclick");
}
}
private void AddCheckPageScript(IeRunWebBrowser webBrowser)
{
HtmlElement sElement = webBrowser.Document.CreateElement("script");
IHTMLScriptElement scriptElement = (IHTMLScriptElement)sElement.DomElement;
scriptElement.text = "function isOldPage(){return 'yes';};";
webBrowser.Document.Body.AppendChild(sElement);
string isOldPage = (string)this.RunPage.InvokeDoScriptMethod(webBrowser, "isOldPage", null);
}
private void Goon(IeRunWebBrowser webBrowser)
{
HtmlElement btnElement = webBrowser.Document.GetElementById("btn_confirm");
if (btnElement != null && btnElement.GetAttribute("value") == "继续")
{
btnElement.InvokeMember("onclick");
}
}
private void GoToMaintenancePage(IeRunWebBrowser webBrowser)
{
HtmlElement btnElement = webBrowser.Document.GetElementById("fst_Maintenance");
if (btnElement != null)
{
btnElement.InvokeMember("onclick");
}
}
private void GoToReboottPage(IeRunWebBrowser webBrowser)
{
HtmlElement btnElement = webBrowser.Document.GetElementById("tab_Reboot");
if (btnElement != null)
{
btnElement.InvokeMember("onclick");
}
}
private string GetIsOldPage(IeRunWebBrowser webBrowser)
{
try
{
string isOldPage = (string)this.RunPage.InvokeDoScriptMethod(webBrowser, "isOldPage", null);
return(isOldPage);
}
catch (Exception ex)
{
return(null);
}
}
private void InvokeRestartNow(IeRunWebBrowser webBrowser, string user, string password, string tabName)
{
webBrowser.Invoke(new LoginInvokeDelegate(Login), new object[] { webBrowser, user, password });
Thread.Sleep(10000);
webBrowser.Invoke(new GoonInvokeDelegate(Goon), new object[] { webBrowser });
Thread.Sleep(10000);
webBrowser.Invoke(new GoToMaintenancePageInvokeDelegate(GoToMaintenancePage), new object[] { webBrowser });
Thread.Sleep(10000);
webBrowser.Invoke(new GoToManagmentPageInvokeDelegate(GoToManagmentPage), new object[] { webBrowser });
Thread.Sleep(10000);
webBrowser.Invoke(new GoToRebootPageInvokeDelegate(GoToReboottPage), new object[] { webBrowser });
Thread.Sleep(10000);
webBrowser.Invoke(new RebootInvokeDelegate(Reboot), new object[] { webBrowser });
}
public static void AutoScroll(IRunWebPage runPage, IeRunWebBrowser webBrowser, int toPosX, int toPosY, int maxStepLength, int minStepSleep, int maxStepSleep)
{
int posX = 0;
int posY = 0;
Random random = new Random(DateTime.Now.Millisecond);
while (posX < toPosX || posY < toPosY)
{
int randomValue = random.Next(maxStepLength);
posX = toPosX <= posX ? toPosX : posX + randomValue;
posY = toPosY <= posY ? toPosY : posY + randomValue;
runPage.InvokeScrollDocumentMethod(webBrowser, new Point(posX, posY));
ProcessThread.SleepRandom(minStepSleep, maxStepSleep);
}
}
private void InputNameScript(IeRunWebBrowser webBrowser, string firstName, string lastName, string country, string state)
{
AddCheckPageScript(webBrowser);
webBrowser.Document.GetElementById("ctl00_content_ctl01_txtbxLastName").SetAttribute("value", lastName);
Thread.Sleep(200);
webBrowser.Document.GetElementById("ctl00_content_ctl01_txtbxFirstName").SetAttribute("value", firstName);
Thread.Sleep(200);
webBrowser.Document.GetElementById("ctl00_content_ctl01_ddlSearchCountry").SetAttribute("value", country);
Thread.Sleep(200);
if (state.Length != 0)
{
webBrowser.Document.GetElementById("ctl00_content_ctl01_ddlSearchState").SetAttribute("value", state);
Thread.Sleep(200);
}
webBrowser.Document.GetElementById("ctl00_content_ctl01_btnSearch").InvokeMember("click");
Thread.Sleep(1000);
}
private void WaitGetPilotInfoPage(IeRunWebBrowser webBrowser)
{
string isOldPage = this.GetIsOldPage(webBrowser);
int waitCount = 0;
while (isOldPage == "yes")
{
if (SysConfig.WebPageRequestInterval * waitCount > WebRequestTimeout)
{
throw new Exception("打开页面超时");
}
//等待
waitCount++;
Thread.Sleep(SysConfig.WebPageRequestInterval);
isOldPage = this.GetIsOldPage(webBrowser);
}
Thread.Sleep(1000);
InvokeAddCheckPageScript(webBrowser);
}
private IeRunWebBrowser ShowWebPage(string url, string tabName)
{
IeRunWebBrowser webBrowser = (IeRunWebBrowser)this.RunPage.InvokeShowWebPage(url, tabName);
int waitCount = 0;
while (!this.RunPage.CheckIsComplete(tabName))
{
if (SysConfig.WebPageRequestInterval * waitCount > WebRequestTimeout)
{
throw new Exception("打开页面超时. PageUrl = " + url);
}
//等待
waitCount++;
Thread.Sleep(SysConfig.WebPageRequestInterval);
}
//再增加个等待，等待异步加载的数据
Thread.Sleep(1000);
return(webBrowser);
}
private void InvokeInputNameScript(IeRunWebBrowser webBrowser, string firstName, string lastName, string country, string state)
{
webBrowser.Invoke(new InputNameScriptInvokeDelegate(InputNameScript), new object[] { webBrowser, firstName, lastName, country, state });
}
private void ClickPilotNameScript(IeRunWebBrowser webBrowser, string linkNodeId)
{
AddCheckPageScript(webBrowser);
webBrowser.Document.GetElementById(linkNodeId).InvokeMember("click");
}
private void InvokeAddCheckPageScript(IeRunWebBrowser webBrowser)
{
webBrowser.Invoke(new AddCheckPageScriptInvokeDelegate(AddCheckPageScript), new object[] { webBrowser });
}
private void Login(IeRunWebBrowser webBrowser, string user, string password)
{
webBrowser.Document.GetElementById("txt_usr_name").SetAttribute("value", user);
webBrowser.Document.GetElementById("txt_password").SetAttribute("value", password);
webBrowser.Document.GetElementById("btn_logon").InvokeMember("onclick");
}
private void AddMyScript(IeRunWebBrowser webBrowser, string p1)
{
webBrowser.ObjectForScripting = this;
}
private void InvokeAddMyScript(IeRunWebBrowser webBrowser)
{
webBrowser.Invoke(new AddMyScriptInvokeDelegate(AddMyScript), new object[] { webBrowser, "" });
}
private void InvokeClickPilotNameScript(IeRunWebBrowser webBrowser, string linkNodeId)
{
webBrowser.Invoke(new ClickPilotNameScriptInvokeDelegate(ClickPilotNameScript), new object[] { webBrowser, linkNodeId });
}
private void BeginGetPilotInfo(string tabName, string pageUrl, string firstName, string lastName, string street1, string country, string med_date, string med_class, string fullName, string state, string city, string localFilePath)
{
string currentUrl = "";
IeRunWebBrowser webBrowser = null;
while (currentUrl != pageUrl)
{
//加载网页
webBrowser = this.ShowWebPage(pageUrl, tabName);
currentUrl = webBrowser.Url.ToString();
}
if (currentUrl != pageUrl)
{
throw new Exception("无法打开页面.");
}
this.InvokeInputNameScript(webBrowser, firstName, lastName, country, state);
this.WaitGetPilotInfoPage(webBrowser);
string pageHtml = this.RunPage.InvokeGetPageHtml(tabName);
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(pageHtml);
HtmlNodeCollection linkNodes = htmlDoc.DocumentNode.SelectNodes("//a[starts-with(@id, 'ctl00_content_ctl01_drAirmenList_ctl')]");
List <String> linkNodeIds = new List <string>();
if (linkNodes == null)
{
throw new CannotFoundException("没有找到对应人员");
//throw new Exception("没有找到对应人员");
}
else
{
foreach (HtmlNode linkNode in linkNodes)
{
string linkNodeId = linkNode.GetAttributeValue("id", "");
linkNodeIds.Add(linkNodeId);
}
}
bool found = false;
foreach (string linkNodeId in linkNodeIds)
{
this.InvokeClickPilotNameScript(webBrowser, linkNodeId);
this.WaitGetPilotInfoPage(webBrowser);
string pilotInfoPageHtml = this.RunPage.InvokeGetPageHtml(tabName);
HtmlAgilityPack.HtmlDocument newHtmlDoc = new HtmlAgilityPack.HtmlDocument();
newHtmlDoc.LoadHtml(pilotInfoPageHtml);
HtmlNode cert_NameNode = newHtmlDoc.DocumentNode.SelectSingleNode("//div[@id='divPersonalInfo']");
//if (cert_NameNode == null || CommonUtil.HtmlDecode(cert_NameNode.InnerText.Trim()).Trim() != fullName)
if (cert_NameNode != null && CommonUtil.HtmlDecode(cert_NameNode.InnerText.Trim()).Trim().ToLower().Replace(" ", " ").Contains(city.ToLower() + " " + state.ToLower() + " "))
{
found = true;
this.RunPage.SaveFile(pilotInfoPageHtml, localFilePath, Encoding.GetEncoding("utf-8"));
break;
}
}
if (!found)
{
throw new CannotFoundException("未找到匹配人. name = " + firstName + " " + lastName);
//throw new Exception("未找到匹配人. name = " + firstName + " " + lastName);
}
}
private void Reboot(IeRunWebBrowser webBrowser)
{
webBrowser.Document.Window.Frames["ifm_func_module"].Document.InvokeScript("showRebootPro");
_RestartSucceed = true;
}
/// <summary>
/// 获取当前列表页及下一页地址
/// </summary>
/// <param name="listPageUrl"></param>
/// <returns></returns>
private bool GetCurrentPageAndNextPageUrl(string seedPageUrl, string keyWords, string listPageUrl, List <string> allListPageUrls)
{
VisitRandomPage();
string localFilePath = this.RunPage.GetFilePath(listPageUrl, this.RunPage.GetDetailSourceFileDir());
if (!File.Exists(localFilePath))
{
string tabName = "ListPage";
IeRunWebBrowser webBrowser = (IeRunWebBrowser)this.RunPage.ShowWebPage(listPageUrl, tabName, SysConfig.WebPageRequestTimeout, false, WebBrowserType.IE);
try
{
this.RunPage.CheckWebBrowserContainsForComplete(webBrowser, new string[] { keyWords }, SysConfig.WebPageRequestTimeout, true);
}
catch (Exception ex)
{
string limitAlert = "计算机网络中存在异常流量";
string errorPageHtml = this.RunPage.InvokeGetPageHtml(tabName);
if (errorPageHtml.Contains(limitAlert))
{
ProcessWebBrowser.ClearWebBrowserTracks();
ProcessWebBrowser.ClearWebBrowserCookie();
this.RunPage.InvokeAppendLogText(limitAlert + ". 正在清理缓存, 并等待重新启动爬取.", LogLevelType.System, true);
throw new Exception("Google" + limitAlert);
}
else
{
throw ex;
}
}
string listPageHtml = this.RunPage.InvokeGetPageHtml(tabName);
ProcessWebBrowser.AutoScroll(this.RunPage, webBrowser, 2000, 1000, 1000, 2000);
this.RunPage.SaveFile(listPageHtml, localFilePath, Encoding.UTF8);
allListPageUrls.Add(listPageUrl);
string scriptMethodCode = "function myGetNextPageUrl(){"
+ "var nextA = document.getElementById('pnnext');"
+ "if(nextA == null){"
+ "return '';"
+ "}"
+ "else{"
+ "return nextA.getAttribute('href');"
+ "}"
+ "}";
this.RunPage.InvokeAddScriptMethod(webBrowser, scriptMethodCode);
string nextPageUrl = CommonUtil.UrlDecodeSymbolAnd((string)this.RunPage.InvokeDoScriptMethod(webBrowser, "myGetNextPageUrl", null));
if (nextPageUrl != null && nextPageUrl.Length > 0)
{
return(true);
}
else
{
return(false);
}
}
else
{
allListPageUrls.Add(listPageUrl);
return(true);
}
}
