protected override Page DowloadContent(Request request, ISpider spider)
{
Site site = spider.Site;
try
{
lock (this)
{
if (_webDriver == null)
{
_webDriver = WebDriverUtil.Open(_browser, _option);
}
if (!_isLogined && SignIn != null)
{
_isLogined = SignIn.Handle(_webDriver as RemoteWebDriver);
if (!_isLogined)
{
throw new SpiderException("Login failed. Please check your login codes.");
}
}
}
//中文乱码URL
Uri uri = request.Url;
string query = string.IsNullOrEmpty(uri.Query) ? "" : $"?{HttpUtility.UrlPathEncode(uri.Query.Substring(1, uri.Query.Length - 1))}";
string realUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}{uri.AbsolutePath}{query}";
var domainUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}";
var options = _webDriver.Manage();
if (options.Cookies.AllCookies.Count == 0 && spider.Site.Cookies.PairPart.Count > 0)
{
_webDriver.Url = domainUrl;
options.Cookies.DeleteAllCookies();
foreach (var c in spider.Site.Cookies.PairPart)
{
options.Cookies.AddCookie(new Cookie(c.Key, c.Value));
}
}
if (UrlHandler != null)
{
realUrl = UrlHandler(realUrl);
}
NetworkCenter.Current.Execute("wd-d", () =>
{
_webDriver.Navigate().GoToUrl(realUrl);
NavigateCompeleted?.Handle((RemoteWebDriver)_webDriver);
});
Thread.Sleep(_webDriverWaitTime);
Page page = new Page(request, spider.Site.ContentType, site.RemoveOutboundLinks ? site.Domain : null)
{
Content = _webDriver.PageSource,
TargetUrl = _webDriver.Url,
Title = _webDriver.Title
};
// 结束后要置空, 这个值存到Redis会导置无限循环跑单个任务
request.PutExtra(Request.CycleTriedTimes, null);
return(page);
}
catch (DownloadException)
{
throw;
}
catch (Exception e)
{
Page page = new Page(request, site.ContentType, null)
{
Exception = e
};
return(page);
}
}