protected override Page DowloadContent(Request request, ISpider spider) { Site site = spider.Site; try { lock (this) { if (_webDriver == null) { _webDriver = WebDriverUtil.Open(_browser, _option); } if (!_isLogined && SignIn != null) { _isLogined = SignIn.Handle(_webDriver as RemoteWebDriver); if (!_isLogined) { throw new SpiderException("Login failed. Please check your login codes."); } } } //中文乱码URL Uri uri = request.Url; string query = string.IsNullOrEmpty(uri.Query) ? "" : $"?{HttpUtility.UrlPathEncode(uri.Query.Substring(1, uri.Query.Length - 1))}"; string realUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}{uri.AbsolutePath}{query}"; var domainUrl = $"{uri.Scheme}://{uri.DnsSafeHost}{(uri.Port == 80 ? "" : ":" + uri.Port)}"; var options = _webDriver.Manage(); if (options.Cookies.AllCookies.Count == 0 && spider.Site.Cookies.PairPart.Count > 0) { _webDriver.Url = domainUrl; options.Cookies.DeleteAllCookies(); foreach (var c in spider.Site.Cookies.PairPart) { options.Cookies.AddCookie(new Cookie(c.Key, c.Value)); } } if (UrlHandler != null) { realUrl = UrlHandler(realUrl); } NetworkCenter.Current.Execute("wd-d", () => { _webDriver.Navigate().GoToUrl(realUrl); NavigateCompeleted?.Handle((RemoteWebDriver)_webDriver); }); Thread.Sleep(_webDriverWaitTime); Page page = new Page(request, spider.Site.ContentType, site.RemoveOutboundLinks ? site.Domain : null) { Content = _webDriver.PageSource, TargetUrl = _webDriver.Url, Title = _webDriver.Title }; // 结束后要置空, 这个值存到Redis会导置无限循环跑单个任务 request.PutExtra(Request.CycleTriedTimes, null); return(page); } catch (DownloadException) { throw; } catch (Exception e) { Page page = new Page(request, site.ContentType, null) { Exception = e }; return(page); } }