public void ReturnToPool(WebDriverItem webDriver)
{
CheckRunning();
if (_webDriverList.Contains(webDriver))
{
_innerQueue.Enqueue(webDriver);
}
}
public void Close(WebDriverItem webDriver)
{
try
{
if (_webDriverList.Contains(webDriver))
{
_webDriverList.TryTake(out webDriver);
webDriver.WebDriver.Quit();
}
}
catch (Exception)
{
// ignored
}
}
public void Close(WebDriverItem webDriver)
{
try
{
if (_webDriverList.Contains(webDriver))
{
_webDriverList.TryTake(out webDriver);
webDriver.WebDriver.Quit();
}
}
catch (Exception)
{
// ignored
}
}
public override bool Equals(object o)
{
if (this == o)
{
return(true);
}
if (o == null || GetType() != o.GetType())
{
return(false);
}
WebDriverItem request = (WebDriverItem)o;
if (!WebDriver.Equals(request.WebDriver))
{
return(false);
}
return(true);
}
public override Page Download(Request request, ISpider spider)
{
WebDriverItem driverService = null;
try
{
driverService = Pool.Get();
lock (this)
{
if (!_isLogined && Login != null)
{
_isLogined = Login.Invoke(driverService.WebDriver as RemoteWebDriver);
if (!_isLogined)
{
throw new SpiderExceptoin("Login failed. Please check your login codes.");
}
}
}
//Logger.Info("Downloading page " + request.Url);
//中文乱码URL
Uri uri = request.Url;
string query = uri.Query;
string realUrl = uri.Scheme + "://" + uri.DnsSafeHost + (uri.Port == 80 ? "" : (":" + uri.Port)) + uri.AbsolutePath + (string.IsNullOrEmpty(query)
? ""
: ("?" + HttpUtility.UrlPathEncode(uri.Query.Substring(1, uri.Query.Length - 1))));
if (UrlFormat != null)
{
realUrl = UrlFormat(realUrl);
}
RedialManagerUtils.Execute("webdriverdownloader-download", () =>
{
driverService.WebDriver.Navigate().GoToUrl(realUrl);
});
Thread.Sleep(_webDriverWaitTime);
AfterNavigate?.Invoke((RemoteWebDriver)driverService.WebDriver);
Page page = new Page(request, spider.Site.ContentType);
page.Content = driverService.WebDriver.PageSource;
page.Url = request.Url.ToString();
page.TargetUrl = driverService.WebDriver.Url;
page.Title = driverService.WebDriver.Title;
ValidatePage(page);
// 结束后要置空, 这个值存到Redis会导置无限循环跑单个任务
request.PutExtra(Request.CycleTriedTimes, null);
return(page);
}
finally
{
Pool.ReturnToPool(driverService);
}
}
public void ReturnToPool(WebDriverItem webDriver)
{
CheckRunning();
if (_webDriverList.Contains(webDriver))
{
_innerQueue.Enqueue(webDriver);
}
}
