/// <inheritdoc /> public override DriverOptions GetOptions() { var options = new PhantomJSOptions(); options.AddAdditionalCapability(CapabilityType.Proxy, DriverProvider.Proxy); return(options); }
public string xpath_crwal(string url, string xpath0) { /// <summary> /// 利用phantomjs.exe来完成爬虫 /// 需要在工具--NuGet 程序包管理器--安装几个包 /// Selenium.PhantomJS.WebDriver和Selenium.WebDriver /// </summary> /// <param name="url">数据的网址</param> /// <param name="xpath0">数据的xpath</param> /// <returns></returns> PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService(); var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"); service.HideCommandPromptWindow = true; // 隐藏dos窗口 var driver1 = new PhantomJSDriver(service, options); driver1.Navigate().GoToUrl(url); ReadOnlyCollection <IWebElement> res = driver1.FindElementsByXPath(xpath0); // 搜索嘛,结果肯定是一个数组 string res_text; if (res.Count != 0) { res_text = res[0].Text; driver1.Quit(); } else { res_text = "NULL"; } return(res_text); }
public static Tuple <string, string> GetPageSource(string url, int waitSecond = 0) { var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36"); options.AddAdditionalCapability("phantomjs.page.settings.loadImages", false); var JsDriver = new PhantomJSDriver(options); JsDriver.ExecutePhantomJS("this.onResourceRequested = function(request, net) {" + " if (request.url.indexOf('google-analytics') !== -1 || request.url.indexOf('.css') !==-1) {" + " net.abort();" + " }" + "};"); JsDriver.Navigate().GoToUrl(url); JsDriver.WaitUntil(d => !d.Url.Contains("duomai.com") && !d.Url.Contains("guangdiu.com") && !d.Url.Contains("haohuola.com") && !d.Url.Contains("zhuayangmao.com") && !d.Url.Contains("union.") && !d.Url.Contains("click.taobao")); var result = JsDriver.PageSource; var lastUrl = JsDriver.Url; JsDriver.Close(); JsDriver.Quit(); return(new Tuple <string, string>(result, lastUrl)); }
public bool ConnectionTest(string login, string password) { var service = PhantomJSDriverService.CreateDefaultService(); service.IgnoreSslErrors = true; service.LoadImages = false; service.ProxyType = "none"; service.SslProtocol = "any"; var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", _phantomJsUserAgent); var phantomCheckConnection = new PhantomJSDriver(service, options); phantomCheckConnection.Navigate().GoToUrl($"{BaseUrl}/accounts/login"); phantomCheckConnection.FindElement(By.Id("input_login_email")).SendKeys(login); phantomCheckConnection.FindElement(By.Id("input_login_password")).SendKeys(password); phantomCheckConnection.FindElement(By.Id("signin_submit")).Click(); Func <IWebDriver, bool> del = ExpectedConditions.UrlContains($"{BaseUrl}/app/"); var result = del.Invoke(phantomCheckConnection); phantomCheckConnection.Quit(); return(result); }
private System.Net.Cookie SetUpPhantomJS(string link) { System.Net.Cookie cookie = new System.Net.Cookie(); // PhantomJS seems to only work with .NET 4.5 or below. PhantomJSOptions options = new PhantomJSOptions(); options.AddAdditionalCapability("IsJavaScriptEnabled", true); PhantomJSDriver driver = new PhantomJSDriver(options); // https://stackoverflow.com/questions/46666084/how-to-deal-with-javascript-when-fetching-http-response-in-c-sharp-using-httpweb // https://riptutorial.com/phantomjs driver.Url = link; driver.Navigate(); var cookies = driver.Manage().Cookies.AllCookies; foreach (var c in cookies) { if (c.Name.Equals("TSPD_101")) { cookie.Name = c.Name; cookie.Domain = c.Domain; cookie.Value = c.Value; cookie.Path = c.Path; cookie.Expires = DateTime.Now.AddHours(6); } } driver.Close(); driver.Dispose(); return(cookie); }
private static PhantomJSOptions GetPhantomJsptions() { PhantomJSOptions option = new PhantomJSOptions(); option.AddAdditionalCapability("handlesAlerts", true); return(option); }
public void TestPhantomJs() { IWebDriver driver = null; try { PhantomJSOptions options = new PhantomJSOptions(); driver = new PhantomJSDriver(); driver.Url = "http://www.softpost.org"; driver.Manage().Window.Maximize(); Console.WriteLine("Title is " + driver.Title); driver.Navigate(); } catch (Exception e) { Console.WriteLine("Exception ....*********" + e.ToString()); } finally { Thread.Sleep(2000); driver.Close(); driver.Quit(); } }
/// <summary> /// 创同步建爬虫 /// </summary> /// <param name="uri">爬虫URL地址</param> /// <param name="proxy">代理服务器</param> /// <returns>网页源代码</returns> public PhantomJSDriver StartSync(Uri uri, Script script, Operation operation) { try { var _service = PhantomJSDriverService.CreateDefaultService(); var _option = new PhantomJSOptions(); var driver = new PhantomJSDriver(_service, _option); var watch = DateTime.Now; driver.Navigate().GoToUrl(uri.ToString()); if (script != null) { ExecuteScript(script, driver); } if (operation != null) { ExecuteAction(operation, driver); } var threadId = System.Threading.Thread.CurrentThread.ManagedThreadId; var milliseconds = DateTime.Now.Subtract(watch).Milliseconds; //var pageSource = driver.PageSource; return(driver); } catch (Exception ex) { throw ex; } }
public IpModuleManager(IpModule module) { string[] phantomArgs = new string[] { "--webdriver-loglevel=NONE" }; PhantomJSOptions options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.cli.args", phantomArgs); Module = module; var driverService = PhantomJSDriverService.CreateDefaultService(); driverService.HideCommandPromptWindow = true; driverService.LogFile = "phantomJSService.log"; driver = new PhantomJSDriver(driverService, options); driver.Manage().Timeouts().ImplicitlyWait(TimeSpan.FromSeconds(10)); loginUrl = string.Format("{0}/login_page.html", Module.Url.Value.ToString()); logoutUrl = string.Format("{0}/logout.html", Module.Url.Value.ToString()); versionUrl = string.Format("{0}/version.html", Module.Url.Value.ToString()); eventUrl = string.Format("{0}/event.html", Module.Url.Value.ToString()); statusLiveUrl = string.Format("{0}/statuslive.html", Module.Url.Value.ToString()); Devices = new List <Device>(); }
public IWebDriver createNewJSDriver(C_Proxy proxy = null) { IWebDriver _driver; var driverService = PhantomJSDriverService.CreateDefaultService(); driverService.HideCommandPromptWindow = true; driverService.LoadImages = false; //reduce ram usage if (proxy != null) { if (proxy.auth) { driverService.ProxyType = "socks5"; driverService.ProxyAuthentication = String.Format("{0}:{1}", proxy.username, proxy.password); } else { driverService.ProxyType = "http"; } driverService.Proxy = proxy.address; driverService.IgnoreSslErrors = true; } var driverOptions = new PhantomJSOptions(); driverOptions.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"); _driver = new PhantomJSDriver(driverService, driverOptions); return(_driver); }
public Crawler(string proxy = null) { this._chromeOptions = new ChromeOptions(); this._chromeDriverService = ChromeDriverService.CreateDefaultService(); _chromeDriverService.HideCommandPromptWindow = true; //_chromeOptions.AddArgument("headless"); this._options = new PhantomJSOptions(); //定义PhantomJS的参数配置对象 this._service = PhantomJSDriverService.CreateDefaultService(Environment.CurrentDirectory); //初始化Selenium配置,传入存放phantomjs.exe文件的目录 _service.IgnoreSslErrors = true; //忽略证书错误 _service.SslProtocol = "any"; _service.WebSecurity = false; //禁用网页安全 _service.HideCommandPromptWindow = true; //隐藏弹出窗口 _service.LoadImages = true; //禁止加载图片 _service.LocalToRemoteUrlAccess = true; //允许使用本地资源响应远程 URL _options.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"); if (proxy != null) { _service.ProxyType = "HTTP"; //使用HTTP代理 _service.Proxy = proxy; //代理IP及端口 } else { _service.ProxyType = "none";//不使用代理 } }
/// <summary> /// Método responsável por realizar o login no linkedin /// </summary> /// <returns>PhantomJSDriver logado</returns> private static PhantomJSDriver RealizaLogin() { var options = new PhantomJSOptions(); options.AddAdditionalCapability("permissions.default.stylesheet", 2); options.AddAdditionalCapability("permissions.default.image", 2); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/35.0"); var loPhantom = new PhantomJSDriver(options); var time = 20; loPhantom.Manage().Window.Size = new Size(2000, 2000); loPhantom.Navigate().GoToUrl("https://www.linkedin.com/uas/login?goback=&trk=hb_signin"); var loginForm = loPhantom.FindElementByName("session_key"); var passForm = loPhantom.FindElementByName("session_password"); var loButton = loPhantom.FindElementByName("signin"); loginForm.Clear(); passForm.Clear(); loginForm.SendKeys(""); //Login passForm.SendKeys(""); //Senha loButton.Click(); var wait = new WebDriverWait(loPhantom, TimeSpan.FromSeconds(time)); wait.Until(drv => drv.Url.Contains("feed")); return(loPhantom); }
public override DriverWrapper CreateDriver(string agentString) { DriverWrapper driver = null; var sCaps = new DesiredCapabilities(); try { if (string.IsNullOrEmpty(agentString) == false) { var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", agentString); driver = new DriverWrapper(new PhantomJSDriver(options)); } else { driver = new DriverWrapper(new ChromeDriver()); } } catch (Exception e) { log2.Error("Exception: " + e); } return(driver); }
public RemoteWebDriver RegisterNewDriver(AccountViewModel account) { var options = new PhantomJSOptions(); var userAgent = new GetUserAgentQueryHandler(new DataBaseContext()).Handle(new GetUserAgentQuery { UserAgentId = account.UserAgentId }); if (string.IsNullOrWhiteSpace(account.Proxy)) { options.AddAdditionalCapability("phantomjs.page.settings.userAgent", userAgent); return(new PhantomJSDriver(options)); } var service = PhantomJSDriverService.CreateDefaultService(); service.AddArgument(string.Format("--proxy-auth={0}:{1}", account.ProxyLogin, account.ProxyPassword)); service.AddArgument(string.Format("--proxy={0}", account.Proxy)); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", userAgent); var driver = new PhantomJSDriver(service, options); return(driver); }
private static PhantomJSOptions GetPhantomJsOptions() { PhantomJSOptions options = new PhantomJSOptions(); options.AddAdditionalCapability("takesScreenshot", false); return(options); }
public PhantomJSDriverHelper(string cookies, string domain) { var _option = new PhantomJSOptions(); _option.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"); var _service = PhantomJSDriverService.CreateDefaultService(); //代理 _service.ProxyType = "none"; //'http', 'socks5' or 'none' _service.Proxy = ""; //{address}:{port} //不加载图片 _service.LoadImages = false; //忽略SSL证书错误 _service.IgnoreSslErrors = true; //隐藏命令提示窗口 _service.HideCommandPromptWindow = true; //日志文件路径 _service.LogFile = ""; //Cookie文件路径 //_service.CookiesFile = "PhantomJSCookie.json"; //_service.DiskCache = true; //_service.MaxDiskCacheSize = 10240; //_service.LocalStoragePath = "LocalStoragePath"; _webDriver = new PhantomJSDriver(_service, _option); if (!string.IsNullOrEmpty(cookies)) { AddCookie(cookies, domain); } }
/// <summary> /// GetCookiesByUrl /// </summary> /// <param name="url"></param> /// <returns></returns> public static string GetCookiesByUrl(string url) { var userAgent = @"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.108 Safari/537.36 2345Explorer/8.6.2.15784"; var options = new PhantomJSOptions(); options.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", userAgent); using (var driver = new PhantomJSDriver(options)) { var navigation = driver.Navigate(); navigation.GoToUrl(url); var cookies = driver.Manage().Cookies.AllCookies; string cookiesString = ""; bool isFirst = true; foreach (var cookie in cookies) { if (isFirst) { cookiesString += HttpHelper.GetFormatCookies(cookie.ToString()); isFirst = false; } else { cookiesString = $"{cookiesString};{HttpHelper.GetFormatCookies(cookie.ToString())}"; } } Console.WriteLine($"cookies:{cookiesString}"); return(cookiesString); } }
public IWebDriver createDriver(ICapabilities capabilities) { /** * Transform capabilities to Dictionary<string,object> and place it * reflectively as member of ChromeOptions. * * Sucks, but everything here is immutable, non enumerable, etc */ DesiredCapabilities dc = (DesiredCapabilities)capabilities; FieldInfo[] fields = dc.GetType().GetFields(BindingFlags.NonPublic | BindingFlags.Instance); Dictionary <string, object> values = (Dictionary <string, object>)fields[0].GetValue(dc); PhantomJSOptions options = new PhantomJSOptions(); foreach (KeyValuePair <string, object> capability in values) { options.AddAdditionalCapability(capability.Key.ToString(), (capability.Value == null ? "" : capability.Value.ToString())); } string executable = AppDomain.CurrentDomain.BaseDirectory + @"\libs"; PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService(executable); IWebDriver driver = (IWebDriver) new PhantomJSDriver(service, options); return(driver); }
public void MyTest(string chromePath = null) { var path = chromePath ?? System.Environment.CurrentDirectory; var options = new PhantomJSOptions(); using (var webDriver = new PhantomJSDriver(path, options) { Url = "https://stackoverflow.com/users/login" }) { Wait(webDriver, "Log In - Stack Overflow"); Assert.AreEqual("Log In - Stack Overflow", webDriver.Title); var loginPageObject = new LoginPageObject(); PageFactory.InitElements(webDriver, loginPageObject); var username = ConfigurationManager.AppSettings["Username"]; var password = ConfigurationManager.AppSettings["Password"]; loginPageObject.Login(username, password); Wait(webDriver, "Stack Overflow - Where Developers Learn, Share, & Build Careers"); Assert.AreEqual("Stack Overflow - Where Developers Learn, Share, & Build Careers", webDriver.Title); var questionList = webDriver.FindElements(By.CssSelector(".question-summary.narrow.tagged-interesting")); Assert.IsTrue(questionList.Count > 1); } }
public BrowserHelper() { _defaultService = PhantomJSDriverService.CreateDefaultService(CoreConfiguration.WebDriversLocation); _defaultService.HideCommandPromptWindow = true; _options = new PhantomJSOptions(); _options.AddAdditionalCapability("phantomjs.page.settings.userAgent", @"Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0"); }
private void UsePhatomJS() { PhantomJSOptions options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25"); _driver = new PhantomJSDriver(options); }
private static PhantomJSOptions GetPhantomJsptions() { var option = new PhantomJSOptions(); option.AddAdditionalCapability("handlesAlerts", true); Logger.Info("Using PhantomJSOptions"); return(option); }
private IWebDriver CreatePhantomJsDriver() { var options = new PhantomJSOptions(); { }; return(new PhantomJSDriver(options)); }
private static RemoteWebDriver createPhantomJSDriver(string seleniumHub) { RemoteWebDriver driver; PhantomJSOptions opt = new PhantomJSOptions(); driver = new RemoteWebDriver(new Uri(seleniumHub), opt.ToCapabilities(), TimeSpan.FromSeconds(600)); return(driver); }
/// <summary> /// Получить драйвер IWebDriver по имени /// </summary> /// <param name="driverName"></param> /// <returns></returns> public IWebDriver GetDriver(string driverName) { IWebDriver driver = null; switch (driverName) { case "Firefox": FirefoxProfile firefoxProfile = new FirefoxProfile(); firefoxProfile.SetPreference("browser.download.folderList", 2); firefoxProfile.SetPreference("browser.download.dir", GetDownloadPath()); firefoxProfile.SetPreference("browser.helperApps.neverAsk.saveToDisk", "application/xml"); firefoxProfile.SetPreference("browser.helperApps.alwaysAsk.force", false); driver = new FirefoxDriver(firefoxProfile); break; case "Chrome": ChromeOptions chromeOptions = new ChromeOptions(); chromeOptions.AddUserProfilePreference("download.default_directory", GetDownloadPath()); chromeOptions.AddUserProfilePreference("intl.accept_languages", "nl"); chromeOptions.AddUserProfilePreference("disable-popup-blocking", "true"); chromeOptions.AddUserProfilePreference("download.prompt_for_download", false); chromeOptions.AddUserProfilePreference("download.directory_upgrade", true); chromeOptions.AddUserProfilePreference("safebrowsing.enabled", true); driver = new ChromeDriver(chromeOptions); break; case "IE": var options = new InternetExplorerOptions { EnsureCleanSession = true, IgnoreZoomLevel = true, IntroduceInstabilityByIgnoringProtectedModeSettings = true, }; driver = new InternetExplorerDriver(options); break; case "PhantomJS": var phantomJsOptions = new PhantomJSOptions(); phantomJsOptions.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36"); var driverService = PhantomJSDriverService.CreateDefaultService(); driverService.IgnoreSslErrors = true; driverService.SslProtocol = "any"; driver = new PhantomJSDriver(driverService, phantomJsOptions); break; } if (driver != null) { driver.Manage().Cookies.DeleteAllCookies(); driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(30); driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(3); driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromMilliseconds(200); } return(driver); }
public StrongCrawler() { _options = new PhantomJSOptions(); _options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36"); _service = PhantomJSDriverService.CreateDefaultService(Environment.CurrentDirectory); //不加载图片 _service.AddArgument("--load-images=false"); _service.LoadImages = false; }
//PhantomjsDriverPath is where the phantomJS driver is. //PhantomjsDriverName is the name of the PhantomJS driver with its' extension (ex: .exe) public static IWebDriver InvokePhantomJS(String PhantomjsDriverPath = @"C:\Selenium\Browsers drivers\", String PhantomjsDriverName = "phantomjs.exe") { PhantomJSDriverService phantomJSDriverService = PhantomJSDriverService.CreateDefaultService(PhantomjsDriverPath, "phantomjs.exe"); PhantomJSOptions phantomJSOptions = new PhantomJSOptions(); InbrowserObject = new PhantomJSDriver(phantomJSDriverService, phantomJSOptions); return(InbrowserObject); }
public IWebDriver Create() { IWebDriver driver; Func <string, string, string> c = (x, y) => x.Replace(x, y); var currentDay = "Friday"; var horribleDay = "Monday"; var differentDay = c(currentDay, "horribleDay"); var driverToUse = ConfigurationHelper.Get <DriverToUse>("DriverToUse"); var useGrid = ConfigurationHelper.Get <bool>("UseGrid"); if (useGrid) { driver = CreateGridDriver(driverToUse); } else { switch (driverToUse) { case DriverToUse.InternetExplorer: driver = new InternetExplorerDriver(AppDomain.CurrentDomain.BaseDirectory, new InternetExplorerOptions(), TimeSpan.FromMinutes(5)); break; case DriverToUse.Firefox: var firefoxProfile = FirefoxProfile; driver = new FirefoxDriver(firefoxProfile); driver.Manage().Window.Maximize(); break; case DriverToUse.Chrome: driver = new ChromeDriver(); break; case DriverToUse.PhantomJS: driver = new PhantomJSDriver(); PhantomJSOptions o = new PhantomJSOptions(); break; default: throw new ArgumentOutOfRangeException(); } } driver.Manage().Window.Maximize(); var timeouts = driver.Manage().Timeouts(); timeouts.ImplicitlyWait(TimeSpan.FromSeconds(ConfigurationHelper.Get <int>("ImplicitlyWait"))); timeouts.SetPageLoadTimeout(TimeSpan.FromSeconds(ConfigurationHelper.Get <int>("PageLoadTimeout"))); // Suppress the onbeforeunload event first. This prevents the application hanging on a dialog box that does not close. //((IJavaScriptExecutor)driver).ExecuteScript("window.onbeforeunload = function(e){};"); return(driver); }
private void RegisterCustomPhantomBrowser() { var phantomLan = Lan + "," + Lan + ";q=0.5"; var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.customHeaders.Accept-Language", phantomLan); var customPhantomDriver = new CustomPhantomJsSeleniumDriver(options); _browserSession = new BrowserSession(customPhantomDriver); _objectContainer.RegisterInstanceAs(_browserSession); }
public void SetUp() { DriverConfiguration configuration = new DriverConfiguration(); var options = new PhantomJSOptions(); options.AddAdditionalCapability("phantomjs.page.settings.UserAgent", "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X;en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4Mobile/7B334b Safari/531.21.10 "); var services = PhantomJSDriverService.CreateDefaultService(); services.LogFile = "D:\\AutomationReport\\Log.txt"; configuration.DriverServices = services; configuration.DesiredCapabilities = options; DriverManager.StartDriver(Browser.PhantomJSBrowser,configuration); }