public RemoteWebDriver RegisterNewDriver(AccountViewModel account)
{
var options = new PhantomJSOptions();
var userAgent = new GetUserAgentQueryHandler(new DataBaseContext()).Handle(new GetUserAgentQuery
{
UserAgentId = account.UserAgentId
});
if (string.IsNullOrWhiteSpace(account.Proxy))
{
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", userAgent);
return(new PhantomJSDriver(options));
}
var service = PhantomJSDriverService.CreateDefaultService();
service.AddArgument(string.Format("--proxy-auth={0}:{1}", account.ProxyLogin, account.ProxyPassword));
service.AddArgument(string.Format("--proxy={0}", account.Proxy));
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", userAgent);
var driver = new PhantomJSDriver(service, options);
return(driver);
}
/// <summary>
/// Método responsável por realizar o login no linkedin
/// </summary>
/// <returns>PhantomJSDriver logado</returns>
private static PhantomJSDriver RealizaLogin()
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("permissions.default.stylesheet", 2);
options.AddAdditionalCapability("permissions.default.image", 2);
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/35.0");
var loPhantom = new PhantomJSDriver(options);
var time = 20;
loPhantom.Manage().Window.Size = new Size(2000, 2000);
loPhantom.Navigate().GoToUrl("https://www.linkedin.com/uas/login?goback=&trk=hb_signin");
var loginForm = loPhantom.FindElementByName("session_key");
var passForm = loPhantom.FindElementByName("session_password");
var loButton = loPhantom.FindElementByName("signin");
loginForm.Clear();
passForm.Clear();
loginForm.SendKeys(""); //Login
passForm.SendKeys(""); //Senha
loButton.Click();
var wait = new WebDriverWait(loPhantom, TimeSpan.FromSeconds(time));
wait.Until(drv => drv.Url.Contains("feed"));
return(loPhantom);
}
public static Tuple <string, string> GetPageSource(string url, int waitSecond = 0)
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
options.AddAdditionalCapability("phantomjs.page.settings.loadImages", false);
var JsDriver = new PhantomJSDriver(options);
JsDriver.ExecutePhantomJS("this.onResourceRequested = function(request, net) {" +
" if (request.url.indexOf('google-analytics') !== -1 || request.url.indexOf('.css') !==-1) {" +
" net.abort();" +
" }" +
"};");
JsDriver.Navigate().GoToUrl(url);
JsDriver.WaitUntil(d => !d.Url.Contains("duomai.com") &&
!d.Url.Contains("guangdiu.com") &&
!d.Url.Contains("haohuola.com") &&
!d.Url.Contains("zhuayangmao.com") &&
!d.Url.Contains("union.") &&
!d.Url.Contains("click.taobao"));
var result = JsDriver.PageSource;
var lastUrl = JsDriver.Url;
JsDriver.Close();
JsDriver.Quit();
return(new Tuple <string, string>(result, lastUrl));
}
public Driver()
{
writer = new DbWriter();
service = PhantomJSDriverService.CreateDefaultService();
service.HideCommandPromptWindow = true;
opt = new PhantomJSOptions();
opt.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome / 40.0.2214.94 Safari / 537.36");
opt.AddAdditionalCapability("IsJavaScriptEnabled", true);
//opt.AddAdditionalCapability("IsApplicationCacheEnabled", true);
driver = new PhantomJSDriver(service, opt);
exec = (IJavaScriptExecutor)driver;
driver.Navigate().GoToUrl("http://www.oddsportal.com");
login();
}
public IWebDriver createDriver(ICapabilities capabilities)
{
/**
* Transform capabilities to Dictionary<string,object> and place it
* reflectively as member of ChromeOptions.
*
* Sucks, but everything here is immutable, non enumerable, etc
*/
DesiredCapabilities dc = (DesiredCapabilities)capabilities;
FieldInfo[] fields = dc.GetType().GetFields(BindingFlags.NonPublic | BindingFlags.Instance);
Dictionary <string, object> values = (Dictionary <string, object>)fields[0].GetValue(dc);
PhantomJSOptions options = new PhantomJSOptions();
foreach (KeyValuePair <string, object> capability in values)
{
options.AddAdditionalCapability(capability.Key.ToString(), (capability.Value == null ? "" : capability.Value.ToString()));
}
string executable = AppDomain.CurrentDomain.BaseDirectory + @"\libs";
PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService(executable);
IWebDriver driver = (IWebDriver) new PhantomJSDriver(service, options);
return(driver);
}
/// <summary>
/// GetCookiesByUrl
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetCookiesByUrl(string url)
{
var userAgent =
@"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.108 Safari/537.36 2345Explorer/8.6.2.15784";
var options = new PhantomJSOptions();
options.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", userAgent);
using (var driver = new PhantomJSDriver(options))
{
var navigation = driver.Navigate();
navigation.GoToUrl(url);
var cookies = driver.Manage().Cookies.AllCookies;
string cookiesString = "";
bool isFirst = true;
foreach (var cookie in cookies)
{
if (isFirst)
{
cookiesString += HttpHelper.GetFormatCookies(cookie.ToString());
isFirst = false;
}
else
{
cookiesString = $"{cookiesString};{HttpHelper.GetFormatCookies(cookie.ToString())}";
}
}
Console.WriteLine($"cookies:{cookiesString}");
return(cookiesString);
}
}
public bool ConnectionTest(string login, string password)
{
var service = PhantomJSDriverService.CreateDefaultService();
service.IgnoreSslErrors = true;
service.LoadImages = false;
service.ProxyType = "none";
service.SslProtocol = "any";
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", _phantomJsUserAgent);
var phantomCheckConnection = new PhantomJSDriver(service, options);
phantomCheckConnection.Navigate().GoToUrl($"{BaseUrl}/accounts/login");
phantomCheckConnection.FindElement(By.Id("input_login_email")).SendKeys(login);
phantomCheckConnection.FindElement(By.Id("input_login_password")).SendKeys(password);
phantomCheckConnection.FindElement(By.Id("signin_submit")).Click();
Func <IWebDriver, bool> del = ExpectedConditions.UrlContains($"{BaseUrl}/app/");
var result = del.Invoke(phantomCheckConnection);
phantomCheckConnection.Quit();
return(result);
}
public Crawler(string proxy = null)
{
this._chromeOptions = new ChromeOptions();
this._chromeDriverService = ChromeDriverService.CreateDefaultService();
_chromeDriverService.HideCommandPromptWindow = true;
//_chromeOptions.AddArgument("headless");
this._options = new PhantomJSOptions(); //定义PhantomJS的参数配置对象
this._service = PhantomJSDriverService.CreateDefaultService(Environment.CurrentDirectory); //初始化Selenium配置，传入存放phantomjs.exe文件的目录
_service.IgnoreSslErrors = true; //忽略证书错误
_service.SslProtocol = "any";
_service.WebSecurity = false; //禁用网页安全
_service.HideCommandPromptWindow = true; //隐藏弹出窗口
_service.LoadImages = true; //禁止加载图片
_service.LocalToRemoteUrlAccess = true; //允许使用本地资源响应远程 URL
_options.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36");
if (proxy != null)
{
_service.ProxyType = "HTTP"; //使用HTTP代理
_service.Proxy = proxy; //代理IP及端口
}
else
{
_service.ProxyType = "none";//不使用代理
}
}
public IWebDriver createNewJSDriver(C_Proxy proxy = null)
{
IWebDriver _driver;
var driverService = PhantomJSDriverService.CreateDefaultService();
driverService.HideCommandPromptWindow = true;
driverService.LoadImages = false; //reduce ram usage
if (proxy != null)
{
if (proxy.auth)
{
driverService.ProxyType = "socks5";
driverService.ProxyAuthentication = String.Format("{0}:{1}", proxy.username, proxy.password);
}
else
{
driverService.ProxyType = "http";
}
driverService.Proxy = proxy.address;
driverService.IgnoreSslErrors = true;
}
var driverOptions = new PhantomJSOptions();
driverOptions.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
_driver = new PhantomJSDriver(driverService, driverOptions);
return(_driver);
}
private System.Net.Cookie SetUpPhantomJS(string link)
{
System.Net.Cookie cookie = new System.Net.Cookie();
// PhantomJS seems to only work with .NET 4.5 or below.
PhantomJSOptions options = new PhantomJSOptions();
options.AddAdditionalCapability("IsJavaScriptEnabled", true);
PhantomJSDriver driver = new PhantomJSDriver(options);
// https://stackoverflow.com/questions/46666084/how-to-deal-with-javascript-when-fetching-http-response-in-c-sharp-using-httpweb
// https://riptutorial.com/phantomjs
driver.Url = link;
driver.Navigate();
var cookies = driver.Manage().Cookies.AllCookies;
foreach (var c in cookies)
{
if (c.Name.Equals("TSPD_101"))
{
cookie.Name = c.Name;
cookie.Domain = c.Domain;
cookie.Value = c.Value;
cookie.Path = c.Path;
cookie.Expires = DateTime.Now.AddHours(6);
}
}
driver.Close();
driver.Dispose();
return(cookie);
}
public IpModuleManager(IpModule module)
{
string[] phantomArgs = new string[] { "--webdriver-loglevel=NONE" };
PhantomJSOptions options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.cli.args", phantomArgs);
Module = module;
var driverService = PhantomJSDriverService.CreateDefaultService();
driverService.HideCommandPromptWindow = true;
driverService.LogFile = "phantomJSService.log";
driver = new PhantomJSDriver(driverService, options);
driver.Manage().Timeouts().ImplicitlyWait(TimeSpan.FromSeconds(10));
loginUrl = string.Format("{0}/login_page.html", Module.Url.Value.ToString());
logoutUrl = string.Format("{0}/logout.html", Module.Url.Value.ToString());
versionUrl = string.Format("{0}/version.html", Module.Url.Value.ToString());
eventUrl = string.Format("{0}/event.html", Module.Url.Value.ToString());
statusLiveUrl = string.Format("{0}/statuslive.html", Module.Url.Value.ToString());
Devices = new List <Device>();
}
private static PhantomJSOptions GetPhantomJsptions()
{
PhantomJSOptions option = new PhantomJSOptions();
option.AddAdditionalCapability("handlesAlerts", true);
return(option);
}
public PhantomJSDriverHelper(string cookies, string domain)
{
var _option = new PhantomJSOptions();
_option.AddAdditionalCapability(@"phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36");
var _service = PhantomJSDriverService.CreateDefaultService();
//代理
_service.ProxyType = "none"; //'http', 'socks5' or 'none'
_service.Proxy = ""; //{address}:{port}
//不加载图片
_service.LoadImages = false;
//忽略SSL证书错误
_service.IgnoreSslErrors = true;
//隐藏命令提示窗口
_service.HideCommandPromptWindow = true;
//日志文件路径
_service.LogFile = "";
//Cookie文件路径
//_service.CookiesFile = "PhantomJSCookie.json";
//_service.DiskCache = true;
//_service.MaxDiskCacheSize = 10240;
//_service.LocalStoragePath = "LocalStoragePath";
_webDriver = new PhantomJSDriver(_service, _option);
if (!string.IsNullOrEmpty(cookies))
{
AddCookie(cookies, domain);
}
}
public string xpath_crwal(string url, string xpath0)
{
/// <summary>
/// 利用phantomjs.exe来完成爬虫
/// 需要在工具--NuGet 程序包管理器--安装几个包
/// Selenium.PhantomJS.WebDriver和Selenium.WebDriver
/// </summary>
/// <param name="url">数据的网址</param>
/// <param name="xpath0">数据的xpath</param>
/// <returns></returns>
PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService();
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0");
service.HideCommandPromptWindow = true; // 隐藏dos窗口
var driver1 = new PhantomJSDriver(service, options);
driver1.Navigate().GoToUrl(url);
ReadOnlyCollection <IWebElement> res = driver1.FindElementsByXPath(xpath0); // 搜索嘛,结果肯定是一个数组
string res_text;
if (res.Count != 0)
{
res_text = res[0].Text;
driver1.Quit();
}
else
{
res_text = "NULL";
}
return(res_text);
}
/// <inheritdoc />
public override DriverOptions GetOptions()
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability(CapabilityType.Proxy, DriverProvider.Proxy);
return(options);
}
private static PhantomJSOptions GetPhantomJsOptions()
{
PhantomJSOptions options = new PhantomJSOptions();
options.AddAdditionalCapability("takesScreenshot", false);
return(options);
}
public override DriverWrapper CreateDriver(string agentString)
{
DriverWrapper driver = null;
var sCaps = new DesiredCapabilities();
try
{
if (string.IsNullOrEmpty(agentString) == false)
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", agentString);
driver = new DriverWrapper(new PhantomJSDriver(options));
}
else
{
driver = new DriverWrapper(new ChromeDriver());
}
}
catch (Exception e)
{
log2.Error("Exception: " + e);
}
return(driver);
}
public void BrowserSettings()
{
mProxy.currentProxy = mProxy.nextProxy();
//DEFAULT BROWSER SETTINGS
UserAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
WEB_settings.AddArgument(string.Format("--ignore-ssl-errors=true"));
WEB_settings.AddArgument(string.Format("--load-images=false"));
WEB_settings.AddArgument(string.Format("--ssl-protocol=any"));
WEB_settings.AddArgument(string.Format("--web-security=false"));
if (mProxy.proxyTYPE != "none")
{
WEB_settings.AddArgument(string.Format("--proxy={0}", mProxy.currentProxy));
WEB_settings.AddArgument(string.Format("--proxy-type={0}", mProxy.proxyTYPE));
}
Console.WriteLine(mProxy.proxyTYPE + "_type");
WEB_settings.HideCommandPromptWindow = true;
//CUSTOM BROWSER OPTIONS/
WEB_options.AddAdditionalCapability("phantomjs.page.settings.userAgent", UserAgent);
try
{
WEB_Browser = new OpenQA.Selenium.PhantomJS.PhantomJSDriver(WEB_settings, WEB_options);
}
catch
{
if (WEB_Browser == null)
{
WEB_Browser.Quit();
}
BrowserSettings();
}
}
private void UsePhatomJS()
{
PhantomJSOptions options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent",
"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25");
_driver = new PhantomJSDriver(options);
}
public BrowserHelper()
{
_defaultService = PhantomJSDriverService.CreateDefaultService(CoreConfiguration.WebDriversLocation);
_defaultService.HideCommandPromptWindow = true;
_options = new PhantomJSOptions();
_options.AddAdditionalCapability("phantomjs.page.settings.userAgent",
@"Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0");
}
private static PhantomJSOptions GetPhantomJsptions()
{
var option = new PhantomJSOptions();
option.AddAdditionalCapability("handlesAlerts", true);
Logger.Info("Using PhantomJSOptions");
return(option);
}
/// <summary>
/// Получить драйвер IWebDriver по имени
/// </summary>
/// <param name="driverName"></param>
/// <returns></returns>
public IWebDriver GetDriver(string driverName)
{
IWebDriver driver = null;
switch (driverName)
{
case "Firefox":
FirefoxProfile firefoxProfile = new FirefoxProfile();
firefoxProfile.SetPreference("browser.download.folderList", 2);
firefoxProfile.SetPreference("browser.download.dir", GetDownloadPath());
firefoxProfile.SetPreference("browser.helperApps.neverAsk.saveToDisk", "application/xml");
firefoxProfile.SetPreference("browser.helperApps.alwaysAsk.force", false);
driver = new FirefoxDriver(firefoxProfile);
break;
case "Chrome":
ChromeOptions chromeOptions = new ChromeOptions();
chromeOptions.AddUserProfilePreference("download.default_directory", GetDownloadPath());
chromeOptions.AddUserProfilePreference("intl.accept_languages", "nl");
chromeOptions.AddUserProfilePreference("disable-popup-blocking", "true");
chromeOptions.AddUserProfilePreference("download.prompt_for_download", false);
chromeOptions.AddUserProfilePreference("download.directory_upgrade", true);
chromeOptions.AddUserProfilePreference("safebrowsing.enabled", true);
driver = new ChromeDriver(chromeOptions);
break;
case "IE":
var options = new InternetExplorerOptions
{
EnsureCleanSession = true,
IgnoreZoomLevel = true,
IntroduceInstabilityByIgnoringProtectedModeSettings = true,
};
driver = new InternetExplorerDriver(options);
break;
case "PhantomJS":
var phantomJsOptions = new PhantomJSOptions();
phantomJsOptions.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36");
var driverService = PhantomJSDriverService.CreateDefaultService();
driverService.IgnoreSslErrors = true;
driverService.SslProtocol = "any";
driver = new PhantomJSDriver(driverService, phantomJsOptions);
break;
}
if (driver != null)
{
driver.Manage().Cookies.DeleteAllCookies();
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(30);
driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(3);
driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromMilliseconds(200);
}
return(driver);
}
public StrongCrawler()
{
_options = new PhantomJSOptions();
_options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36");
_service = PhantomJSDriverService.CreateDefaultService(Environment.CurrentDirectory);
//不加载图片
_service.AddArgument("--load-images=false");
_service.LoadImages = false;
}
private Jenkins()
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.customHeaders.Accept-Language", "en");
_driver = new PhantomJSDriver(options);
_wait = new WebDriverWait(_driver, TimeSpan.FromSeconds(10));
var jenkinsUrl = Environment.GetEnvironmentVariable("JENKINS_URL");
Url = string.IsNullOrEmpty(jenkinsUrl) ? "http://localhost:8080" : jenkinsUrl;
}
public void OpenChicoryPhantom(string url, string itemId)
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("ignoreProtectedModeSettings", true);
using (var browser = new PhantomJSDriver(pathToPhantomJs, options))
{
browser.Navigate().GoToUrl(url);
Assert.IsNotNull(PhFind(browser, By.Id(itemId)));
}
}
private void RegisterCustomPhantomBrowser()
{
var phantomLan = Lan + "," + Lan + ";q=0.5";
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.customHeaders.Accept-Language", phantomLan);
var customPhantomDriver = new CustomPhantomJsSeleniumDriver(options);
_browserSession = new BrowserSession(customPhantomDriver);
_objectContainer.RegisterInstanceAs(_browserSession);
}
public void SetUp()
{
DriverConfiguration configuration = new DriverConfiguration();
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.UserAgent", "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X;en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4Mobile/7B334b Safari/531.21.10 ");
var services = PhantomJSDriverService.CreateDefaultService();
services.LogFile = "D:\\AutomationReport\\Log.txt";
configuration.DriverServices = services;
configuration.DesiredCapabilities = options;
DriverManager.StartDriver(Browser.PhantomJSBrowser,configuration);
}
public BingBot(int mode)
{
PhantomJSOptions options = new PhantomJSOptions();
if (mode == 1)
{
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Android; Mobile; rv:30.0) Gecko/30.0 Firefox/30.0");
}
else
{
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586 ");
}
PhantomJSDriverService service = PhantomJSDriverService.CreateDefaultService();
service.AddArgument("--ignore-ssl-errors=true");
//arguments for proxy
service.AddArgument("--proxy=127.0.0.1:9050");
service.AddArgument("--proxy-type=socks5");
service.HideCommandPromptWindow = true;
browser = new PhantomJSDriver(service, options);
}
public ScrapeDriver()
{
service = PhantomJSDriverService.CreateDefaultService();
service.IgnoreSslErrors = true;
service.LoadImages = false;
service.ProxyType = "none";
service.SslProtocol = "any";
options = new PhantomJSOptions();
options.AddAdditionalCapability("IsJavaScriptEnabled", true);
driver = new PhantomJSDriver(service, options);
}
public void SetUp()
{
DriverConfiguration configuration = new DriverConfiguration();
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.UserAgent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");
var services = PhantomJSDriverService.CreateDefaultService();
services.LogFile = "D:\\AutomationReport\\Log.txt";
configuration.DriverServices = services;
configuration.DesiredCapabilities = options;
DriverManager.StartDriver(Browser.PhantomJSBrowser, configuration);
}
