public static Tuple <string, string> GetPageSource(string url, int waitSecond = 0)
{
var options = new PhantomJSOptions();
options.AddAdditionalCapability("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
options.AddAdditionalCapability("phantomjs.page.settings.loadImages", false);
var JsDriver = new PhantomJSDriver(options);
JsDriver.ExecutePhantomJS("this.onResourceRequested = function(request, net) {" +
" if (request.url.indexOf('google-analytics') !== -1 || request.url.indexOf('.css') !==-1) {" +
" net.abort();" +
" }" +
"};");
JsDriver.Navigate().GoToUrl(url);
JsDriver.WaitUntil(d => !d.Url.Contains("duomai.com") &&
!d.Url.Contains("guangdiu.com") &&
!d.Url.Contains("haohuola.com") &&
!d.Url.Contains("zhuayangmao.com") &&
!d.Url.Contains("union.") &&
!d.Url.Contains("click.taobao"));
var result = JsDriver.PageSource;
var lastUrl = JsDriver.Url;
JsDriver.Close();
JsDriver.Quit();
return(new Tuple <string, string>(result, lastUrl));
}
public static string GrabPage(PhantomJSDriver driver, string targetPgUrl, Tuple <string, string> proxy = null)
{
if (driver == null)
{
throw new InvalidOperationException("driver was not supplied");
}
if (proxy != null)
{
var script = $"return phantom.setProxy(\"{proxy.Item1}\", {proxy.Item2}, \"http\", \"\", \"";
var obj = driver?.ExecutePhantomJS(script);
}
driver.Url = targetPgUrl;
driver.Navigate();
return(driver?.PageSource);
}
