public XWebCrawler()
{
_webClient = new WebClientExtended(new CookieContainer());
Parser = GetParser();
Initialize(_webClient);
if (GhostConfiguration.ScriptEngine != null)
{
GhostConfiguration.ScriptEngine.CreateEngine();
}
}
private void Initialize(WebClientExtended wc)
{
Headers = new WebHeaderCollection();
UserAgent = "User-Agent Mozilla/5.0 (Windows NT 6.3; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0";
HtmlEncoding = Encoding.UTF8;
_navigation = new NavigationHandle(this, wc);
_navigation.OnFinishedNavigation += (s, html) =>
{
Page = _webClient.ResponseUri;
var htmlWithScripts = ScriptingParser.Execute(html, Parser, Page, _webClient.CookieContainer);
TextDocument = htmlWithScripts;
Parser.Load(TextDocument);
HtmlDocument = Parser.Document;
};
}
public StandAloneNavigationHandle StandAloneNavigate()
{
var wc = new WebClientExtended(_webClient.CookieContainer);
if (wc.Headers.AllKeys.All(x => x != HttpRequestHeader.UserAgent.ToString()))
{
wc.Headers.Add(HttpRequestHeader.UserAgent, UserAgent);
}
if (Headers != null && Headers.Count > 0)
{
foreach (var key in Headers.AllKeys)
{
if (wc.Headers.AllKeys.All(x => x != key))
{
wc.Headers.Add(key, Headers[key]);
}
}
}
return(new StandAloneNavigationHandle(this, wc));
}
public NavigationHandle(XWebCrawler browser, WebClientExtended webClient)
{
_browser = browser;
_webClient = webClient;
}
public StandAloneNavigationHandle(XWebCrawler browser, WebClientExtended webClient)
{
_browser = browser;
_webClient = webClient;
}