示例#1
0
        public XWebCrawler()
        {
            _webClient = new WebClientExtended(new CookieContainer());
            Parser     = GetParser();
            Initialize(_webClient);

            if (GhostConfiguration.ScriptEngine != null)
            {
                GhostConfiguration.ScriptEngine.CreateEngine();
            }
        }
示例#2
0
        private void Initialize(WebClientExtended wc)
        {
            Headers      = new WebHeaderCollection();
            UserAgent    = "User-Agent	Mozilla/5.0 (Windows NT 6.3; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0";
            HtmlEncoding = Encoding.UTF8;

            _navigation = new NavigationHandle(this, wc);
            _navigation.OnFinishedNavigation += (s, html) =>
            {
                Page = _webClient.ResponseUri;
                var htmlWithScripts = ScriptingParser.Execute(html, Parser, Page, _webClient.CookieContainer);
                TextDocument = htmlWithScripts;
                Parser.Load(TextDocument);
                HtmlDocument = Parser.Document;
            };
        }
示例#3
0
        public StandAloneNavigationHandle StandAloneNavigate()
        {
            var wc = new WebClientExtended(_webClient.CookieContainer);

            if (wc.Headers.AllKeys.All(x => x != HttpRequestHeader.UserAgent.ToString()))
            {
                wc.Headers.Add(HttpRequestHeader.UserAgent, UserAgent);
            }

            if (Headers != null && Headers.Count > 0)
            {
                foreach (var key in Headers.AllKeys)
                {
                    if (wc.Headers.AllKeys.All(x => x != key))
                    {
                        wc.Headers.Add(key, Headers[key]);
                    }
                }
            }

            return(new StandAloneNavigationHandle(this, wc));
        }
 public NavigationHandle(XWebCrawler browser, WebClientExtended webClient)
 {
     _browser = browser;
     _webClient = webClient;
 }
示例#5
0
 public StandAloneNavigationHandle(XWebCrawler browser, WebClientExtended webClient)
 {
     _browser   = browser;
     _webClient = webClient;
 }