Beispiel #1
0
        /// <summary>
        /// request by chromium
        /// </summary>
        /// <param name="request">Crawl Request</param>
        /// <returns>Crawl Response</returns>
        public async Task <Response> RequestAsync(Request request)
        {
            var args = new List <string> {
                "--no-sandbox"
            };

            if (request.Proxy != null)
            {
                args.Add("--proxy-server=" + request.Proxy.Uri.ToString());
            }

            var launchOptions = new LaunchOptions {
                Headless = true, Args = args.ToArray()
            };

            using (var browser = await Puppeteer.LaunchAsync(launchOptions))
                using (var page = await browser.NewPageAsync())
                {
                    try
                    {
                        //Authenticate set
                        if (!string.IsNullOrEmpty(request.Username) && !string.IsNullOrEmpty(request.Password))
                        {
                            await page.AuthenticateAsync(new Credentials { Username = request.Username, Password = request.Password });
                        }

                        //cookie set
                        var cookie  = GetCookie(request);
                        var cookies = GenerateCookieParam(cookie);
                        await page.SetCookieAsync(cookies);

                        //hearder set
                        var dic = request.Headers.ToDictionary(h => h.Name, h => h.Value);
                        await page.SetExtraHttpHeadersAsync(dic);

                        //ua set
                        var ua = request.Headers.SingleOrDefault(m => m.Name == "User-Agent").Value;
                        await page.SetUserAgentAsync(ua);

                        var res = await page.GoToAsync(Uri.EscapeUriString(request.Uri.ToString()));

                        var htmlString = await page.GetContentAsync();

                        var response = new Response();
                        response.Headers    = WebHeader.FromDictionary(res.Headers);
                        response.Data       = htmlString;
                        response.StatusCode = res.Status;

                        return(response);
                    }
                    catch (Exception ex)
                    {
                        throw new Exception("chromium error:" + ex.Message);
                    }
                }
        }
Beispiel #2
0
        /// <summary>
        /// request by chromium
        /// </summary>
        /// <param name="request">Crawl Request</param>
        /// <returns>Crawl Response</returns>
        public async Task <Response> RequestAsync(Request request)
        {
            var args = new List <string> {
                "--no-sandbox"
            };

            if (request.Proxy != null)
            {
                args.Add("--proxy-server=" + request.Proxy.Uri.ToString());
            }

            var launchOptions = new LaunchOptions {
                Headless = true, Args = args.ToArray()
            };

            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
            {
                launchOptions.ExecutablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "chromium", "chrome.exe");
            }

            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                launchOptions.ExecutablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "chromium", "chrome");
            }

            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
            {
            }

            using (var browser = await Puppeteer.LaunchAsync(launchOptions))
                using (var page = await browser.NewPageAsync())
                {
                    try
                    {
                        //Authenticate set
                        if (!string.IsNullOrEmpty(request.Username) && !string.IsNullOrEmpty(request.Password))
                        {
                            await page.AuthenticateAsync(new Credentials { Username = request.Username, Password = request.Password });
                        }

                        //cookie set
                        var cookie  = GetCookie(request);
                        var cookies = GenerateCookieParam(cookie);
                        await page.SetCookieAsync(cookies);

                        //hearder set
                        var dic = request.Headers.ToDictionary(h => h.Name, h => h.Value);
                        await page.SetExtraHttpHeadersAsync(dic);

                        //ua set
                        var ua = request.Headers.SingleOrDefault(m => m.Name == "User-Agent").Value;
                        await page.SetUserAgentAsync(ua);

                        var res = await page.GoToAsync(Uri.EscapeUriString(request.Uri.ToString()));

                        if (!string.IsNullOrEmpty(request.WaitDom))
                        {
                            await page.WaitForSelectorAsync("table.hpl_roomInfo");
                        }

                        var htmlString = await page.GetContentAsync();

                        var response = new Response();
                        response.Headers    = WebHeader.FromDictionary(res.Headers);
                        response.Data       = htmlString;
                        response.StatusCode = res.Status;

                        return(response);
                    }
                    catch (Exception ex)
                    {
                        throw new Exception("chromium error:" + ex.Message);
                    }
                }
        }