/// <summary> /// request by chromium /// </summary> /// <param name="request">Crawl Request</param> /// <returns>Crawl Response</returns> public async Task <Response> RequestAsync(Request request) { var args = new List <string> { "--no-sandbox" }; if (request.Proxy != null) { args.Add("--proxy-server=" + request.Proxy.Uri.ToString()); } var launchOptions = new LaunchOptions { Headless = true, Args = args.ToArray() }; using (var browser = await Puppeteer.LaunchAsync(launchOptions)) using (var page = await browser.NewPageAsync()) { try { //Authenticate set if (!string.IsNullOrEmpty(request.Username) && !string.IsNullOrEmpty(request.Password)) { await page.AuthenticateAsync(new Credentials { Username = request.Username, Password = request.Password }); } //cookie set var cookie = GetCookie(request); var cookies = GenerateCookieParam(cookie); await page.SetCookieAsync(cookies); //hearder set var dic = request.Headers.ToDictionary(h => h.Name, h => h.Value); await page.SetExtraHttpHeadersAsync(dic); //ua set var ua = request.Headers.SingleOrDefault(m => m.Name == "User-Agent").Value; await page.SetUserAgentAsync(ua); var res = await page.GoToAsync(Uri.EscapeUriString(request.Uri.ToString())); var htmlString = await page.GetContentAsync(); var response = new Response(); response.Headers = WebHeader.FromDictionary(res.Headers); response.Data = htmlString; response.StatusCode = res.Status; return(response); } catch (Exception ex) { throw new Exception("chromium error:" + ex.Message); } } }
/// <summary> /// request by chromium /// </summary> /// <param name="request">Crawl Request</param> /// <returns>Crawl Response</returns> public async Task <Response> RequestAsync(Request request) { var args = new List <string> { "--no-sandbox" }; if (request.Proxy != null) { args.Add("--proxy-server=" + request.Proxy.Uri.ToString()); } var launchOptions = new LaunchOptions { Headless = true, Args = args.ToArray() }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { launchOptions.ExecutablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "chromium", "chrome.exe"); } if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { launchOptions.ExecutablePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "chromium", "chrome"); } if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { } using (var browser = await Puppeteer.LaunchAsync(launchOptions)) using (var page = await browser.NewPageAsync()) { try { //Authenticate set if (!string.IsNullOrEmpty(request.Username) && !string.IsNullOrEmpty(request.Password)) { await page.AuthenticateAsync(new Credentials { Username = request.Username, Password = request.Password }); } //cookie set var cookie = GetCookie(request); var cookies = GenerateCookieParam(cookie); await page.SetCookieAsync(cookies); //hearder set var dic = request.Headers.ToDictionary(h => h.Name, h => h.Value); await page.SetExtraHttpHeadersAsync(dic); //ua set var ua = request.Headers.SingleOrDefault(m => m.Name == "User-Agent").Value; await page.SetUserAgentAsync(ua); var res = await page.GoToAsync(Uri.EscapeUriString(request.Uri.ToString())); if (!string.IsNullOrEmpty(request.WaitDom)) { await page.WaitForSelectorAsync("table.hpl_roomInfo"); } var htmlString = await page.GetContentAsync(); var response = new Response(); response.Headers = WebHeader.FromDictionary(res.Headers); response.Data = htmlString; response.StatusCode = res.Status; return(response); } catch (Exception ex) { throw new Exception("chromium error:" + ex.Message); } } }