private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new") { Uri uri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type); Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword); using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri) { Version = HttpVersion.Version11 }; var headers = reqMsg.Headers; headers.Referrer = referer; headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"); headers.Add("Pragma", "no-cache"); headers.Add("Cache-Control", "no-cache"); headers.Add("Upgrade-Insecure-Requests", "1"); try { var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); if (rspMsg.IsSuccessStatusCode) { FixCookies(client.CookieContainer, uri); } return(rspMsg); } catch { return(null); } }
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam) { string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName); string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword); var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType); LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection(); using var browser = await PuppeteerConsole.LauncherBrowser(false); using var page = await browser.NewPageAsync(); page.Response += OnResponsed; await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly. for (int i = 0; i < searchParam.MaxPage - 1; i++) { await Task.Delay(RandomTool.NextInt(2000, 4000)); await page.HoverAsync(NextPageElementSelector); await Task.Delay(RandomTool.NextInt(3000, 5000)); await page.ClickAsync(NextPageElementSelector); } return(dataCollection); async void OnResponsed(object sender, ResponseCreatedEventArgs args) { var rsp = args.Response; if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix)) { return; } if (!rsp.Ok) { return; } string rspText = null; try { rspText = await rsp.TextAsync(); } catch (Exception ex) { ShowLogException(ex); return; } var data = PcWebApiProvider.CreateResponseData(rspText, out string _); if (data is null) { return; } dataCollection.AddResponseData(data); dataCollection.FillPositions(searchParam.Keyword); dataCollection.FillCompanies(searchParam.CityName); } }