private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new")
{
Uri uri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type);
Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword);
using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri)
{
Version = HttpVersion.Version11
};
var headers = reqMsg.Headers;
headers.Referrer = referer;
headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
headers.Add("Pragma", "no-cache");
headers.Add("Cache-Control", "no-cache");
headers.Add("Upgrade-Insecure-Requests", "1");
try
{
var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (rspMsg.IsSuccessStatusCode)
{
FixCookies(client.CookieContainer, uri);
}
return(rspMsg);
}
catch
{
return(null);
}
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using var browser = await PuppeteerConsole.LauncherBrowser(false);
using var page = await browser.NewPageAsync();
page.Response += OnResponsed;
await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly.
for (int i = 0; i < searchParam.MaxPage - 1; i++)
{
await Task.Delay(RandomTool.NextInt(2000, 4000));
await page.HoverAsync(NextPageElementSelector);
await Task.Delay(RandomTool.NextInt(3000, 5000));
await page.ClickAsync(NextPageElementSelector);
}
return(dataCollection);
async void OnResponsed(object sender, ResponseCreatedEventArgs args)
{
var rsp = args.Response;
if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix))
{
return;
}
if (!rsp.Ok)
{
return;
}
string rspText = null;
try
{
rspText = await rsp.TextAsync();
}
catch (Exception ex)
{
ShowLogException(ex);
return;
}
var data = PcWebApiProvider.CreateResponseData(rspText, out string _);
if (data is null)
{
return;
}
dataCollection.AddResponseData(data);
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
}
}