public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using var browser = await PuppeteerConsole.LauncherBrowser(false);
using var page = await browser.NewPageAsync();
page.Response += OnResponsed;
await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly.
for (int i = 0; i < searchParam.MaxPage - 1; i++)
{
await Task.Delay(RandomTool.NextInt(2000, 4000));
await page.HoverAsync(NextPageElementSelector);
await Task.Delay(RandomTool.NextInt(3000, 5000));
await page.ClickAsync(NextPageElementSelector);
}
return(dataCollection);
async void OnResponsed(object sender, ResponseCreatedEventArgs args)
{
var rsp = args.Response;
if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix))
{
return;
}
if (!rsp.Ok)
{
return;
}
string rspText = null;
try
{
rspText = await rsp.TextAsync();
}
catch (Exception ex)
{
ShowLogException(ex);
return;
}
var data = PcWebApiProvider.CreateResponseData(rspText, out string _);
if (data is null)
{
return;
}
dataCollection.AddResponseData(data);
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
}
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using (var client = CreateHttpClient())
{
await TryInitCookiesAsync(client, encodedCityName, encodedKeyword, searchParam.SearchType);
await Task.Delay(100);
string sid = null;
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, "1"))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
for (int i = 2; i <= searchParam.MaxPage; i++)
{
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, i.ToString(), sid))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
}
}
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
return(dataCollection);
}
