private static async Task <HttpResponseMessage> GetJobListAsync(SpiderHttpClient client, string encodedCityName, string encodedKeyword, string type = "new")
{
Uri uri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, type);
Uri referer = PcWebApiProvider.GetJobListReferer(encodedKeyword);
using var reqMsg = new HttpRequestMessage(HttpMethod.Get, uri)
{
Version = HttpVersion.Version11
};
var headers = reqMsg.Headers;
headers.Referrer = referer;
headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
headers.Add("Pragma", "no-cache");
headers.Add("Cache-Control", "no-cache");
headers.Add("Upgrade-Insecure-Requests", "1");
try
{
var rspMsg = await client.SendAsync(reqMsg, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
if (rspMsg.IsSuccessStatusCode)
{
FixCookies(client.CookieContainer, uri);
}
return(rspMsg);
}
catch
{
return(null);
}
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using var browser = await PuppeteerConsole.LauncherBrowser(false);
using var page = await browser.NewPageAsync();
page.Response += OnResponsed;
await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly.
for (int i = 0; i < searchParam.MaxPage - 1; i++)
{
await Task.Delay(RandomTool.NextInt(2000, 4000));
await page.HoverAsync(NextPageElementSelector);
await Task.Delay(RandomTool.NextInt(3000, 5000));
await page.ClickAsync(NextPageElementSelector);
}
return(dataCollection);
async void OnResponsed(object sender, ResponseCreatedEventArgs args)
{
var rsp = args.Response;
if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix))
{
return;
}
if (!rsp.Ok)
{
return;
}
string rspText = null;
try
{
rspText = await rsp.TextAsync();
}
catch (Exception ex)
{
ShowLogException(ex);
return;
}
var data = PcWebApiProvider.CreateResponseData(rspText, out string _);
if (data is null)
{
return;
}
dataCollection.AddResponseData(data);
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
}
}
public override async Task <int> CollectAsync(string areaIdStr)
{
Uri apiUri = PcWebApiProvider.GetApiUri_GetLiveRoomCountByAreaID(areaIdStr);
var proxyUriLoader = CreateProxyUriLoader();
var proxySelector = new SimpleWebProxySelector(proxyUriLoader);
string rspText = await HttpConsole.GetResponseTextByProxyAsync(apiUri, proxySelector, GetResponseTextByProxyAsync, 49);
if (string.IsNullOrEmpty(rspText))
{
return(0);
}
return(PcWebApiProvider.GetLiveRoomCount(rspText));
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using (var client = CreateHttpClient())
{
await TryInitCookiesAsync(client, encodedCityName, encodedKeyword, searchParam.SearchType);
await Task.Delay(100);
string sid = null;
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, "1"))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
for (int i = 2; i <= searchParam.MaxPage; i++)
{
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, i.ToString(), sid))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
}
}
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
return(dataCollection);
}
private static async Task <string> PostPositionAjaxAsync(HttpClient client, HttpContent content, string encodedCityName, string encodedKeyword, string type = "new")
{
Uri uri = PcWebApiProvider.GetPositionAjaxUri(encodedCityName, type);
Uri referer = PcWebApiProvider.GetPostionAjaxReferer(encodedCityName, encodedKeyword, type);
HttpRequestMessage reqMsg = new HttpRequestMessage(HttpMethod.Post, uri)
{
Content = content, Version = HttpVersion.Version11
};
var headers = reqMsg.Headers;
headers.Referrer = referer;
headers.Add("Accept", "application/json, text/javascript, */*; q=0.01");
headers.Add("X-Requested-With", "XMLHttpRequest");
headers.Add("X-Anit-Forge-Code", "0");
headers.Add("X-Anit-Forge-Token", "None");
headers.Add("Origin", PcWebApiProvider.HomePageUri.AbsoluteUri);
HttpResponseMessage rspMsg = null;
try
{
rspMsg = await client.SendAsync(reqMsg).ConfigureAwait(false);
}
catch
{
rspMsg?.Dispose();
return(null);
}
using (rspMsg)
{
if (!rspMsg.IsSuccessStatusCode)
{
return(null);
}
return(await rspMsg.ToTextAsync().ConfigureAwait(false));
}
}
