public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using var browser = await PuppeteerConsole.LauncherBrowser(false);
using var page = await browser.NewPageAsync();
page.Response += OnResponsed;
await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly.
for (int i = 0; i < searchParam.MaxPage - 1; i++)
{
await Task.Delay(RandomTool.NextInt(2000, 4000));
await page.HoverAsync(NextPageElementSelector);
await Task.Delay(RandomTool.NextInt(3000, 5000));
await page.ClickAsync(NextPageElementSelector);
}
return(dataCollection);
async void OnResponsed(object sender, ResponseCreatedEventArgs args)
{
var rsp = args.Response;
if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix))
{
return;
}
if (!rsp.Ok)
{
return;
}
string rspText = null;
try
{
rspText = await rsp.TextAsync();
}
catch (Exception ex)
{
ShowLogException(ex);
return;
}
var data = PcWebApiProvider.CreateResponseData(rspText, out string _);
if (data is null)
{
return;
}
dataCollection.AddResponseData(data);
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
}
}
public override async Task RunAsync()
{
const int settingLength = 5;
var args = RunSettings;//[SchemeKey,City,Keyword,SearchType(default or new)]
if (args == null)
{
ShowLogError("RunSettings is NULL.");
return;
}
if (args.Length != settingLength)
{
ShowLogError($"Invalid length of RunSettings: {args.Length.ToString()}. The right length should be {settingLength.ToString()}.");
return;
}
string schemeKey = args[0];
if (string.IsNullOrEmpty(schemeKey))
{
ShowLogError($"Invalid RunSettings[0]: {schemeKey}.");
return;
}
var schemePair = Array.Find(_schemeFactories, s => s.Key.Equals(schemeKey, StringComparison.CurrentCultureIgnoreCase));
var scheme = schemePair.Value?.Invoke();
if (scheme == null)
{
ShowLogError($"Scheme Invoke Error: {schemeKey}.");
return;
}
for (byte i = 1; i < args.Length; i++)
{
if (string.IsNullOrEmpty(args[i]))
{
ShowLogError($"Invalid RunSettings[{i.ToString()}]: {args[i]}.");
return;
}
}
var searchParam = new LaGouSearchParam()
{
CityName = args[1],
Keyword = args[2],
SearchType = args[3]
};
if (int.TryParse(args[4], out int maxPage))
{
searchParam.MaxPage = Math.Max(1, maxPage);
}
await scheme.RunAsync(searchParam).ConfigureAwait(false);
return;
}
public override async Task RunAsync()
{
const int settingLength = 5;
var args = RunSettings;//[SchemeKey,City,Keyword,SearchType(default or new)]
if (args == null)
{
ShowLogError("RunSettings is NULL.");
return;
}
if (args.Length != settingLength)
{
ShowLogError($"Invalid length of RunSettings: {args.Length}. The right length should be {settingLength}.");
return;
}
string schemeKey = args[0];
if (string.IsNullOrEmpty(schemeKey) || !_schemeFactories.TryGetValue(schemeKey.ToLowerInvariant(), out var schemeFactory))
{
ShowLogError($"Invalid RunSettings[0]: {schemeKey}.");
return;
}
var scheme = schemeFactory.Invoke();
if (scheme == null)
{
ShowLogError($"Scheme Invoke Error: {schemeKey}.");
return;
}
for (byte i = 1; i < args.Length; i++)
{
if (string.IsNullOrEmpty(args[i]))
{
ShowLogError($"Invalid RunSettings[{i}]: {args[i]}.");
return;
}
}
var searchParam = new LaGouSearchParam()
{
CityName = args[1],
Keyword = args[2],
SearchType = args[3]
};
if (int.TryParse(args[4], out int maxPage))
{
searchParam.MaxPage = Math.Max(1, maxPage);
}
await scheme.RunAsync(searchParam).ConfigureAwait(false);
return;
}
public override async Task RunAsync(LaGouSearchParam searchParam)
{
var datas = await Collector.CollectAsync(searchParam).ConfigureAwait(false);
using var context = new LaGouSqlServerContext();
context.Database.EnsureCreated();
int insertCount = 0;
insertCount = InsertData(context, c => c.Positions, datas.Positions.Values, p => p.PositionId);
ShowLogInfo($"{nameof(context.Positions)} inserted {insertCount.ToString()}.");
insertCount = InsertData(context, c => c.Companies, datas.Companies.Values, p => p.CompanyId);
ShowLogInfo($"{nameof(context.Companies)} inserted {insertCount.ToString()}.");
insertCount = InsertData(context, c => c.HrInfos, datas.HrInfos.Values, p => p.UserId);
ShowLogInfo($"{nameof(context.HrInfos)} inserted {insertCount.ToString()}.");
insertCount = InsertData(context, c => c.HrDailyRecords, datas.HrDailyRecords.Values);
ShowLogInfo($"{nameof(context.HrDailyRecords)} inserted {insertCount.ToString()}.");
}
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam)
{
string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName);
string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword);
LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection();
using (var client = CreateHttpClient())
{
await TryInitCookiesAsync(client, encodedCityName, encodedKeyword, searchParam.SearchType);
await Task.Delay(100);
string sid = null;
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, "1"))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
for (int i = 2; i <= searchParam.MaxPage; i++)
{
await Task.Delay(RandomTool.NextIntSafely(4000, 6000));
using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, i.ToString(), sid))
{
string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType);
var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid);
if (data != null)
{
dataCollection.AddResponseData(data);
}
}
}
}
dataCollection.FillPositions(searchParam.Keyword);
dataCollection.FillCompanies(searchParam.CityName);
return(dataCollection);
}
public abstract Task RunAsync(LaGouSearchParam searchParam);
public abstract Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam);