public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam) { string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName); string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword); var jobListUri = PcWebApiProvider.GetJobListUri(encodedCityName, encodedKeyword, searchParam.SearchType); LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection(); using var browser = await PuppeteerConsole.LauncherBrowser(false); using var page = await browser.NewPageAsync(); page.Response += OnResponsed; await page.GoToAsync(jobListUri.AbsoluteUri).ConfigureAwait(false); //Get the first page directly. for (int i = 0; i < searchParam.MaxPage - 1; i++) { await Task.Delay(RandomTool.NextInt(2000, 4000)); await page.HoverAsync(NextPageElementSelector); await Task.Delay(RandomTool.NextInt(3000, 5000)); await page.ClickAsync(NextPageElementSelector); } return(dataCollection); async void OnResponsed(object sender, ResponseCreatedEventArgs args) { var rsp = args.Response; if (!rsp.Url.StartsWith(PcWebApiProvider.PositionAjaxUrlPrefix)) { return; } if (!rsp.Ok) { return; } string rspText = null; try { rspText = await rsp.TextAsync(); } catch (Exception ex) { ShowLogException(ex); return; } var data = PcWebApiProvider.CreateResponseData(rspText, out string _); if (data is null) { return; } dataCollection.AddResponseData(data); dataCollection.FillPositions(searchParam.Keyword); dataCollection.FillCompanies(searchParam.CityName); } }
public override async Task RunAsync() { const int settingLength = 5; var args = RunSettings;//[SchemeKey,City,Keyword,SearchType(default or new)] if (args == null) { ShowLogError("RunSettings is NULL."); return; } if (args.Length != settingLength) { ShowLogError($"Invalid length of RunSettings: {args.Length.ToString()}. The right length should be {settingLength.ToString()}."); return; } string schemeKey = args[0]; if (string.IsNullOrEmpty(schemeKey)) { ShowLogError($"Invalid RunSettings[0]: {schemeKey}."); return; } var schemePair = Array.Find(_schemeFactories, s => s.Key.Equals(schemeKey, StringComparison.CurrentCultureIgnoreCase)); var scheme = schemePair.Value?.Invoke(); if (scheme == null) { ShowLogError($"Scheme Invoke Error: {schemeKey}."); return; } for (byte i = 1; i < args.Length; i++) { if (string.IsNullOrEmpty(args[i])) { ShowLogError($"Invalid RunSettings[{i.ToString()}]: {args[i]}."); return; } } var searchParam = new LaGouSearchParam() { CityName = args[1], Keyword = args[2], SearchType = args[3] }; if (int.TryParse(args[4], out int maxPage)) { searchParam.MaxPage = Math.Max(1, maxPage); } await scheme.RunAsync(searchParam).ConfigureAwait(false); return; }
public override async Task RunAsync() { const int settingLength = 5; var args = RunSettings;//[SchemeKey,City,Keyword,SearchType(default or new)] if (args == null) { ShowLogError("RunSettings is NULL."); return; } if (args.Length != settingLength) { ShowLogError($"Invalid length of RunSettings: {args.Length}. The right length should be {settingLength}."); return; } string schemeKey = args[0]; if (string.IsNullOrEmpty(schemeKey) || !_schemeFactories.TryGetValue(schemeKey.ToLowerInvariant(), out var schemeFactory)) { ShowLogError($"Invalid RunSettings[0]: {schemeKey}."); return; } var scheme = schemeFactory.Invoke(); if (scheme == null) { ShowLogError($"Scheme Invoke Error: {schemeKey}."); return; } for (byte i = 1; i < args.Length; i++) { if (string.IsNullOrEmpty(args[i])) { ShowLogError($"Invalid RunSettings[{i}]: {args[i]}."); return; } } var searchParam = new LaGouSearchParam() { CityName = args[1], Keyword = args[2], SearchType = args[3] }; if (int.TryParse(args[4], out int maxPage)) { searchParam.MaxPage = Math.Max(1, maxPage); } await scheme.RunAsync(searchParam).ConfigureAwait(false); return; }
public override async Task RunAsync(LaGouSearchParam searchParam) { var datas = await Collector.CollectAsync(searchParam).ConfigureAwait(false); using var context = new LaGouSqlServerContext(); context.Database.EnsureCreated(); int insertCount = 0; insertCount = InsertData(context, c => c.Positions, datas.Positions.Values, p => p.PositionId); ShowLogInfo($"{nameof(context.Positions)} inserted {insertCount.ToString()}."); insertCount = InsertData(context, c => c.Companies, datas.Companies.Values, p => p.CompanyId); ShowLogInfo($"{nameof(context.Companies)} inserted {insertCount.ToString()}."); insertCount = InsertData(context, c => c.HrInfos, datas.HrInfos.Values, p => p.UserId); ShowLogInfo($"{nameof(context.HrInfos)} inserted {insertCount.ToString()}."); insertCount = InsertData(context, c => c.HrDailyRecords, datas.HrDailyRecords.Values); ShowLogInfo($"{nameof(context.HrDailyRecords)} inserted {insertCount.ToString()}."); }
public override async Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam) { string encodedCityName = WebTool.UrlEncodeByW3C(searchParam.CityName); string encodedKeyword = WebTool.UrlEncodeByW3C(searchParam.Keyword); LaGouResponseDataCollection dataCollection = new LaGouResponseDataCollection(); using (var client = CreateHttpClient()) { await TryInitCookiesAsync(client, encodedCityName, encodedKeyword, searchParam.SearchType); await Task.Delay(100); string sid = null; using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, "1")) { string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType); var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid); if (data != null) { dataCollection.AddResponseData(data); } } for (int i = 2; i <= searchParam.MaxPage; i++) { await Task.Delay(RandomTool.NextIntSafely(4000, 6000)); using (var postContent = PcWebApiProvider.GetPositionAjaxContent(encodedKeyword, i.ToString(), sid)) { string ajaxRsp = await PostPositionAjaxAsync(client, postContent, encodedCityName, encodedKeyword, searchParam.SearchType); var data = PcWebApiProvider.CreateResponseData(ajaxRsp, out sid); if (data != null) { dataCollection.AddResponseData(data); } } } } dataCollection.FillPositions(searchParam.Keyword); dataCollection.FillCompanies(searchParam.CityName); return(dataCollection); }
public abstract Task RunAsync(LaGouSearchParam searchParam);
public abstract Task <LaGouResponseDataCollection> CollectAsync(LaGouSearchParam searchParam);