public async Task RunAtTimeOfAsync(DateTime now) { var config = _ctx.JobGettingConfig.FirstOrDefault(); if (config == null) { return; } HttpClient client = new HttpClient(); var httpResult = await client.GetAsync("https://remotive.io/api/remote-jobs"); if (httpResult.IsSuccessStatusCode) { var contentString = await httpResult.Content.ReadAsStringAsync(); try { var remotiveIO = RemotiveIoData.FromJson(contentString); foreach (var okRemoteJob in remotiveIO.Jobs) { if (string.IsNullOrEmpty(okRemoteJob.Title)) { continue; } if (!_ctx.JobPostings.Any(x => x.Title == okRemoteJob.Title && x.Description == okRemoteJob.Description)) { var Description = new string(okRemoteJob.Description.Where(c => !char.IsPunctuation(c)).ToArray()); SummaryDTO nltkSummary = await _nltkService.ExtractSummary(Description); KeyPhrasesWrapperDTO wrapper = await _nltkService.ExtractKeyPhrases(Description); var newJobPosting = new JobPosting() { Title = okRemoteJob.Title, Description = okRemoteJob.Description, URL = okRemoteJob.Url.OriginalString, Company = okRemoteJob.CompanyName, Location = okRemoteJob.CandidateRequiredLocation, PostDate = okRemoteJob.PublicationDate.ToString(), Salary = okRemoteJob.Salary, JobSource = "RemotiveIo", CompanyLogoUrl = okRemoteJob.CompanyLogoUrl == null ? "" : okRemoteJob.CompanyLogoUrl.OriginalString, Summary = nltkSummary.SummaryText }; await _ctx.AddAsync(newJobPosting); await _ctx.SaveChangesAsync(); _logger.LogInformation("List<KeyPhrase> ListKeyPhrase"); foreach (KeyPhraseDTO KeyPhrase in wrapper.rank_list) { if (KeyPhrase.Affinty > config.MinAffintyScore) { _ctx.KeyPhrase.Add(new KeyPhrase { Affinty = KeyPhrase.Affinty, Text = KeyPhrase.Text, JobPostingId = newJobPosting.Id }); } _logger.LogInformation($"item.Affinty {KeyPhrase.Affinty}"); _logger.LogInformation($"item.Text {KeyPhrase.Text}"); } await _ctx.SaveChangesAsync(); if (_ctx.Tags.Any(x => x.Text.Trim() == okRemoteJob.Category)) { var tagFromDb = _ctx.Tags .FirstOrDefault(x => x.Text.Trim() == okRemoteJob.Category.Trim()); newJobPosting.Tags.Add(tagFromDb); } else { newJobPosting.Tags.Add(new Models.Entity.Tag() { Text = okRemoteJob.Category.Trim() }); } await _ctx.SaveChangesAsync(); } } } catch (Exception ex) { _logger.Log(LogLevel.Error, "Error Occured int RemotiveIoDataJob", ex); } } else { _logger.Log(LogLevel.Error, "httpResult.IsSuccessStatusCode was false"); } }
public async Task RunAtTimeOfAsync(DateTime now) { var config = _ctx.JobGettingConfig.FirstOrDefault(); if (config == null) { return; } var tags = _ctx.PositionName.ToList(); foreach (var tag in tags) { HttpClient client = new HttpClient(); var stuff = await client.GetAsync($"https://remoteok.io/api?tags=" + tag.Name); if (stuff.IsSuccessStatusCode) { var contentString = await stuff.Content.ReadAsStringAsync(); try { var okRemoteData = OkRemoteData.FromJson(contentString); foreach (var okRemoteJob in okRemoteData) { if (string.IsNullOrEmpty(okRemoteJob.Position)) { continue; } if (!_ctx.JobPostings.Any(x => x.Title == okRemoteJob.Position && x.Description == okRemoteJob.Description)) { var Description = new string(okRemoteJob.Description.Where(c => !char.IsPunctuation(c)).ToArray()); SummaryDTO nltkSummary = await _nltkService.ExtractSummary(Description); KeyPhrasesWrapperDTO wrapper = await _nltkService.ExtractKeyPhrases(Description); var newJobPosting = new JobPosting() { Title = okRemoteJob.Position, Description = okRemoteJob.Description, URL = okRemoteJob.Url.OriginalString, Company = okRemoteJob.Company, Location = okRemoteJob.Location, PostDate = okRemoteJob.Date.ToString(), Salary = "", JobSource = "RemoteOk", CompanyLogoUrl = okRemoteJob.CompanyLogo, Summary = nltkSummary.SummaryText }; await _ctx.AddAsync(newJobPosting); await _ctx.SaveChangesAsync(); _logger.LogInformation("List<KeyPhrase> ListKeyPhrase"); foreach (KeyPhraseDTO KeyPhrase in wrapper.rank_list) { if (KeyPhrase.Affinty > config.MinAffintyScore) { _ctx.KeyPhrase.Add(new KeyPhrase { Affinty = KeyPhrase.Affinty, Text = KeyPhrase.Text, JobPostingId = newJobPosting.Id }); } _logger.LogInformation($"item.Affinty {KeyPhrase.Affinty}"); _logger.LogInformation($"item.Text {KeyPhrase.Text}"); } await _ctx.SaveChangesAsync(); foreach (var item in okRemoteJob.Tags) { if (_ctx.Tags.Any(x => x.Text.Trim() == item.Trim())) { var tagFromDB = _ctx.Tags.Where(x => x.Text.Trim() == item.Trim()).FirstOrDefault(); newJobPosting.Tags.Add(tagFromDB); } else { newJobPosting.Tags.Add(new Models.Entity.Tag() { Text = item.Trim() }); } } await _ctx.SaveChangesAsync(); } } } catch (Exception ex) { _logger.LogError(ex, "Failed to deserialize"); } } Thread.Sleep(config.OkRemoteTimeBetweenTags); } }
public async Task RunAtTimeOf(DateTime now) { var config = _ctx.JobGettingConfig.FirstOrDefault(); if (config == null) { return; } _logger.LogInformation("KeyPhraseGeneratorJob Job Starts... "); string connectionString = Secrets.GetDBConnectionString(_configuration); using (MySqlConnection connection = new MySqlConnection(connectionString)) { MySqlCommand command = new MySqlCommand(@" SELECT `JobPostings`.Id, `JobPostings`.Description FROM `jobtransparency`.`JobPostings` WHERE `JobPostings`.Id NOT IN(SELECT `KeyPhrase`.JobPostingId FROM `jobtransparency`.`KeyPhrase`)", connection); command.CommandTimeout = config.SQLCommandTimeOut; try { connection.Open(); var reader = await command.ExecuteReaderAsync(); while (reader.Read()) { var Description = (string)reader[1]; var Id = (int)reader[0]; if (Description.Length <= 5) { continue; } Description = new string(Description.Where(c => !char.IsPunctuation(c)).ToArray()); KeyPhrasesWrapperDTO wrapper = await _NLTKService.ExtractKeyPhrases(Description); if (wrapper != null && wrapper.rank_list != null && wrapper.rank_list.Count > 0) { List <KeyPhrase> ListKeyPhrase = new List <KeyPhrase>(); _logger.LogInformation("List<KeyPhrase> ListKeyPhrase"); foreach (KeyPhraseDTO item in wrapper.rank_list) { if (item.Affinty > config.MinAffintyScore) { ListKeyPhrase.Add(new KeyPhrase { Affinty = item.Affinty, Text = item.Text, JobPostingId = Id }); } _logger.LogInformation($"item.Affinty {item.Affinty}"); _logger.LogInformation($"item.Text {item.Text}"); } _KeyPharseRepository.CreateKeyPhrases(ListKeyPhrase); _logger.LogInformation("_KeyPharseRepository.CreateKeyPhrases(ListKeyPhrase);"); } } reader.Close(); } catch (Exception ex) { _logger.LogError(ex, "KeyPhraseGeneratorJob Ends... "); } } _logger.LogInformation("KeyPhraseGeneratorJob Ends... "); }