public async Task RunAtTimeOf(DateTime now)
{
var config = _ctx.JobGettingConfig.FirstOrDefault();
if (config == null)
{
return;
}
_logger.LogInformation("SummaryGeneratorJob Starts... ");
string connectionString = Secrets.GetDBConnectionString(_configuration);
using (MySqlConnection connection = new MySqlConnection(connectionString))
{
// Create the Command and Parameter objects.
MySqlCommand command = new MySqlCommand(@"
SELECT Id, Description FROM `jobtransparency`.`JobPostings` WHERE Description = ''
", connection);
command.CommandTimeout = config.SQLCommandTimeOut;
try
{
connection.Open();
var reader = await command.ExecuteReaderAsync();
while (reader.Read())
{
var Id = (int)reader[0];
var Description = (string)reader[1];
Description = new string(Description.Where(c => !char.IsPunctuation(c)).ToArray());
SummaryDTO nltkSummary = await _nltkService.ExtractSummary(Description);
var Job = await _jobPostingRepository.GetById(Id);
Job.Summary = nltkSummary.SummaryText;
await _jobPostingRepository.Put(Job.Id, Job);
}
reader.Close();
}
catch (Exception ex)
{
_logger.LogInformation(ex, "SummaryGeneratorJob Ends... ");
}
}
_logger.LogInformation("SummaryGeneratorJob Ends... ");
}
public async Task RunAtTimeOfAsync(DateTime now)
{
var config = _ctx.JobGettingConfig.FirstOrDefault();
if (config == null)
{
return;
}
var tags = _ctx.PositionName.ToList();
foreach (var tag in tags)
{
HttpClient client = new HttpClient();
var stuff = await client.GetAsync($"https://remoteok.io/api?tags=" + tag.Name);
if (stuff.IsSuccessStatusCode)
{
var contentString = await stuff.Content.ReadAsStringAsync();
try
{
var okRemoteData = OkRemoteData.FromJson(contentString);
foreach (var okRemoteJob in okRemoteData)
{
if (string.IsNullOrEmpty(okRemoteJob.Position))
{
continue;
}
if (!_ctx.JobPostings.Any(x => x.Title == okRemoteJob.Position && x.Description == okRemoteJob.Description))
{
var Description = new string(okRemoteJob.Description.Where(c => !char.IsPunctuation(c)).ToArray());
SummaryDTO nltkSummary = await _nltkService.ExtractSummary(Description);
KeyPhrasesWrapperDTO wrapper = await _nltkService.ExtractKeyPhrases(Description);
var newJobPosting = new JobPosting()
{
Title = okRemoteJob.Position,
Description = okRemoteJob.Description,
URL = okRemoteJob.Url.OriginalString,
Company = okRemoteJob.Company,
Location = okRemoteJob.Location,
PostDate = okRemoteJob.Date.ToString(),
Salary = "",
JobSource = "RemoteOk",
CompanyLogoUrl = okRemoteJob.CompanyLogo,
Summary = nltkSummary.SummaryText
};
await _ctx.AddAsync(newJobPosting);
await _ctx.SaveChangesAsync();
_logger.LogInformation("List<KeyPhrase> ListKeyPhrase");
foreach (KeyPhraseDTO KeyPhrase in wrapper.rank_list)
{
if (KeyPhrase.Affinty > config.MinAffintyScore)
{
_ctx.KeyPhrase.Add(new KeyPhrase
{
Affinty = KeyPhrase.Affinty,
Text = KeyPhrase.Text,
JobPostingId = newJobPosting.Id
});
}
_logger.LogInformation($"item.Affinty {KeyPhrase.Affinty}");
_logger.LogInformation($"item.Text {KeyPhrase.Text}");
}
await _ctx.SaveChangesAsync();
foreach (var item in okRemoteJob.Tags)
{
if (_ctx.Tags.Any(x => x.Text.Trim() == item.Trim()))
{
var tagFromDB = _ctx.Tags.Where(x => x.Text.Trim() == item.Trim()).FirstOrDefault();
newJobPosting.Tags.Add(tagFromDB);
}
else
{
newJobPosting.Tags.Add(new Models.Entity.Tag()
{
Text = item.Trim()
});
}
}
await _ctx.SaveChangesAsync();
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to deserialize");
}
}
Thread.Sleep(config.OkRemoteTimeBetweenTags);
}
}
public async Task RunAtTimeOfAsync(DateTime now)
{
var config = _ctx.JobGettingConfig.FirstOrDefault();
if (config == null)
{
return;
}
HttpClient client = new HttpClient();
var httpResult = await client.GetAsync("https://remotive.io/api/remote-jobs");
if (httpResult.IsSuccessStatusCode)
{
var contentString = await httpResult.Content.ReadAsStringAsync();
try
{
var remotiveIO = RemotiveIoData.FromJson(contentString);
foreach (var okRemoteJob in remotiveIO.Jobs)
{
if (string.IsNullOrEmpty(okRemoteJob.Title))
{
continue;
}
if (!_ctx.JobPostings.Any(x =>
x.Title == okRemoteJob.Title && x.Description == okRemoteJob.Description))
{
var Description =
new string(okRemoteJob.Description.Where(c => !char.IsPunctuation(c)).ToArray());
SummaryDTO nltkSummary = await _nltkService.ExtractSummary(Description);
KeyPhrasesWrapperDTO wrapper = await _nltkService.ExtractKeyPhrases(Description);
var newJobPosting = new JobPosting()
{
Title = okRemoteJob.Title,
Description = okRemoteJob.Description,
URL = okRemoteJob.Url.OriginalString,
Company = okRemoteJob.CompanyName,
Location = okRemoteJob.CandidateRequiredLocation,
PostDate = okRemoteJob.PublicationDate.ToString(),
Salary = okRemoteJob.Salary,
JobSource = "RemotiveIo",
CompanyLogoUrl = okRemoteJob.CompanyLogoUrl == null
? ""
: okRemoteJob.CompanyLogoUrl.OriginalString,
Summary = nltkSummary.SummaryText
};
await _ctx.AddAsync(newJobPosting);
await _ctx.SaveChangesAsync();
_logger.LogInformation("List<KeyPhrase> ListKeyPhrase");
foreach (KeyPhraseDTO KeyPhrase in wrapper.rank_list)
{
if (KeyPhrase.Affinty > config.MinAffintyScore)
{
_ctx.KeyPhrase.Add(new KeyPhrase
{
Affinty = KeyPhrase.Affinty,
Text = KeyPhrase.Text,
JobPostingId = newJobPosting.Id
});
}
_logger.LogInformation($"item.Affinty {KeyPhrase.Affinty}");
_logger.LogInformation($"item.Text {KeyPhrase.Text}");
}
await _ctx.SaveChangesAsync();
if (_ctx.Tags.Any(x => x.Text.Trim() == okRemoteJob.Category))
{
var tagFromDb = _ctx.Tags
.FirstOrDefault(x => x.Text.Trim() == okRemoteJob.Category.Trim());
newJobPosting.Tags.Add(tagFromDb);
}
else
{
newJobPosting.Tags.Add(new Models.Entity.Tag()
{
Text = okRemoteJob.Category.Trim()
});
}
await _ctx.SaveChangesAsync();
}
}
}
catch (Exception ex)
{
_logger.Log(LogLevel.Error, "Error Occured int RemotiveIoDataJob", ex);
}
}
else
{
_logger.Log(LogLevel.Error, "httpResult.IsSuccessStatusCode was false");
}
}