public void Do(JobPortals jobPortal, IUrlFilter filter)
{
Log.Information($"Scraping urls for: {jobPortal.GetDescription()}");
var urls = ExtractPageUrls();
filter.Apply(ref urls);
UpdateUrls(urls);
}
public void ProcessSalaries(JobPortals jobPortal)
{
Log.Information($"processing Salaries for {jobPortal.GetDescription()}");
var jobUrls = _unitOfWork.JobUrlRepository.GetAll();
var jobsWithSalaries = jobUrls.Where(j => !String.IsNullOrEmpty(j.SalaryText)).ToList();
foreach (var jobUrl in jobsWithSalaries)
{
var salary = _analyser.GetSalary(jobUrl.SalaryText);
salary.JObUrlId = jobUrl.Id;
_unitOfWork.SalaryRepository.Upsert(salary, salary.Id);
Log.Information($"Updating salary for: {jobPortal.GetDescription()}");
_unitOfWork.SaveChanges();
}
}
public void ScrapePageInfos(string elementId, JobPortals jobPortal)
{
Log.Information($"Scraping page infos for {jobPortal.GetDescription()}");
var jobPortalsUrls = _unitOfWork.JobUrlRepository.GetAll()
.Where(j => j.JobPortalId == (int)jobPortal).ToList();
foreach (var url in jobPortalsUrls)
{
var html = ScrapeJobHtml(url.Url, elementId);
Log.Information($"Updating HtmlCode for Job Url {url.Id}");
var jobInfo = new JobInfo()
{
HtmlCode = html,
JobUrlId = url.Id
};
_unitOfWork.JobInfoRepository.Upsert(jobInfo, jobInfo.Id);
_unitOfWork.SaveChanges();
}
}
public void Do(JobPortals jobPortal)
{
Log.Information($"Scraping urls for: {jobPortal.GetDescription()}");
UpdateUrls(ExtractPageUrls());
}
