public static string GetDescription(this JobPortals val)
{
DescriptionAttribute[] attributes = (DescriptionAttribute[])val
.GetType()
.GetField(val.ToString())
.GetCustomAttributes(typeof(DescriptionAttribute), false);
return(attributes.Length > 0 ? attributes[0].Description : string.Empty);
}
public void Do(JobPortals jobPortal, IUrlFilter filter)
{
Log.Information($"Scraping urls for: {jobPortal.GetDescription()}");
var urls = ExtractPageUrls();
filter.Apply(ref urls);
UpdateUrls(urls);
}
public IParser BuildParser(JobPortals website)
{
switch (website)
{
case JobPortals.CvOnline:
return(new CvOnlineParser());
default:
throw new Exception("There is no parser for this website");
}
}
public IAnalyser BuildAnalyser(JobPortals website)
{
switch (website)
{
case JobPortals.CvOnline:
return(new BaseAnalyser());
case JobPortals.CvBankas:
return(new BaseAnalyser());
case JobPortals.CvLt:
return(new BaseAnalyser());
default:
throw new Exception("There is no Analyser for this website");
}
}
public IScraper BuildScraper(JobPortals website)
{
switch (website)
{
case JobPortals.CvOnline:
return(new CvOnlineScraper());
case JobPortals.CvBankas:
return(new CvBankasScraper());
case JobPortals.CvLt:
return(new CvLtScraper());
default:
throw new Exception("There is no Scraper for this website");
}
}
public void ProcessSalaries(JobPortals jobPortal)
{
Log.Information($"processing Salaries for {jobPortal.GetDescription()}");
var jobUrls = _unitOfWork.JobUrlRepository.GetAll();
var jobsWithSalaries = jobUrls.Where(j => !String.IsNullOrEmpty(j.SalaryText)).ToList();
foreach (var jobUrl in jobsWithSalaries)
{
var salary = _analyser.GetSalary(jobUrl.SalaryText);
salary.JObUrlId = jobUrl.Id;
_unitOfWork.SalaryRepository.Upsert(salary, salary.Id);
Log.Information($"Updating salary for: {jobPortal.GetDescription()}");
_unitOfWork.SaveChanges();
}
}
public void ScrapePageInfos(string elementId, JobPortals jobPortal)
{
Log.Information($"Scraping page infos for {jobPortal.GetDescription()}");
var jobPortalsUrls = _unitOfWork.JobUrlRepository.GetAll()
.Where(j => j.JobPortalId == (int)jobPortal).ToList();
foreach (var url in jobPortalsUrls)
{
var html = ScrapeJobHtml(url.Url, elementId);
Log.Information($"Updating HtmlCode for Job Url {url.Id}");
var jobInfo = new JobInfo()
{
HtmlCode = html,
JobUrlId = url.Id
};
_unitOfWork.JobInfoRepository.Upsert(jobInfo, jobInfo.Id);
_unitOfWork.SaveChanges();
}
}
public void Do(JobPortals jobPortal)
{
Log.Information($"Scraping urls for: {jobPortal.GetDescription()}");
UpdateUrls(ExtractPageUrls());
}
public void Do(JobPortals jobPortal)
{
this.ProcessSalaries(jobPortal);
}
public void Do(JobPortals portalName)
{
this.ScrapePageInfos("jobad_content_main", JobPortals.CvBankas);
}
