public async Task <ActionResult <JobPortal> > PostJobPortal(JobPortal jobPortal)
{
_context.JobPortal.Add(jobPortal);
await _context.SaveChangesAsync();
return(CreatedAtAction("GetJobPortal", new { id = jobPortal.Id }, jobPortal));
}
public static string Crawl(JobPortal portal, string searchTerm)
{
Logger.LogStart(Logger.MethodName);
RequestConfig requestConfig = RequestConfigProvider.GetRequestConfig(portal, searchTerm);
RestClient client = new RestClient();
client.BaseUrl = requestConfig.BaseUrl;
RestRequest request = new RestRequest();
request.Resource = requestConfig.Resource;
foreach (string key in requestConfig.QueryStringParameters.Keys)
{
request.AddParameter(key, requestConfig.QueryStringParameters[key], ParameterType.UrlSegment);
}
string urlToCrawl = client.BaseUrl.ToString() + request.Resource;
Logger.LogStart("Crawling " + urlToCrawl);
IRestResponse resposne = client.Execute(request);
Logger.LogStop("Crawling finished.");
if (resposne.ErrorException != null)
{
throw resposne.ErrorException;
}
Logger.LogStop(Logger.MethodName);
return resposne.Content;
}
public void WriteTableContentToConsole(JobPortal table)
{
string sql = string.Format("SELECT * FROM {0}", table);
_dbConnection.Open();
SQLiteCommand command = new SQLiteCommand(sql, _dbConnection);
SQLiteDataReader reader = command.ExecuteReader();
Console.WriteLine("TABLE: {0}", table);
while (reader.Read())
{
Console.WriteLine("Id: {0}", reader["Id"]);
Console.WriteLine("DateTime: {0}", reader["DateTime"]);
foreach (Technology value in _technologyValues)
{
string technology = value.ToString();
Console.WriteLine("{0}: {1}", technology, reader[technology]);
}
Console.WriteLine();
}
_dbConnection.Close();
}
public async Task <IActionResult> PutJobPortal(int id, JobPortal jobPortal)
{
if (id != jobPortal.Id)
{
return(BadRequest());
}
_context.Entry(jobPortal).State = EntityState.Modified;
try
{
await _context.SaveChangesAsync();
}
catch (DbUpdateConcurrencyException)
{
if (!JobPortalExists(id))
{
return(NotFound());
}
else
{
throw;
}
}
return(NoContent());
}
// TODO: try catch
public void StoreData(JobPortal portal, IDictionary<Technology, uint> jobOffers)
{
Logger.LogStart(Logger.MethodName);
if (jobOffers == null || jobOffers.Count == 0)
{
Logger.LogWarning("No job offers were passed to method.");
Logger.LogStop(Logger.MethodName);
return;
}
if (!TableExists(portal))
{
Logger.LogInfo(string.Format("Tabel '{0}' does not exist, yet.", portal));
CreateTable(portal);
}
string columns = CreateColumnsForInsert();
string values = CreateColumnValues(jobOffers);
string sql = string.Format("INSERT INTO {0} ({1}) VALUES({2})", portal, columns, values);
_dbConnection.Open();
SQLiteCommand command = new SQLiteCommand(sql, _dbConnection);
command.ExecuteNonQuery();
Logger.LogInfo(string.Format("Data stored into '{0}'.", portal));
_dbConnection.Close();
Logger.LogStop(Logger.MethodName);
}
public void DeleteRow(JobPortal portal, int id)
{
string tableName = portal.ToString();
string sql = string.Format("DELETE FROM {0} WHERE id={1}", tableName, id);
_dbConnection.Open();
SQLiteCommand command = new SQLiteCommand(sql, _dbConnection);
command.ExecuteNonQuery();
_dbConnection.Close();
}
private static IDictionary<Technology, uint> GetJobOffers(JobPortal portal)
{
Logger.LogStart(Logger.MethodName);
Logger.LogToConsole(string.Format("Crawling data for {0}", portal));
IDictionary<Technology, uint> jobOffers = new Dictionary<Technology, uint>();
foreach (Technology technology in Technologies)
{
string term = HandleSpecialCases(portal, SearchTermProvider.GetSearchTerms(technology));
string content = null;
try
{
content = Crawler.Crawl(portal, term);
}
catch (Exception ex)
{
string errorMsg = string.Format("Unable to crawl '{0}' from {1}.", term, portal);
errorMsg += "\nDetail: " + ex.Message;
Logger.LogError(ex, errorMsg, Logger.MethodName); // TODO: send email
continue;
}
uint jobsCount = 0;
try
{
jobsCount = Parser.ParseJobOffersCount(portal, content);
}
catch (Exception ex)
{
string errorMsg = string.Format("Unable to parse job count for '{0}' from {1}.", term, portal);
errorMsg += "\nDetail: " + ex.Message;
Logger.LogError(ex, errorMsg, Logger.MethodName); // TODO: send email
continue;
}
jobOffers.Add(technology, jobsCount);
Logger.LogToConsole(string.Format("For {0}, there is currently {1} jobs.", term, jobsCount));
int randomDelay = RequestDelay();
Thread.Sleep(randomDelay);
}
Logger.LogStop(Logger.MethodName);
return jobOffers;
}
public static RequestConfig GetRequestConfig(JobPortal portal, string searchTerm)
{
switch(portal)
{
case JobPortal.Profesia:
return GetProfesiaRequestConfig(searchTerm);
case JobPortal.CareersStackOverflow:
return GetCareersStackOverflowRequestConfig(searchTerm);
case JobPortal.Karriere:
return GetKarriereRequestConfig(searchTerm);
case JobPortal.CWJobs:
return GetCWJobsRequestConfig(searchTerm);
default:
return null;
}
}
private static string GetJobOffersTextToParse(JobPortal portal, string innerText)
{
switch (portal)
{
case JobPortal.Profesia:
int leftBracket = innerText.IndexOf("(");
int rightBracket = innerText.IndexOf(")");
return innerText.Substring(leftBracket + 1, rightBracket - (leftBracket + 1));
case JobPortal.CareersStackOverflow:
case JobPortal.Karriere:
case JobPortal.CWJobs:
innerText = innerText.Trim();
return GetTextCountForCwJobs(innerText);
default:
return null;
}
}
public static uint ParseJobOffersCount(JobPortal portal, string htmlContent)
{
Logger.LogStart(Logger.MethodName);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(htmlContent);
string xPath = GetXPath(portal);
HtmlNodeCollection jobOffersElement = doc.DocumentNode.SelectNodes(xPath);
if (jobOffersElement == null || jobOffersElement.Count != 1)
{
string jobOfferElementsCount = jobOffersElement == null ? "null" : jobOffersElement.Count.ToString(CultureInfo.InvariantCulture);
throw new Exception(string.Format("For portal '{0}' found none or more than 1 job offer elements. Job offer elements: " + jobOfferElementsCount, portal));
}
HtmlNode jobOffers = jobOffersElement[0];
Logger.LogInfo("Job offers inner text: " + jobOffers.InnerText);
string texCount = GetJobOffersTextToParse(portal, jobOffers.InnerText);
if (texCount == null)
{
Logger.LogWarning("Cannot get job count from inner text.");
return 0; // assume there is 0 jobs
}
Logger.LogInfo("Job offers text to parse: " + texCount);
uint count;
if (uint.TryParse(texCount, out count))
{
Logger.LogStop(Logger.MethodName);
return count;
}
Logger.LogStop(Logger.MethodName);
throw new Exception(string.Format("Cannot parse integer from '{0}'.", jobOffers.InnerText));
}
private static string GetXPath(JobPortal portal)
{
switch (portal)
{
case JobPortal.Profesia:
return ProfesiaJobOffersXPath;
case JobPortal.CareersStackOverflow:
return CareersStackOverflowJobOffersXPath;
case JobPortal.Karriere:
return KarriereJobOffersXPath;
case JobPortal.CWJobs:
return CwJobsJobOffersXPath;
default:
return null;
}
}
private static string HandleSpecialCases(JobPortal portal, string searchTerm)
{
switch (portal)
{
case JobPortal.Profesia:
return searchTerm.Equals("Objective-C") ? "Objective C" : searchTerm;
default:
return searchTerm;
}
}
// TODO: try catch
private bool TableExists(JobPortal table)
{
string tableName = table.ToString();
string sql = string.Format(
"SELECT count(*) AS count FROM sqlite_master WHERE type='table' AND name='{0}'", tableName);
_dbConnection.Open();
SQLiteCommand command = new SQLiteCommand(sql, _dbConnection);
SQLiteDataReader reader = command.ExecuteReader();
long count = 0;
while (reader.Read())
{
count = (long)reader["count"];
}
_dbConnection.Close();
return count == 1;
}
// TODO: try catch
private void CreateTable(JobPortal table)
{
Logger.LogStart(Logger.MethodName);
string tableName = table.ToString();
string columnsDefinition = CreateColumnsDefinition();
string sql = string.Format(
"CREATE TABLE {0} (" +
"Id INTEGER PRIMARY KEY NOT NULL, " +
"DateTime TEXT NOT NULL, " +
"{1})", tableName, columnsDefinition);
_dbConnection.Open();
SQLiteCommand command = new SQLiteCommand(sql, _dbConnection);
command.ExecuteNonQuery();
Logger.LogInfo(string.Format("Tabel '{0}' created.", table));
_dbConnection.Close();
Logger.LogStop(Logger.MethodName);
}
public StepDefinitions(IWebDriver driver)
{
_page = new JobPortal(driver);
}
