public void RemoveClosingDateWithDay()
{
var parsedClosingDate = "Closing date: Sunday 23 April 2017";
var result = new RemoveDuplicateTextFormatter(parsedClosingDate).FormatHtml(Properties.Resources.ClosingDateInBody1);
Assert.AreEqual(Properties.Resources.ClosingDateInBody1.Replace("<p><span><span>Closing date: Sunday 23 April 2017</span></span></p>", String.Empty), result);
}
/// <summary>
/// Parses a job.
/// </summary>
/// <param name="sourceData">The source data for the job.</param>
/// <returns></returns>
public async Task <Job> ParseJob(string sourceData, string jobId)
{
try
{
HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
htmlDocument.OptionFixNestedTags = true;
htmlDocument.LoadHtml(sourceData);
// ParseErrors is an ArrayList containing any errors from the Load statement
if (htmlDocument.ParseErrors != null && htmlDocument.ParseErrors.Count() > 0)
{
foreach (var error in htmlDocument.ParseErrors)
{
var exception = new HttpParseException("Unable to parse job HTML from TalentLink");
exception.Data.Add("Job ID", jobId);
exception.Data.Add("Type of error", error.Code);
exception.Data.Add("Reason", error.Reason);
exception.Data.Add("Line", error.Line);
exception.Data.Add("Position on line", error.LinePosition);
exception.Data.Add("Source HTML", error.SourceText);
exception.ToExceptionless().Submit();
}
return(null);
}
else
{
if (htmlDocument.DocumentNode != null)
{
var job = new Job();
job.JobTitle = ParseValueFromElementById(htmlDocument, "h3", "JDText-Title");
job.Reference = ParseValueFromElementById(htmlDocument, "span", "JDText-Param2");
if (!String.IsNullOrEmpty(job.Reference))
{
job.JobTitle = job.JobTitle.Replace(" (" + job.Reference + ")", String.Empty);
}
job.Locations.Add(ParseValueFromElementById(htmlDocument, "span", "JDText-Param3"));
job.Organisation = ParseValueFromElementById(htmlDocument, "span", "JDText-Param4");
job.Department = ParseValueFromElementById(htmlDocument, "span", "JDText-Param5");
job.ContractType = ParseValueFromElementById(htmlDocument, "span", "JDText-Param6");
job.JobType = ParseValueFromElementById(htmlDocument, "span", "JDText-Param7");
job.Salary = await _salaryParser.ParseSalary(htmlDocument.DocumentNode.OuterHtml);
DateTime closingDate;
DateTime.TryParse(ParseValueFromElementById(htmlDocument, "span", "JDText-Param9"), new CultureInfo("en-GB"), DateTimeStyles.AssumeLocal, out closingDate);
job.ClosingDate = closingDate;
job.DatePublished = DateTime.UtcNow;
var agilityPackFormatters = new IHtmlAgilityPackHtmlFormatter[]
{
new RemoveUnwantedAttributesFormatter(new string[] { "style" }),
new ReplaceElementNameFormatter("h5", "h2"),
new RemoveUnwantedNodesFormatter(new[] { "u" }),
new RemoveElementByNameAndContentFormatter("h2", "Job Details"),
new RemoveElementsWithNoContentFormatter(new[] { "strong", "p" }),
new TruncateLongLinksFormatter(new HtmlLinkFormatter()),
new EmbeddedYouTubeVideosFormatter()
};
foreach (var formatter in agilityPackFormatters)
{
formatter.FormatHtml(htmlDocument);
}
// JDText-Field4 is additional info (ie small print) and JDText-Field6 is information for redeployees (ie more small print)
var additionalInfo = ParseValueFromElementById(htmlDocument, "span", "JDText-Field4") + Environment.NewLine +
ParseValueFromElementById(htmlDocument, "span", "JDText-Field6");
if (!String.IsNullOrEmpty(additionalInfo))
{
additionalInfo = ApplyStringFormatters(additionalInfo);
job.AdditionalInformationHtml = new HtmlString(additionalInfo);
}
var equalOpportunities = ParseValueFromElementById(htmlDocument, "span", "JDText-Field5");
if (!String.IsNullOrEmpty(equalOpportunities))
{
equalOpportunities = ApplyStringFormatters(equalOpportunities);
job.EqualOpportunitiesHtml = new HtmlString(equalOpportunities);
}
var parsedHtml = ParseValueFromElementById(htmlDocument, "div", "JD-Field1") + Environment.NewLine +
ParseValueFromElementById(htmlDocument, "div", "JD-Field2") + Environment.NewLine +
ParseValueFromElementById(htmlDocument, "div", "JD-Documents");
parsedHtml = ApplyStringFormatters(parsedHtml);
parsedHtml = new RemoveDuplicateTextFormatter("Closing date: " + job.ClosingDate.Value.ToBritishDate()).FormatHtml(parsedHtml);
parsedHtml = new RemoveDuplicateTextFormatter("Closing date: " + job.ClosingDate.Value.ToBritishDateWithDay()).FormatHtml(parsedHtml);
parsedHtml = new RemoveDuplicateTextFormatter("Salary: " + job.Salary.SalaryRange).FormatHtml(parsedHtml);
parsedHtml = new RemoveDuplicateTextFormatter("Salary: " + job.Salary.SalaryRange?.Replace(" to ", " - ")).FormatHtml(parsedHtml);
parsedHtml = new RemoveDuplicateTextFormatter("Contract type: " + job.ContractType).FormatHtml(parsedHtml);
job.AdvertHtml = new HtmlString(parsedHtml);
job.WorkPattern = await _workPatternParser.ParseWorkPattern(parsedHtml);
var applyLink = htmlDocument.DocumentNode.SelectSingleNode($"//div[@id='JD-ActApplyDirect']/a");
if (applyLink != null)
{
job.ApplyUrl = new Uri(HttpUtility.HtmlDecode(applyLink.Attributes["href"].Value), UriKind.RelativeOrAbsolute);
}
return(job);
}
return(null);
}
}
catch (Exception exception)
{
exception.Data.Add("Job ID", jobId);
exception.ToExceptionless().Submit();
return(null);
}
}
