public void Upsert(CarAd item)
{
var db = _mongoClient.GetDatabase("Delos");
var collection = db.GetCollection <CarAd>("bayfordHoldenCars");
collection.ReplaceOneAsync(p => p.Id == item.Id, item, new UpdateOptions {
IsUpsert = true
});
}
private static void ProcessItem(string detailsUrl, string imageUri, MongoRepository repository)
{
Logger.Info("Scraping Page {0}", detailsUrl);
var browser = new ScrapingBrowser();
var detailsPage = browser.NavigateToPage(new Uri(detailsUrl));
try
{
var carAd = new CarAd();
var year = detailsPage.Html.SelectSingleNode("//header/div[@class='h1']/span[@itemprop='releaseDate']/text()[1]").InnerText.CleanInnerText();
var make = detailsPage.Html.SelectSingleNode("//header/div[@class='h1']/span[@itemprop='manufacturer']/text()[1]").InnerText.CleanInnerText();
var model = detailsPage.Html.SelectSingleNode("//header/div[@class='h1']/span[@itemprop='model']/text()[1]").InnerText.CleanInnerText();
var trim = detailsPage.Html.SelectSingleNode("//header/div[@class='h1']/span[@itemprop='trim']/text()[1]").InnerText.CleanInnerText();
var transmission = detailsPage.Html.SelectSingleNode("//span[@title='TRANSMISSION']")?.GetNextSibling("span");
var engineData = detailsPage.Html.SelectSingleNode("//span[@title='ENGINE DATA']")?.GetNextSibling("span");
var bodyType = detailsPage.Html.SelectSingleNode("//span[contains(text(), 'Body Description')]")?.NextSibling;
carAd.Title = string.Format("{0} {1} {2} {3}", year, make, model, trim);
carAd.StockNumber = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:stock']").GetAttributeValue("value", "");
carAd.Id = carAd.StockNumber;
carAd.Vin = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:vin']").GetAttributeValue("value", "");
carAd.Make = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:make']").GetAttributeValue("value", "");
carAd.Model = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:model']").GetAttributeValue("value", "");
carAd.Year = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:year']").GetAttributeValue("value", "");
carAd.Price = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:price:asking']").GetAttributeValue("value", "");
carAd.Odometer = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:odometer:mi']").GetAttributeValue("value", "");
carAd.Colour = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:colorcombination:exteriorcolor_1']").GetAttributeValue("value", "");
carAd.Rego = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:reg_plate']")?.GetAttributeValue("value", "");
carAd.Condition = detailsPage.Html.SelectSingleNode("//input[@name='vehicle:buy:reg_plate']")?.GetAttributeValue("value", "");
carAd.ImageUrl = imageUri.Replace("x200", "x650");
carAd.FinalUrl = detailsUrl;
carAd.LastModified = DateTime.UtcNow;
carAd.Condition = detailsPage.Html.SelectSingleNode("//span[@itemprop='itemCondition']/text()[1]").InnerText;
carAd.Transmission = transmission != null ? transmission.InnerText : string.Empty;
carAd.Engine = engineData != null ? engineData.InnerText : string.Empty;
carAd.Body = bodyType != null ? bodyType.InnerText : string.Empty;
repository.Upsert(carAd);
}
catch (Exception ex)
{
Logger.Error(ex, "Error for item {0}", detailsUrl);
}
}