public UsedCarViewModel CreateFrom(UsedCarModel model) { int yearTemp; return(new UsedCarViewModel() { Year = int.TryParse(model.Year, out yearTemp) ? yearTemp : default(int), Brand = model.Brand, Model = model.Model, Body = model.Body, Gearbox = model.Gearbox, Fuel = model.Fuel, Mileage = model.Mileage, EngineCapacity = model.EngineCapacity, Price = model.Price }); }
public async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req, ILogger log) { log.LogInformation("C# HTTP trigger function processed a request."); string requestBody = await new StreamReader(req.Body).ReadToEndAsync(); UsedCarPricePrediction prediction = null; try { UsedCarModel data = JsonConvert.DeserializeObject <UsedCarModel>(requestBody); UsedCarMlModel mlModel = data.ToMlModel(); prediction = _predictionEnginePool.Predict(modelName: "UsedCarsPricePredictionModel", example: mlModel); } catch (Exception e) { log.LogError(e, $"Error trying to estimate the price for {requestBody}"); } return(new OkObjectResult(prediction?.Price)); }
public override UsedCarModel ScrapeAdvert(UsedCarModel usedCar) { HttpStatusCode status = HttpStatusCode.NotFound; string html = HttpUtils.DownloadPage(usedCar.Url, ref status); if (status == HttpStatusCode.OK) { HtmlDocument htmlDoc = new HtmlDocument(); htmlDoc.OptionFixNestedTags = true; htmlDoc.LoadHtml(html); // ParseErrors is an ArrayList containing any errors from the Load statement if (htmlDoc.ParseErrors != null && htmlDoc.ParseErrors.Count() > 0) { Log.Debug("Error trying to scrape " + usedCar.Url); foreach (var error in htmlDoc.ParseErrors) { Log.Debug(error.ToString()); } } usedCar = ParseDocument(usedCar, htmlDoc, html); if (htmlDoc.DocumentNode != null) { usedCar.Scraped = (int)ProcessingStatus.Processed; } } else { usedCar.Scraped = (int)ProcessingStatus.Invalid; } usedCar.LastModified = DateTime.Now; Interlocked.Increment(ref Program.UrlsScraped); return(usedCar); }
public static void ScrapeAdverts(Scraper scraper) { using (UsedCarsDbContext db = new UsedCarsDbContext()) { while (true) { List <UsedCarModel> cars = db.Usedcars.Where(c => c.Scraped == (int)ProcessingStatus.Unprocessed && scraper.BaseUrls.Any(u => c.Url.Contains(u))) .Take(1000).ToList(); if (cars == null || cars.Count == 0) { break; } for (int i = 0; i < cars.Count; i++) { UsedCarModel tempCar = cars.ElementAt(i); scraper.ScrapeAdvert(tempCar); } db.SaveChanges(); } } }
public override UsedCarModel ScrapeAdvert(UsedCarModel usedCar) { HttpStatusCode status = HttpStatusCode.NotFound; string html = HttpUtils.DownloadPage(usedCar.Url, ref status); if (status == HttpStatusCode.OK) { HtmlDocument htmlDoc = new HtmlDocument(); htmlDoc.OptionFixNestedTags = true; htmlDoc.LoadHtml(html); if (htmlDoc.DocumentNode != null) { try { usedCar.Title = htmlDoc.DocumentNode.SelectSingleNode("//h1")?.FirstChild.InnerText; usedCar.Title = HtmlUtils.SanitizeString(usedCar.Title); var descriere = htmlDoc.DocumentNode.Descendants().Where(n => n.Name == "div" && n.HasClass("offer-description__description")).FirstOrDefault()?.InnerText; usedCar.Description = HtmlUtils.SanitizeString(descriere); //Features usedCar.OferitDe = ScrapeAdvertFeature("Oferit de", htmlDoc); usedCar.Model = ScrapeAdvertFeature("Model", htmlDoc); usedCar.Brand = ScrapeAdvertFeature("Brand", htmlDoc); usedCar.Year = ScrapeAdvertFeature("Anul fabricatiei", htmlDoc); usedCar.Fuel = ScrapeAdvertFeature("Fuel", htmlDoc); usedCar.Gearbox = ScrapeAdvertFeature("Cutie de viteze", htmlDoc); usedCar.Body = ScrapeAdvertFeature("Body", htmlDoc); usedCar.Color = ScrapeAdvertFeature("Color", htmlDoc); usedCar.Stare = FixCarState(ScrapeAdvertFeature("Stare", htmlDoc)); float capacitateMotorTemp; bool capacitateWasParsed = float.TryParse(ScrapeAdvertFeature("Capacitate cilindrica", htmlDoc, removeNonNumericCharacters: true, capacitateMotor: true), out capacitateMotorTemp); if (capacitateWasParsed) { usedCar.EngineCapacity = capacitateMotorTemp; } float rulajTemp; bool rulajWasParsed = float.TryParse(ScrapeAdvertFeature("Km", htmlDoc, removeNonNumericCharacters: true), out rulajTemp); if (rulajWasParsed) { usedCar.Mileage = rulajTemp; } //Price string pret = htmlDoc.DocumentNode.Descendants() .Where(n => n.Name == "span" && n.HasClass("offer-price__number")) .FirstOrDefault()?.InnerText; pret = HtmlUtils.RemoveNonNumericCharacters(pret); float pretTemp; bool pretWasParsed = float.TryParse(pret, out pretTemp); if (pretWasParsed) { usedCar.Price = pretTemp; } usedCar.Scraped = (int)ProcessingStatus.Processed; } catch (Exception e) { Log.Debug(e, "Error trying to scrape " + usedCar.Url); return(usedCar); } } } else { usedCar.Scraped = (int)ProcessingStatus.Invalid; } usedCar.LastModified = DateTime.Now; Interlocked.Increment(ref Program.UrlsScraped); return(usedCar); }
private UsedCarModel ParseDocument(UsedCarModel usedCar, HtmlDocument htmlDoc, string html) { try { var pret = htmlDoc.DocumentNode.Descendants("strong").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("pricelabel__value")).FirstOrDefault()?.InnerText; if (pret != null) { float pretParsed; bool parsed = float.TryParse(HtmlUtils.RemoveNonNumericCharacters(pret), out pretParsed); if (parsed) { usedCar.Price = pretParsed; } } //an fabricatie var anDeFabricatie = GetDetail(htmlDoc, "An de fabricatie"); usedCar.Year = anDeFabricatie; usedCar.Title = HtmlUtils.RemoveNewLineCharacters(htmlDoc.DocumentNode.SelectSingleNode("//h1")?.InnerText); usedCar.Description = HtmlUtils.RemoveNewLineCharacters(htmlDoc.GetElementbyId("textContent")?.InnerText); var oferitDe = GetDetail(htmlDoc, "Oferit de"); usedCar.OferitDe = oferitDe; //model var model = GetDetail(htmlDoc, "Model"); usedCar.Model = model; //combustibil var combustibil = GetDetail(htmlDoc, "Fuel"); usedCar.Fuel = combustibil = GetDetail(htmlDoc, "Fuel"); //caroserie var caroserie = GetDetail(htmlDoc, "Body"); usedCar.Body = caroserie != null ? caroserie : null; //stare var stare = GetDetail(htmlDoc, "Stare"); usedCar.Stare = stare; //marca var marca = GetDetail(htmlDoc, "Brand"); usedCar.Brand = marca; //culoare var culoare = GetDetail(htmlDoc, "Color"); usedCar.Color = culoare; //Cutie de viteze var cutie = GetDetail(htmlDoc, "Cutie de viteze"); usedCar.Gearbox = cutie; //Turnover var rulaj = GetDetail(htmlDoc, "Turnover", true); if (rulaj != null) { usedCar.Mileage = float.Parse(rulaj); } //Capacitate motor var capacitate = GetDetail(htmlDoc, "Capacitate motor", true); if (capacitate != null) { usedCar.EngineCapacity = float.Parse(capacitate); } return(usedCar); } catch (Exception e) { Console.WriteLine("Error trying to scrape " + usedCar.Url); Console.WriteLine(e.Message); Log.Debug(e, "Error trying to scrape " + usedCar.Url); Log.Information("HTML *************"); Log.Information(html); Log.Information(Environment.NewLine); Log.Information(Environment.NewLine); return(usedCar); } }
public abstract UsedCarModel ScrapeAdvert(UsedCarModel car);
public ActionResult UsedCarInventory() { IRepository rep = new Repository(); UsedCarModel uc = new UsedCarModel(); DataTable dt = rep.getUsedCarsData("none", "none"); //for full list:- List <CarsData> carsdata = new List <CarsData>(); carsdata = DBList.CovertDataTableToList <CarsData>(dt); uc.carsdata = carsdata; //for car years filter:- List <CarYear> caryears = new List <CarYear>(); caryears = DBList.CovertDataTableToList <CarYear>(dt); List <string> caryearsstring = new List <string>(); foreach (var item in caryears) { string items; items = item.Year.ToString(); caryearsstring.Add(items); } List <int> caryearsint = new List <int>(); caryearsint = caryearsstring.Select(int.Parse).ToList(); List <int> distinctcaryears = new List <int>(); distinctcaryears = caryearsint.Distinct().ToList(); distinctcaryears.Sort(); uc.caryears = new List <SelectListItem>(); uc.caryears.Add(new SelectListItem() { Text = "None", Value = "none" }); foreach (var item in distinctcaryears) { SelectListItem si = new SelectListItem() { Value = item.ToString(), Text = item.ToString() }; uc.caryears.Add(si); } //for makes filter:- List <CarMake> carmake = new List <CarMake>(); carmake = DBList.CovertDataTableToList <CarMake>(dt); List <string> carmakestring = new List <string>(); foreach (var item in carmake) { string items; items = item.Make.ToString(); carmakestring.Add(items); } List <string> distinctcarmake = new List <string>(); distinctcarmake = carmakestring.Distinct().ToList(); distinctcarmake.Sort(); uc.carmakes = new List <SelectListItem>(); uc.carmakes.Add(new SelectListItem() { Text = "None", Value = "none" }); foreach (var item in distinctcarmake) { SelectListItem si = new SelectListItem() { Value = item, Text = item }; uc.carmakes.Add(si); } return(View(uc)); }