public static void PersistScrapingLog(int newUrlsAdded, string urlsScrapingDuration, int newArticlesScraped, string articlesScrapingDuration, string totalDuration, bool sendEmail) { using (UsedCarsDbContext db = new UsedCarsDbContext()) { int totalValidRecords = db.Usedcars.Count(c => c.Scraped == (int)ProcessingStatus.Processed && c.Brand != null && c.Body != null && c.Gearbox != null && c.Fuel != null && c.EngineCapacity != null && c.Mileage != null && c.Price != null); db.Scrapinglogs.Add(new ScrapingLogs() { CreatedOn = DateTime.Now, NewUrlsAdded = newUrlsAdded, UrlScrapingDuration = urlsScrapingDuration, NewArticlesScraped = newArticlesScraped, ArticlesScrapingDuration = articlesScrapingDuration, TotalDuration = totalDuration, TotalValidRecords = totalValidRecords }); db.SaveChanges(); if (sendEmail) { string body = "New URLs scraped: " + newUrlsAdded + ", duration: " + urlsScrapingDuration + Environment.NewLine + "New articles scraped " + newArticlesScraped + ", duration: " + articlesScrapingDuration + Environment.NewLine + "Total time: " + totalDuration + Environment.NewLine + "Total valid records: " + totalValidRecords; CommonUtils.SendEmail(config["EmailTo"], "Scraped cars report", body); } } }
public static void InsertUrls(HashSet <string> urls) { using (UsedCarsDbContext db = new UsedCarsDbContext()) { foreach (var url in urls) { if (!db.Usedcars.Any(l => l.Url == url)) { db.Usedcars.Add(new UsedCarModel() { Url = url, Scraped = 0 }); Interlocked.Increment(ref Program.NewUrlsCountThreadSafe); } } try { db.SaveChanges(); } catch (Exception e) { Log.Debug(e, "Error trying to insert URLs."); } } }
public void BulkUpdateUsedCars(List <UsedCarModel> cars) { using (UsedCarsDbContext db = new UsedCarsDbContext()) { //db.UsedCarModel.AttachRange(cars); db.Usedcars.UpdateRange(cars); db.SaveChanges(); } }
public static void GenerateDropdownValuesJson() { using (UsedCarsDbContext db = new UsedCarsDbContext()) { var brands = db.Usedcars.GroupBy(c => c.Brand) .Select(n => new { marca = n.Key, count = n.Count() }) .Where(n => n.count >= MARCI_MIN_COUNT && n.marca != null && !BrandsIgnored.Contains(n.marca)) .OrderBy(n => n.marca) .Select(x => x.marca) .ToList(); Dictionary <string, List <string> > models = new Dictionary <string, List <string> >(); foreach (var brand in brands) { var modeleMarca = db.Usedcars.Where(c => c.Brand == brand) .GroupBy(g => g.Model) .Select(n => new { model = n.Key, count = n.Count() }) .Where(n => n.count >= MODEL_MIN_COUNT && n.model != null && !ModelsIgnored.Contains(n.model)) .OrderBy(n => n.model) .Select(x => x.model) .ToList(); models.Add(brand, modeleMarca); } var fuels = new List <DropdownItem>(); fuels.Add(new DropdownItem("Benzina", "Benzină")); fuels.Add(new DropdownItem("Diesel", "Diesel")); var gears = new List <DropdownItem>(); gears.Add(new DropdownItem("Manuala", "Manuală")); gears.Add(new DropdownItem("Automata", "Automată")); Dictionary <string, object> dropdownValues = new Dictionary <string, object>(); dropdownValues.Add("carBrands", brands); dropdownValues.Add("brandsModels", models); dropdownValues.Add("fuels", fuels); dropdownValues.Add("gears", gears); //TODO caroserii var dropdownValuesJson = JsonSerializer.Serialize(dropdownValues); WriteTextToFile("dropdownValues.json", dropdownValuesJson); } }
public static void DeleteInvalidLinks() { try { using (UsedCarsDbContext db = new UsedCarsDbContext()) { db.Usedcars.RemoveRange(db.Usedcars.Where(c => c.Scraped == (int)ProcessingStatus.Invalid)); db.SaveChanges(); } } catch (Exception e) { Log.Error("Exception in DeleteInvalidLinks", e); } }
public void BulkInsert(List <UsedCarModel> cars) { try { using (UsedCarsDbContext db = new UsedCarsDbContext()) { db.Usedcars.AddRange(cars); db.SaveChanges(); } } catch (Exception e) { Log.Error("Exception in BulkInsert", e); } }
public static void ScrapeAdverts(Scraper scraper) { using (UsedCarsDbContext db = new UsedCarsDbContext()) { while (true) { List <UsedCarModel> cars = db.Usedcars.Where(c => c.Scraped == (int)ProcessingStatus.Unprocessed && scraper.BaseUrls.Any(u => c.Url.Contains(u))) .Take(1000).ToList(); if (cars == null || cars.Count == 0) { break; } for (int i = 0; i < cars.Count; i++) { UsedCarModel tempCar = cars.ElementAt(i); scraper.ScrapeAdvert(tempCar); } db.SaveChanges(); } } }
public UserRepository(UsedCarsDbContext context) { _dbContext = context; }
/// <summary> /// Initializes a new instance of the <see cref="AdvertisementRepository"/> class. /// </summary> public AdvertisementRepository(UsedCarsDbContext context) { _dbContext = context; }