public static void PersistScrapingLog(int newUrlsAdded, string urlsScrapingDuration, int newArticlesScraped, string articlesScrapingDuration, string totalDuration, bool sendEmail)
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
int totalValidRecords = db.Usedcars.Count(c => c.Scraped == (int)ProcessingStatus.Processed && c.Brand != null && c.Body != null && c.Gearbox != null && c.Fuel != null && c.EngineCapacity != null && c.Mileage != null && c.Price != null);
db.Scrapinglogs.Add(new ScrapingLogs()
{
CreatedOn = DateTime.Now,
NewUrlsAdded = newUrlsAdded,
UrlScrapingDuration = urlsScrapingDuration,
NewArticlesScraped = newArticlesScraped,
ArticlesScrapingDuration = articlesScrapingDuration,
TotalDuration = totalDuration,
TotalValidRecords = totalValidRecords
});
db.SaveChanges();
if (sendEmail)
{
string body = "New URLs scraped: " + newUrlsAdded + ", duration: " + urlsScrapingDuration + Environment.NewLine +
"New articles scraped " + newArticlesScraped + ", duration: " + articlesScrapingDuration + Environment.NewLine +
"Total time: " + totalDuration + Environment.NewLine +
"Total valid records: " + totalValidRecords;
CommonUtils.SendEmail(config["EmailTo"], "Scraped cars report", body);
}
}
}
public static void InsertUrls(HashSet <string> urls)
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
foreach (var url in urls)
{
if (!db.Usedcars.Any(l => l.Url == url))
{
db.Usedcars.Add(new UsedCarModel()
{
Url = url,
Scraped = 0
});
Interlocked.Increment(ref Program.NewUrlsCountThreadSafe);
}
}
try
{
db.SaveChanges();
}
catch (Exception e)
{
Log.Debug(e, "Error trying to insert URLs.");
}
}
}
public void BulkUpdateUsedCars(List <UsedCarModel> cars)
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
//db.UsedCarModel.AttachRange(cars);
db.Usedcars.UpdateRange(cars);
db.SaveChanges();
}
}
public static void GenerateDropdownValuesJson()
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
var brands = db.Usedcars.GroupBy(c => c.Brand)
.Select(n => new
{
marca = n.Key,
count = n.Count()
})
.Where(n => n.count >= MARCI_MIN_COUNT && n.marca != null && !BrandsIgnored.Contains(n.marca))
.OrderBy(n => n.marca)
.Select(x => x.marca)
.ToList();
Dictionary <string, List <string> > models = new Dictionary <string, List <string> >();
foreach (var brand in brands)
{
var modeleMarca = db.Usedcars.Where(c => c.Brand == brand)
.GroupBy(g => g.Model)
.Select(n => new
{
model = n.Key,
count = n.Count()
})
.Where(n => n.count >= MODEL_MIN_COUNT && n.model != null && !ModelsIgnored.Contains(n.model))
.OrderBy(n => n.model)
.Select(x => x.model)
.ToList();
models.Add(brand, modeleMarca);
}
var fuels = new List <DropdownItem>();
fuels.Add(new DropdownItem("Benzina", "Benzină"));
fuels.Add(new DropdownItem("Diesel", "Diesel"));
var gears = new List <DropdownItem>();
gears.Add(new DropdownItem("Manuala", "Manuală"));
gears.Add(new DropdownItem("Automata", "Automată"));
Dictionary <string, object> dropdownValues = new Dictionary <string, object>();
dropdownValues.Add("carBrands", brands);
dropdownValues.Add("brandsModels", models);
dropdownValues.Add("fuels", fuels);
dropdownValues.Add("gears", gears);
//TODO caroserii
var dropdownValuesJson = JsonSerializer.Serialize(dropdownValues);
WriteTextToFile("dropdownValues.json", dropdownValuesJson);
}
}
public static void DeleteInvalidLinks()
{
try
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
db.Usedcars.RemoveRange(db.Usedcars.Where(c => c.Scraped == (int)ProcessingStatus.Invalid));
db.SaveChanges();
}
}
catch (Exception e)
{
Log.Error("Exception in DeleteInvalidLinks", e);
}
}
public void BulkInsert(List <UsedCarModel> cars)
{
try
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
db.Usedcars.AddRange(cars);
db.SaveChanges();
}
}
catch (Exception e)
{
Log.Error("Exception in BulkInsert", e);
}
}
public static void ScrapeAdverts(Scraper scraper)
{
using (UsedCarsDbContext db = new UsedCarsDbContext())
{
while (true)
{
List <UsedCarModel> cars = db.Usedcars.Where(c => c.Scraped == (int)ProcessingStatus.Unprocessed && scraper.BaseUrls.Any(u => c.Url.Contains(u)))
.Take(1000).ToList();
if (cars == null || cars.Count == 0)
{
break;
}
for (int i = 0; i < cars.Count; i++)
{
UsedCarModel tempCar = cars.ElementAt(i);
scraper.ScrapeAdvert(tempCar);
}
db.SaveChanges();
}
}
}
public UserRepository(UsedCarsDbContext context)
{
_dbContext = context;
}
/// <summary>
/// Initializes a new instance of the <see cref="AdvertisementRepository"/> class.
/// </summary>
public AdvertisementRepository(UsedCarsDbContext context)
{
_dbContext = context;
}
