public Listing( CarsContext context, Cache cache, SourceSite source, string sourceId, string make, string model, int year, int price, int mileage, string location, string description) { Active = true; DateAdded = DateTime.Now; DateLastSeen = DateTime.Now; Source = source; SourceId = sourceId; Year = year; Price = price; Mileage = mileage; Description = description; MakeId = cache.GetOrCreate<Make>(make, () => context.Set<Make>().GetOrCreate(make)); ModelId = cache.GetOrCreate<Model>(model, () => context.Set<Model>().GetOrCreate(model)); County = EnumHelpers.FromString<County>(location); }
public void DumpAll() { using (var context = new CarsContext()) { var data = context.Listings.Include("Make").Include("Model").ToList(); DumpJson("data.js", data); } }
public void LoadCarzoneFromJson_Works() { var scraper = new AllScraper(); scraper.LoadCarzoneFromJson(); using (var context = new CarsContext()) { Assert.IsTrue(context.Listings.ToList().Count > 0); } }
public void LoadCarsIrelandFromJson() { var carsIrelandApi = new CarsIrelandApi(new Requests.WebRequester()); var cache = new Cache(); var dumpFiles = Directory.GetFiles(@"C:\dev\cars\Scraper.Test\bin\Debug", "carsirelandjsondump*"); var listings = dumpFiles .AsParallel() .Select(path => File.ReadAllText(path)) .SelectMany(text => carsIrelandApi.Deserialize(text)) .ToList(); for (int i = 0; i < listings.Count(); i += 100) { using (var context = new CarsContext()) { for (int j = i; j < i + 100 && j < listings.Count(); j++) { var carsIrelandListing = listings[j]; var culture = new CultureInfo("en-IE"); int price; if (!Int32.TryParse(carsIrelandListing.Price, NumberStyles.Currency, culture, out price)) { price = -1; } var listing = new Listing( context, cache, SourceSite.CarsIreland, carsIrelandListing.Ad_Id.ToString(), carsIrelandListing.Make, carsIrelandListing.Model, carsIrelandListing.Reg_Year, price, carsIrelandListing.Mileage ?? -1, carsIrelandListing.Location, carsIrelandListing.Variant); listing.FuelType = EnumHelpers.FromString<FuelType>(carsIrelandListing.Fuel_Type); double engineSize; if (Double.TryParse(carsIrelandListing.Engine_Size, out engineSize)) { listing.EngineSize = (int)(engineSize * 100); } context.Listings.Add(listing); } context.SaveChanges(); } } }
public void LoadCarzoneFromJson() { using (var container = Builder.Build()) { var carzoneApi = new CarzoneApi(container.Resolve<Requests.IRequester>()); var cache = new Cache(); var dumpFiles = Directory.GetFiles(@"C:\dev\cars\Scraper.Test\bin\Debug", "jsondump*"); var carzoneListings = dumpFiles .AsParallel() .Select(path => File.ReadAllText(path)) .SelectMany(text => carzoneApi.Deserialize(text)) .Where(cl => cl.VehicleMake != null && cl.VehicleModel != null) .ToList(); for (int i = 0; i < carzoneListings.Count(); i += 100) { using (var context = new CarsContext()) { for (int j = i; j < i + 100 && j < carzoneListings.Count(); j++) { var carzoneListing = carzoneListings[j]; var listing = new Listing(context, cache, SourceSite.Carzone, carzoneListing.AdvertId.ToString(), carzoneListing.VehicleMake, carzoneListing.VehicleModel, carzoneListing.VehicleYearOfManufacture, carzoneListing.VehiclePriceEuro, -1, carzoneListing.AdvertiserCounty, carzoneListing.VehicleDerivative); context.Listings.Add(listing); } context.SaveChanges(); } } } }
public void UpdateCarzoneEntries() { var carzoneApi = new CarzoneApi(new Requests.WebRequester()); using (var context = new CarsContext()) { var listings = context.Listings .Where(l => l.Source == SourceSite.Carzone) .Where(l => l.Mileage == -1) .ToList(); foreach (var listing in listings) { CarzoneListingDetails details; try { details = carzoneApi.GetListingDetails(Convert.ToInt64(listing.SourceId)); } catch (Exception e) { var a = 1; continue; } listing.Mileage = details.VehicleMileage; } context.SaveChanges(); } }