public void LoadCarsIrelandFromJson() { var carsIrelandApi = new CarsIrelandApi(new Requests.WebRequester()); var cache = new Cache(); var dumpFiles = Directory.GetFiles(@"C:\dev\cars\Scraper.Test\bin\Debug", "carsirelandjsondump*"); var listings = dumpFiles .AsParallel() .Select(path => File.ReadAllText(path)) .SelectMany(text => carsIrelandApi.Deserialize(text)) .ToList(); for (int i = 0; i < listings.Count(); i += 100) { using (var context = new CarsContext()) { for (int j = i; j < i + 100 && j < listings.Count(); j++) { var carsIrelandListing = listings[j]; var culture = new CultureInfo("en-IE"); int price; if (!Int32.TryParse(carsIrelandListing.Price, NumberStyles.Currency, culture, out price)) { price = -1; } var listing = new Listing( context, cache, SourceSite.CarsIreland, carsIrelandListing.Ad_Id.ToString(), carsIrelandListing.Make, carsIrelandListing.Model, carsIrelandListing.Reg_Year, price, carsIrelandListing.Mileage ?? -1, carsIrelandListing.Location, carsIrelandListing.Variant); listing.FuelType = EnumHelpers.FromString<FuelType>(carsIrelandListing.Fuel_Type); double engineSize; if (Double.TryParse(carsIrelandListing.Engine_Size, out engineSize)) { listing.EngineSize = (int)(engineSize * 100); } context.Listings.Add(listing); } context.SaveChanges(); } } }
private void GetAndWrite(int from, int to) { for (int i = from; i < to; i++) { var apilocal = new CarsIrelandApi(new Requests.WebRequester()); var theString = apilocal.GetListingsString(i); using (var tw = File.CreateText(string.Format("carsirelandjsondump{0}.json", i))) { tw.Write(theString); } } }