public void LoadCarsIrelandFromJson()
{
var carsIrelandApi = new CarsIrelandApi(new Requests.WebRequester());
var cache = new Cache();
var dumpFiles = Directory.GetFiles(@"C:\dev\cars\Scraper.Test\bin\Debug", "carsirelandjsondump*");
var listings = dumpFiles
.AsParallel()
.Select(path => File.ReadAllText(path))
.SelectMany(text => carsIrelandApi.Deserialize(text))
.ToList();
for (int i = 0; i < listings.Count(); i += 100)
{
using (var context = new CarsContext())
{
for (int j = i; j < i + 100 && j < listings.Count(); j++)
{
var carsIrelandListing = listings[j];
var culture = new CultureInfo("en-IE");
int price;
if (!Int32.TryParse(carsIrelandListing.Price, NumberStyles.Currency, culture, out price))
{
price = -1;
}
var listing = new Listing(
context,
cache,
SourceSite.CarsIreland,
carsIrelandListing.Ad_Id.ToString(),
carsIrelandListing.Make,
carsIrelandListing.Model,
carsIrelandListing.Reg_Year,
price,
carsIrelandListing.Mileage ?? -1,
carsIrelandListing.Location,
carsIrelandListing.Variant);
listing.FuelType = EnumHelpers.FromString<FuelType>(carsIrelandListing.Fuel_Type);
double engineSize;
if (Double.TryParse(carsIrelandListing.Engine_Size, out engineSize))
{
listing.EngineSize = (int)(engineSize * 100);
}
context.Listings.Add(listing);
}
context.SaveChanges();
}
}
}
private void GetAndWrite(int from, int to)
{
for (int i = from; i < to; i++)
{
var apilocal = new CarsIrelandApi(new Requests.WebRequester());
var theString = apilocal.GetListingsString(i);
using (var tw = File.CreateText(string.Format("carsirelandjsondump{0}.json", i)))
{
tw.Write(theString);
}
}
}