static void Main(string[] args)
{
//configure azure blob storage
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString);
});
var context = new RentlerDataContext();
//pull buildings
var buildings = (from b in context.Buildings
where b.sid != null
select b).ToList();
//download images using matched buildingid and sid
var tasksArray = new Task[buildings.Count()];
foreach (var item in buildings)
{
//queue up all the tasks
var t = new Task((b) =>
{
//store them in the archive the photo uploader understands
KslPhotoService.DownloadPhotos(
((Building)b).BuildingId,
int.Parse(((Building)b).sid));
}, item);
tasksArray[buildings.ToList().IndexOf(item)] = t;
}
//Run that script on the right over there ----->
//get it started! 5 at a time.
int page = 1;
PaginatedList <Task> tasks = null;
var hasNextPage = true;
var stopwatch = new Stopwatch();
var execTimes = new List <double>();
while (hasNextPage)
{
tasks = new PaginatedList <Task>(tasksArray.AsQueryable(), page, 5);
Console.WriteLine("Processing page {0} of {1}, {2} at a time",
page, tasks.TotalPages, tasks.PageSize);
stopwatch.Reset();
stopwatch.Start();
tasks.Each(t => t.Start());
Task.WaitAll(tasks.ToArray());
hasNextPage = tasks.HasNextPage;
page++;
stopwatch.Stop();
execTimes.Add(stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Done, took {0} seconds, estimated completion time is {1}",
stopwatch.Elapsed.TotalSeconds,
DateTime.Now.AddSeconds(execTimes.Average() * (tasks.TotalPages - page)).ToLongTimeString());
Console.WriteLine("Average execution time is {0}", execTimes.Average());
Console.WriteLine("{0} images processed", ImageCounter.ImageCount);
}
Console.WriteLine();
Console.WriteLine("Done!");
//???
//profit
}
static void Main(string[] args)
{
//configure azure blob storage
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString);
});
var context = new RentlerDataContext();
//pull buildings
var buildings = (from b in context.Buildings
where b.sid != null
select b).ToList();
//download images using matched buildingid and sid
var tasksArray = new Task[buildings.Count()];
foreach(var item in buildings)
{
//queue up all the tasks
var t = new Task((b) =>
{
//store them in the archive the photo uploader understands
KslPhotoService.DownloadPhotos(
((Building)b).BuildingId,
int.Parse(((Building)b).sid));
}, item);
tasksArray[buildings.ToList().IndexOf(item)] = t;
}
//Run that script on the right over there ----->
//get it started! 5 at a time.
int page = 1;
PaginatedList<Task> tasks = null;
var hasNextPage = true;
var stopwatch = new Stopwatch();
var execTimes = new List<double>();
while(hasNextPage)
{
tasks = new PaginatedList<Task>(tasksArray.AsQueryable(), page, 5);
Console.WriteLine("Processing page {0} of {1}, {2} at a time",
page, tasks.TotalPages, tasks.PageSize);
stopwatch.Reset();
stopwatch.Start();
tasks.Each(t => t.Start());
Task.WaitAll(tasks.ToArray());
hasNextPage = tasks.HasNextPage;
page++;
stopwatch.Stop();
execTimes.Add(stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Done, took {0} seconds, estimated completion time is {1}",
stopwatch.Elapsed.TotalSeconds,
DateTime.Now.AddSeconds(execTimes.Average() * (tasks.TotalPages - page)).ToLongTimeString());
Console.WriteLine("Average execution time is {0}", execTimes.Average());
Console.WriteLine("{0} images processed", ImageCounter.ImageCount);
}
Console.WriteLine();
Console.WriteLine("Done!");
//???
//profit
}
static void Main(string[] args)
{
//configure azure blob storage
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString);
});
//csv
var desc = new CsvFileDescription()
{
SeparatorChar = ',',
FirstLineHasColumnNames = true
};
var csvContext = new CsvContext();
//read the stuff in
Console.WriteLine("Reading CSV files...");
var classifieds = ReadCsv <Classified>("classifieds.csv", csvContext);
var homes = ReadCsv <Home>("homes.csv", csvContext);
var homeCoolings = ReadCsv <HomeCooling>("homes_cooling.csv", csvContext);
var homeHeatings = ReadCsv <HomeHeating>("homes_heating.csv", csvContext);
var homeProperties = ReadCsv <HomeProperty>("homes_property.csv", csvContext);
var members = ReadCsv <Member>("member.csv", csvContext);
List <User> users = RentlerConverter.ConvertToUsers(members).ToList();
var buildings = RentlerConverter.ConvertToBuildings(
homes, classifieds, homeCoolings,
homeHeatings, homeProperties, users);
//todo: remove this, just limiting
//results so I can do this stuff more quickly.
buildings = buildings.ToList();
users = (from u in users
where buildings.Any(b => b.UserId.Equals(u.UserId))
select u).ToList();
var landlordInfos = RentlerConverter.CreateLandlordInfos(users);
//and put them in the db
var rentlerContext = new RentlerDataContext();
//find existing users and handle them
var apiKey = new Guid("DA05C498-C338-4EE4-9A70-6872E38349CC");
var currentUsers = (from u in rentlerContext.Users
where u.AffiliateUser.ApiKey == apiKey
select u).ToList();
for (int i = 0; i < users.Count(); i++)
{
var existingUser = currentUsers.FirstOrDefault(u => u.Email.Equals(users[i].Email));
if (existingUser == null)
{
continue;
}
//user exists already, update the building reference
var buildingsToChange = buildings.Where(b => b.UserId.Equals(users[i].UserId)).ToList();
foreach (var toChange in buildingsToChange)
{
toChange.UserId = existingUser.UserId;
}
//remove the user and landlord info from the new list.
var oldLandlordInfo = landlordInfos.SingleOrDefault(l => l.UserId.Equals(users[i].UserId));
if (oldLandlordInfo != null)
{
landlordInfos.Remove(oldLandlordInfo);
}
users.Remove(users[i]);
}
Console.WriteLine("Inserting {0} users...", users.Count());
rentlerContext.Users.InsertAllOnSubmit(users);
rentlerContext.SubmitChanges();
Console.WriteLine("Inserting Landlord Info for {0} users...", landlordInfos.Count());
rentlerContext.LandlordInfos.InsertAllOnSubmit(landlordInfos);
rentlerContext.SubmitChanges();
Console.WriteLine("Inserting {0} buildings...", buildings.Count);
rentlerContext.Buildings.InsertAllOnSubmit(buildings);
rentlerContext.SubmitChanges();
Console.WriteLine("Uploading photos");
//upload photos
var tasksArray = new Task[buildings.Count()];
foreach (var item in buildings)
{
//queue up all the tasks
var t = new Task((b) =>
{
KslPhotoService.DownloadPhotos(
(int)((Building)b).BuildingId,
int.Parse(((Building)b).sid));
}, item);
tasksArray[buildings.ToList().IndexOf(item)] = t;
}
//get it started! 5 at a time.
int page = 1;
PaginatedList <Task> tasks = null;
var hasNextPage = true;
var stopwatch = new Stopwatch();
var execTimes = new List <double>();
while (hasNextPage)
{
tasks = new PaginatedList <Task>(tasksArray.AsQueryable(), page, 5);
Console.WriteLine("Processing page {0} of {1}, {2} at a time",
page, tasks.TotalPages, tasks.PageSize);
stopwatch.Reset();
stopwatch.Start();
tasks.Each(t => t.Start());
Task.WaitAll(tasks.ToArray());
hasNextPage = tasks.HasNextPage;
page++;
stopwatch.Stop();
execTimes.Add(stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Done, took {0} seconds, estimated completion time is {1}",
stopwatch.Elapsed.TotalSeconds,
DateTime.Now.AddSeconds(execTimes.Average() * (tasks.TotalPages - page)).ToLongTimeString());
Console.WriteLine("Average execution time is {0}", execTimes.Average());
Console.WriteLine("{0} images processed", ImageCounter.ImageCount);
}
Console.WriteLine();
Console.WriteLine("Done!");
//finally, run the blob uploaders to get this stuff in the cloud
Console.WriteLine("Launching Blob Uploaders...");
var process = new Process();
process.StartInfo.FileName =
@"C:\Users\Dusda\Projects\Rentler\Ksl Integration\RentlerPhotoUploader\RentlerPhotoUploader\bin\Debug\RentlerPhotoUploader.exe";
if (ImageCounter.ImageCount < 10000)
{
process.StartInfo.Arguments =
@"""C:\Users\Dusda\Projects\Rentler\Ksl Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\0""";
process.Start();
}
for (int i = 0; i < ImageCounter.ImageCount / 10000; i++)
{
process.StartInfo.Arguments =
@"""C:\Users\Dusda\Projects\Rentler\Ksl%20Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\""" + i;
process.Start();
}
}