static void Main(string[] args) { //configure azure blob storage CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString); }); //csv var desc = new CsvFileDescription() { SeparatorChar = ',', FirstLineHasColumnNames = true }; var csvContext = new CsvContext(); //read the stuff in Console.WriteLine("Reading CSV files..."); var classifieds = ReadCsv <Classified>("classifieds.csv", csvContext); var homes = ReadCsv <Home>("homes.csv", csvContext); var homeCoolings = ReadCsv <HomeCooling>("homes_cooling.csv", csvContext); var homeHeatings = ReadCsv <HomeHeating>("homes_heating.csv", csvContext); var homeProperties = ReadCsv <HomeProperty>("homes_property.csv", csvContext); var members = ReadCsv <Member>("member.csv", csvContext); var context = new RentlerDataContext(); //get all the current buildings var currentBuildings = context.Buildings .ToList(); //var currentUsers = context.Users // .Where(u => u.ApiKeys // .Any(a => a.ApiKeyId.Equals(new Guid("DA05C498-C338-4EE4-9A70-6872E38349CC")))) // .ToList(); var buildingsWithSidYo = RentlerConverter.ConvertToBuildingsLight(homes, classifieds); //add sid //foreach(var b in currentBuildings) //{ // var withSid = (from s in buildingsWithSidYo // where s.AddressLine1.Equals(b.AddressLine1) && // s.AddressLine2.Equals(b.AddressLine2) // select s).SingleOrDefault(); // if(withSid != null) // { // b.sid = withSid.sid; // //b.Listings[0].Description = withSid.Listings[0].Description; // } //} //Console.WriteLine("Submitting repaired buildings..."); //context.SubmitChanges(); //and fix descriptions var currentListings = (from b in context.Buildings select new { Building = b, Listings = b.Listings }).ToList(); foreach (var item in currentListings) { var description = (from l in buildingsWithSidYo where l.sid.Equals(item.Building.sid) select l.Listings[0].Description).SingleOrDefault(); if (description != null) { item.Listings[0].Description = description; } } Console.WriteLine("Submitting repaired descriptions..."); context.SubmitChanges(); }
static void FixUserGroup(IGrouping <string, User> userGroup) { // sort them descending var list = userGroup.ToList(); var users = (from l in list orderby l.CreateDate descending select l).ToList(); Console.WriteLine("Processing {0}", users[0].Email); List <UserContract> userConts = new List <UserContract>(); List <Building> userBuildings = new List <Building>(); List <UserInterest> landlordInterests = new List <UserInterest>(); List <UserInterest> tenantInterests = new List <UserInterest>(); List <SavedListing> userSavedListings = new List <SavedListing>(); for (int x = 1; x < users.Count; ++x) { userBuildings.AddRange((from b in buildings where b.UserId == users[x].UserId select b).ToList()); userConts.AddRange((from uc in userContracts where uc.UserId == users[x].UserId select uc).ToList()); tenantInterests.AddRange((from ui in userInterests where ui.UserId == users[x].UserId select ui).ToList()); landlordInterests.AddRange((from ui in userInterests where ui.LandlordUserId == users[x].UserId select ui).ToList()); userSavedListings.AddRange((from ui in savedListings where ui.UserId == users[x].UserId select ui).ToList()); } Console.WriteLine("Fixing buildings and contracts..."); for (int x = 0; x < userBuildings.Count; ++x) { userBuildings[x].UserId = users[0].UserId; } for (int x = 0; x < userConts.Count; ++x) { userConts[x].UserId = users[0].UserId; } for (int x = 0; x < tenantInterests.Count; ++x) { tenantInterests[x].UserId = users[0].UserId; } for (int x = 0; x < landlordInterests.Count; ++x) { landlordInterests[x].LandlordUserId = users[0].UserId; } for (int x = 0; x < userSavedListings.Count; ++x) { userSavedListings[x].UserId = users[0].UserId; } context.SubmitChanges(); Console.WriteLine("Fixing Affiliate, Auth, and Roles"); var affiliateUsers = (from a in context.AffiliateUsers where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.AffiliateUsers.DeleteAllOnSubmit(affiliateUsers); var authTokens = (from a in context.AuthTokens where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.AuthTokens.DeleteAllOnSubmit(authTokens); var roleUsers = (from a in context.RoleUsers where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.RoleUsers.DeleteAllOnSubmit(roleUsers); var landlordInfo = (from a in context.LandlordInfos where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.LandlordInfos.DeleteAllOnSubmit(landlordInfo); var applicationInfo = (from a in context.ApplicationInfos where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.ApplicationInfos.DeleteAllOnSubmit(applicationInfo); var alerts = (from a in context.Alerts where a.User.Email == users[0].Email && a.User.UserId != users[0].UserId select a).ToList(); context.Alerts.DeleteAllOnSubmit(alerts); for (int x = 1; x < users.Count; ++x) { context.Users.DeleteOnSubmit(users[x]); } Console.WriteLine("Committing"); context.SubmitChanges(); }
static void Main(string[] args) { //configure azure blob storage CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString); }); //csv var desc = new CsvFileDescription() { SeparatorChar = ',', FirstLineHasColumnNames = true }; var csvContext = new CsvContext(); //read the stuff in Console.WriteLine("Reading CSV files..."); var classifieds = ReadCsv <Classified>("classifieds.csv", csvContext); var homes = ReadCsv <Home>("homes.csv", csvContext); var homeCoolings = ReadCsv <HomeCooling>("homes_cooling.csv", csvContext); var homeHeatings = ReadCsv <HomeHeating>("homes_heating.csv", csvContext); var homeProperties = ReadCsv <HomeProperty>("homes_property.csv", csvContext); var members = ReadCsv <Member>("member.csv", csvContext); List <User> users = RentlerConverter.ConvertToUsers(members).ToList(); var buildings = RentlerConverter.ConvertToBuildings( homes, classifieds, homeCoolings, homeHeatings, homeProperties, users); //todo: remove this, just limiting //results so I can do this stuff more quickly. buildings = buildings.ToList(); users = (from u in users where buildings.Any(b => b.UserId.Equals(u.UserId)) select u).ToList(); var landlordInfos = RentlerConverter.CreateLandlordInfos(users); //and put them in the db var rentlerContext = new RentlerDataContext(); //find existing users and handle them var apiKey = new Guid("DA05C498-C338-4EE4-9A70-6872E38349CC"); var currentUsers = (from u in rentlerContext.Users where u.AffiliateUser.ApiKey == apiKey select u).ToList(); for (int i = 0; i < users.Count(); i++) { var existingUser = currentUsers.FirstOrDefault(u => u.Email.Equals(users[i].Email)); if (existingUser == null) { continue; } //user exists already, update the building reference var buildingsToChange = buildings.Where(b => b.UserId.Equals(users[i].UserId)).ToList(); foreach (var toChange in buildingsToChange) { toChange.UserId = existingUser.UserId; } //remove the user and landlord info from the new list. var oldLandlordInfo = landlordInfos.SingleOrDefault(l => l.UserId.Equals(users[i].UserId)); if (oldLandlordInfo != null) { landlordInfos.Remove(oldLandlordInfo); } users.Remove(users[i]); } Console.WriteLine("Inserting {0} users...", users.Count()); rentlerContext.Users.InsertAllOnSubmit(users); rentlerContext.SubmitChanges(); Console.WriteLine("Inserting Landlord Info for {0} users...", landlordInfos.Count()); rentlerContext.LandlordInfos.InsertAllOnSubmit(landlordInfos); rentlerContext.SubmitChanges(); Console.WriteLine("Inserting {0} buildings...", buildings.Count); rentlerContext.Buildings.InsertAllOnSubmit(buildings); rentlerContext.SubmitChanges(); Console.WriteLine("Uploading photos"); //upload photos var tasksArray = new Task[buildings.Count()]; foreach (var item in buildings) { //queue up all the tasks var t = new Task((b) => { KslPhotoService.DownloadPhotos( (int)((Building)b).BuildingId, int.Parse(((Building)b).sid)); }, item); tasksArray[buildings.ToList().IndexOf(item)] = t; } //get it started! 5 at a time. int page = 1; PaginatedList <Task> tasks = null; var hasNextPage = true; var stopwatch = new Stopwatch(); var execTimes = new List <double>(); while (hasNextPage) { tasks = new PaginatedList <Task>(tasksArray.AsQueryable(), page, 5); Console.WriteLine("Processing page {0} of {1}, {2} at a time", page, tasks.TotalPages, tasks.PageSize); stopwatch.Reset(); stopwatch.Start(); tasks.Each(t => t.Start()); Task.WaitAll(tasks.ToArray()); hasNextPage = tasks.HasNextPage; page++; stopwatch.Stop(); execTimes.Add(stopwatch.Elapsed.TotalSeconds); Console.WriteLine("Done, took {0} seconds, estimated completion time is {1}", stopwatch.Elapsed.TotalSeconds, DateTime.Now.AddSeconds(execTimes.Average() * (tasks.TotalPages - page)).ToLongTimeString()); Console.WriteLine("Average execution time is {0}", execTimes.Average()); Console.WriteLine("{0} images processed", ImageCounter.ImageCount); } Console.WriteLine(); Console.WriteLine("Done!"); //finally, run the blob uploaders to get this stuff in the cloud Console.WriteLine("Launching Blob Uploaders..."); var process = new Process(); process.StartInfo.FileName = @"C:\Users\Dusda\Projects\Rentler\Ksl Integration\RentlerPhotoUploader\RentlerPhotoUploader\bin\Debug\RentlerPhotoUploader.exe"; if (ImageCounter.ImageCount < 10000) { process.StartInfo.Arguments = @"""C:\Users\Dusda\Projects\Rentler\Ksl Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\0"""; process.Start(); } for (int i = 0; i < ImageCounter.ImageCount / 10000; i++) { process.StartInfo.Arguments = @"""C:\Users\Dusda\Projects\Rentler\Ksl%20Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\""" + i; process.Start(); } }