static void Main(string[] args)
{
//configure azure blob storage
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString);
});
//csv
var desc = new CsvFileDescription()
{
SeparatorChar = ',',
FirstLineHasColumnNames = true
};
var csvContext = new CsvContext();
//read the stuff in
Console.WriteLine("Reading CSV files...");
var classifieds = ReadCsv <Classified>("classifieds.csv", csvContext);
var homes = ReadCsv <Home>("homes.csv", csvContext);
var homeCoolings = ReadCsv <HomeCooling>("homes_cooling.csv", csvContext);
var homeHeatings = ReadCsv <HomeHeating>("homes_heating.csv", csvContext);
var homeProperties = ReadCsv <HomeProperty>("homes_property.csv", csvContext);
var members = ReadCsv <Member>("member.csv", csvContext);
var context = new RentlerDataContext();
//get all the current buildings
var currentBuildings = context.Buildings
.ToList();
//var currentUsers = context.Users
// .Where(u => u.ApiKeys
// .Any(a => a.ApiKeyId.Equals(new Guid("DA05C498-C338-4EE4-9A70-6872E38349CC"))))
// .ToList();
var buildingsWithSidYo = RentlerConverter.ConvertToBuildingsLight(homes, classifieds);
//add sid
//foreach(var b in currentBuildings)
//{
// var withSid = (from s in buildingsWithSidYo
// where s.AddressLine1.Equals(b.AddressLine1) &&
// s.AddressLine2.Equals(b.AddressLine2)
// select s).SingleOrDefault();
// if(withSid != null)
// {
// b.sid = withSid.sid;
// //b.Listings[0].Description = withSid.Listings[0].Description;
// }
//}
//Console.WriteLine("Submitting repaired buildings...");
//context.SubmitChanges();
//and fix descriptions
var currentListings = (from b in context.Buildings
select new { Building = b, Listings = b.Listings }).ToList();
foreach (var item in currentListings)
{
var description = (from l in buildingsWithSidYo
where l.sid.Equals(item.Building.sid)
select l.Listings[0].Description).SingleOrDefault();
if (description != null)
{
item.Listings[0].Description = description;
}
}
Console.WriteLine("Submitting repaired descriptions...");
context.SubmitChanges();
}
static void FixUserGroup(IGrouping <string, User> userGroup)
{
// sort them descending
var list = userGroup.ToList();
var users = (from l in list
orderby l.CreateDate
descending
select l).ToList();
Console.WriteLine("Processing {0}", users[0].Email);
List <UserContract> userConts = new List <UserContract>();
List <Building> userBuildings = new List <Building>();
List <UserInterest> landlordInterests = new List <UserInterest>();
List <UserInterest> tenantInterests = new List <UserInterest>();
List <SavedListing> userSavedListings = new List <SavedListing>();
for (int x = 1; x < users.Count; ++x)
{
userBuildings.AddRange((from b in buildings
where b.UserId == users[x].UserId
select b).ToList());
userConts.AddRange((from uc in userContracts
where uc.UserId == users[x].UserId
select uc).ToList());
tenantInterests.AddRange((from ui in userInterests
where ui.UserId == users[x].UserId
select ui).ToList());
landlordInterests.AddRange((from ui in userInterests
where ui.LandlordUserId == users[x].UserId
select ui).ToList());
userSavedListings.AddRange((from ui in savedListings
where ui.UserId == users[x].UserId
select ui).ToList());
}
Console.WriteLine("Fixing buildings and contracts...");
for (int x = 0; x < userBuildings.Count; ++x)
{
userBuildings[x].UserId = users[0].UserId;
}
for (int x = 0; x < userConts.Count; ++x)
{
userConts[x].UserId = users[0].UserId;
}
for (int x = 0; x < tenantInterests.Count; ++x)
{
tenantInterests[x].UserId = users[0].UserId;
}
for (int x = 0; x < landlordInterests.Count; ++x)
{
landlordInterests[x].LandlordUserId = users[0].UserId;
}
for (int x = 0; x < userSavedListings.Count; ++x)
{
userSavedListings[x].UserId = users[0].UserId;
}
context.SubmitChanges();
Console.WriteLine("Fixing Affiliate, Auth, and Roles");
var affiliateUsers = (from a in context.AffiliateUsers
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.AffiliateUsers.DeleteAllOnSubmit(affiliateUsers);
var authTokens = (from a in context.AuthTokens
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.AuthTokens.DeleteAllOnSubmit(authTokens);
var roleUsers = (from a in context.RoleUsers
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.RoleUsers.DeleteAllOnSubmit(roleUsers);
var landlordInfo = (from a in context.LandlordInfos
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.LandlordInfos.DeleteAllOnSubmit(landlordInfo);
var applicationInfo = (from a in context.ApplicationInfos
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.ApplicationInfos.DeleteAllOnSubmit(applicationInfo);
var alerts = (from a in context.Alerts
where a.User.Email == users[0].Email &&
a.User.UserId != users[0].UserId
select a).ToList();
context.Alerts.DeleteAllOnSubmit(alerts);
for (int x = 1; x < users.Count; ++x)
{
context.Users.DeleteOnSubmit(users[x]);
}
Console.WriteLine("Committing");
context.SubmitChanges();
}
static void Main(string[] args)
{
//configure azure blob storage
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(ConfigurationManager.ConnectionStrings[configName].ConnectionString);
});
//csv
var desc = new CsvFileDescription()
{
SeparatorChar = ',',
FirstLineHasColumnNames = true
};
var csvContext = new CsvContext();
//read the stuff in
Console.WriteLine("Reading CSV files...");
var classifieds = ReadCsv <Classified>("classifieds.csv", csvContext);
var homes = ReadCsv <Home>("homes.csv", csvContext);
var homeCoolings = ReadCsv <HomeCooling>("homes_cooling.csv", csvContext);
var homeHeatings = ReadCsv <HomeHeating>("homes_heating.csv", csvContext);
var homeProperties = ReadCsv <HomeProperty>("homes_property.csv", csvContext);
var members = ReadCsv <Member>("member.csv", csvContext);
List <User> users = RentlerConverter.ConvertToUsers(members).ToList();
var buildings = RentlerConverter.ConvertToBuildings(
homes, classifieds, homeCoolings,
homeHeatings, homeProperties, users);
//todo: remove this, just limiting
//results so I can do this stuff more quickly.
buildings = buildings.ToList();
users = (from u in users
where buildings.Any(b => b.UserId.Equals(u.UserId))
select u).ToList();
var landlordInfos = RentlerConverter.CreateLandlordInfos(users);
//and put them in the db
var rentlerContext = new RentlerDataContext();
//find existing users and handle them
var apiKey = new Guid("DA05C498-C338-4EE4-9A70-6872E38349CC");
var currentUsers = (from u in rentlerContext.Users
where u.AffiliateUser.ApiKey == apiKey
select u).ToList();
for (int i = 0; i < users.Count(); i++)
{
var existingUser = currentUsers.FirstOrDefault(u => u.Email.Equals(users[i].Email));
if (existingUser == null)
{
continue;
}
//user exists already, update the building reference
var buildingsToChange = buildings.Where(b => b.UserId.Equals(users[i].UserId)).ToList();
foreach (var toChange in buildingsToChange)
{
toChange.UserId = existingUser.UserId;
}
//remove the user and landlord info from the new list.
var oldLandlordInfo = landlordInfos.SingleOrDefault(l => l.UserId.Equals(users[i].UserId));
if (oldLandlordInfo != null)
{
landlordInfos.Remove(oldLandlordInfo);
}
users.Remove(users[i]);
}
Console.WriteLine("Inserting {0} users...", users.Count());
rentlerContext.Users.InsertAllOnSubmit(users);
rentlerContext.SubmitChanges();
Console.WriteLine("Inserting Landlord Info for {0} users...", landlordInfos.Count());
rentlerContext.LandlordInfos.InsertAllOnSubmit(landlordInfos);
rentlerContext.SubmitChanges();
Console.WriteLine("Inserting {0} buildings...", buildings.Count);
rentlerContext.Buildings.InsertAllOnSubmit(buildings);
rentlerContext.SubmitChanges();
Console.WriteLine("Uploading photos");
//upload photos
var tasksArray = new Task[buildings.Count()];
foreach (var item in buildings)
{
//queue up all the tasks
var t = new Task((b) =>
{
KslPhotoService.DownloadPhotos(
(int)((Building)b).BuildingId,
int.Parse(((Building)b).sid));
}, item);
tasksArray[buildings.ToList().IndexOf(item)] = t;
}
//get it started! 5 at a time.
int page = 1;
PaginatedList <Task> tasks = null;
var hasNextPage = true;
var stopwatch = new Stopwatch();
var execTimes = new List <double>();
while (hasNextPage)
{
tasks = new PaginatedList <Task>(tasksArray.AsQueryable(), page, 5);
Console.WriteLine("Processing page {0} of {1}, {2} at a time",
page, tasks.TotalPages, tasks.PageSize);
stopwatch.Reset();
stopwatch.Start();
tasks.Each(t => t.Start());
Task.WaitAll(tasks.ToArray());
hasNextPage = tasks.HasNextPage;
page++;
stopwatch.Stop();
execTimes.Add(stopwatch.Elapsed.TotalSeconds);
Console.WriteLine("Done, took {0} seconds, estimated completion time is {1}",
stopwatch.Elapsed.TotalSeconds,
DateTime.Now.AddSeconds(execTimes.Average() * (tasks.TotalPages - page)).ToLongTimeString());
Console.WriteLine("Average execution time is {0}", execTimes.Average());
Console.WriteLine("{0} images processed", ImageCounter.ImageCount);
}
Console.WriteLine();
Console.WriteLine("Done!");
//finally, run the blob uploaders to get this stuff in the cloud
Console.WriteLine("Launching Blob Uploaders...");
var process = new Process();
process.StartInfo.FileName =
@"C:\Users\Dusda\Projects\Rentler\Ksl Integration\RentlerPhotoUploader\RentlerPhotoUploader\bin\Debug\RentlerPhotoUploader.exe";
if (ImageCounter.ImageCount < 10000)
{
process.StartInfo.Arguments =
@"""C:\Users\Dusda\Projects\Rentler\Ksl Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\0""";
process.Start();
}
for (int i = 0; i < ImageCounter.ImageCount / 10000; i++)
{
process.StartInfo.Arguments =
@"""C:\Users\Dusda\Projects\Rentler\Ksl%20Integration\KslListingImporter\KslListingImporter\bin\Debug\userphotostest\""" + i;
process.Start();
}
}
