public void CreateRoleScratch()
{
Guid id = Guid.NewGuid();
using (IdentityCloudContext context = new IdentityCloudContext())
{
var doc = new { id = id.ToString(), SpiderMonkey = "Monkey baby" };
var docTask = context.Client.CreateDocumentAsync(context.RoleDocumentCollection.SelfLink,
doc, context.RequestOptions);
docTask.Wait();
var docResult = docTask.Result;
Debug.WriteLine(docResult.Resource.ToString());
var docQrTask = context.Client.CreateDocumentQuery(context.RoleDocumentCollection.DocumentsLink
, new Microsoft.Azure.Documents.Client.FeedOptions()
{
SessionToken = context.SessionToken, MaxItemCount = 1
})
.Where(d => d.Id == doc.id)
.Select(s => s)
.ToList()
.FirstOrDefault();
Debug.WriteLine(docQrTask.ToString());
}
}
public void ProcessMigrate(IdentityCloudContext targetContext, IdentityCloudContext sourceContext, IList <DynamicTableEntity> sourceUserKeysResults, int maxDegreesParallel, Action updateComplete = null, Action <string> updateError = null)
{
var result2 = Parallel.ForEach(sourceUserKeysResults, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (dte) =>
{
try
{
var sourceUserEntities = GetUserEntitiesBySourceId(dte.PartitionKey, sourceContext);
string targetUserId = KeyHelper.GenerateUserId();
var targetEntities = ConvertToTargetUserEntities(targetUserId, sourceUserEntities);
List <Task> mainTasks = new List <Task>(2);
List <Task> indexTasks = new List <Task>(100);
BatchOperationHelper batchOperation = new BatchOperationHelper();
targetEntities.targetUserEntities
.ForEach(targetUserRecord => batchOperation.Add(TableOperation.InsertOrReplace(targetUserRecord)));
mainTasks.Add(batchOperation.ExecuteBatchAsync(targetContext.UserTable));
targetEntities.targetUserIndexes
.ForEach(targetIndexRecord => indexTasks.Add(targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(targetIndexRecord))));
mainTasks.Add(Task.WhenAll(indexTasks));
Task.WhenAll(mainTasks).Wait();
updateComplete?.Invoke();
}
catch (AggregateException exagg)
{
updateError?.Invoke(string.Format("{0}-{1}\t{2}", dte.PartitionKey, dte.RowKey, exagg.Flatten().Message));
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}-{1}\t{2}", dte.PartitionKey, dte.RowKey, ex.Message));
}
});
}
private static async Task <TempUser> ConvertUser(TempUser user, IdentityCloudContext <IdentityUser> ic, bool deleteDocuments)
{
var response = await ic.Client.ReplaceDocumentAsync(user.User, new Microsoft.Azure.Documents.Client.RequestOptions()
{
SessionToken = ic.SessionToken
});
ic.SetSessionTokenIfEmpty(response.SessionToken);
if (deleteDocuments)
{
foreach (var r in user.Roles)
{
await ic.Client.DeleteDocumentAsync(r.SelfLink);
}
foreach (var c in user.Claims)
{
await ic.Client.DeleteDocumentAsync(c.SelfLink);
}
foreach (var l in user.Logins)
{
await ic.Client.DeleteDocumentAsync(l.SelfLink);
}
}
return(user);
}
public void ProcessMigrate(IdentityCloudContext targetContext,
IdentityCloudContext sourceContext,
IList <DynamicTableEntity> userResults,
int maxDegreesParallel,
Action updateComplete = null,
Action <string> updateError = null)
{
var userIds = userResults
.Where(UserWhereFilter)
.Select(d => new { UserId = d.PartitionKey, Email = d.Properties["Email"].StringValue })
.ToList();
var result2 = Parallel.ForEach(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (userId) =>
{
//Add the email index
try
{
IdentityUserIndex index = CreateEmailIndex(userId.UserId, userId.Email);
var r = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}\t{1}", userId.UserId, ex.Message));
}
});
}
public void ProcessMigrate(IdentityCloudContext targetContext, IdentityCloudContext sourceContext, IList <DynamicTableEntity> sourceUserResults, int maxDegreesParallel, Action updateComplete = null, Action <string> updateError = null)
{
var rolesAndClaims = sourceUserResults
.Where(UserWhereFilter);
var result2 = Parallel.ForEach(rolesAndClaims, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (dte) =>
{
//Add the role or claim index
try
{
IdentityUserIndex index = new IdentityUserIndex()
{
Id = dte.PartitionKey,
PartitionKey = dte.RowKey,
RowKey = dte.PartitionKey,
KeyVersion = _keyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
var r = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}\t{1}", dte.PartitionKey, ex.Message));
}
});
}
public void ProcessMigrate(IdentityCloudContext targetContext,
IdentityCloudContext sourceContext,
IList <DynamicTableEntity> claimResults,
int maxDegreesParallel,
Action updateComplete = null,
Action <string> updateError = null)
{
const string KeyVersion = "KeyVersion";
var claims = claimResults
.Where(UserWhereFilter)
.ToList();
var result2 = Parallel.ForEach(claims, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (claim) =>
{
//Add the new claim index
try
{
var claimNew = new DynamicTableEntity(claim.PartitionKey,
_keyHelper.GenerateRowKeyIdentityUserClaim(claim.Properties["ClaimType"].StringValue, claim.Properties["ClaimValue"].StringValue),
Constants.ETagWildcard,
claim.Properties);
if (claimNew.Properties.ContainsKey(KeyVersion))
{
claimNew.Properties[KeyVersion].DoubleValue = _keyHelper.KeyVersion;
}
else
{
claimNew.Properties.Add(KeyVersion, EntityProperty.GeneratePropertyForDouble(_keyHelper.KeyVersion));
}
var taskExecute = targetContext.UserTable.ExecuteAsync(TableOperation.InsertOrReplace(claimNew));
taskExecute.Wait();
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}\t{1}", claim.PartitionKey, ex.Message));
}
});
}
private string GetRoleNameBySourceId(string roleRowKey, IdentityCloudContext sourcesContext)
{
var tr = sourcesContext.RoleTable.ExecuteAsync(
TableOperation.Retrieve <DynamicTableEntity>(KeyHelper.ParsePartitionKeyIdentityRoleFromRowKey(roleRowKey),
roleRowKey, new List <string>()
{
"Name", "PartitionKey", "RowKey"
})).Result;
if (tr.Result != null)
{
var role = (DynamicTableEntity)tr.Result;
if (role.Properties.TryGetValue("Name", out EntityProperty nameProperty))
{
return(nameProperty.StringValue);
}
}
return(null);
}
public void ProcessMigrate(IdentityCloudContext targetContext, IdentityCloudContext sourceContext, IList <DynamicTableEntity> sourceUserKeysResults, int maxDegreesParallel, Action updateComplete = null, Action <string> updateError = null)
{
var result2 = Parallel.ForEach(sourceUserKeysResults, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (dte) =>
{
try
{
targetContext.RoleTable.ExecuteAsync(TableOperation.InsertOrReplace(ConvertToTargetRoleEntity(dte, sourceContext)));
updateComplete?.Invoke();
}
catch (AggregateException exagg)
{
updateError?.Invoke(string.Format("{0}-{1}\t{2}", dte.PartitionKey, dte.RowKey, exagg.Flatten().Message));
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}-{1}\t{2}", dte.PartitionKey, dte.RowKey, ex.Message));
}
});
}
public void ProcessMigrate(IdentityCloudContext targetContext,
IdentityCloudContext sourceContext,
IList <DynamicTableEntity> sourceUserResults,
int maxDegreesParallel,
Action updateComplete = null,
Action <string> updateError = null)
{
var userIds = sourceUserResults
.Where(UserWhereFilter)
.Select(d => new
{
UserId = d.PartitionKey,
LoginProvider = d.Properties["LoginProvider"].StringValue,
ProviderKey = d.Properties["ProviderKey"].StringValue
})
.ToList();
Parallel.ForEach(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (userId) =>
{
//Add the email index
try
{
IdentityUserIndex index = CreateLoginIndex(userId.UserId, userId.LoginProvider, userId.ProviderKey);
var unused = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke($"{userId.UserId}\t{ex.Message}");
}
});
}
public static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
var builder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false);
Configuration = builder.Build();
IdentityConfiguration idconfig = new IdentityConfiguration();
idconfig.TablePrefix = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:TablePrefix").Value;
idconfig.StorageConnectionString = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:StorageConnectionString").Value;
idconfig.LocationMode = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:LocationMode").Value;
Console.WriteLine("MaxDegreeOfParallelism: {0}", iMaxdegreesparallel);
Console.WriteLine("PageSize: {0}", iPageSize);
using (IdentityCloudContext ic = new IdentityCloudContext(idconfig))
{
DateTime startLoad = DateTime.UtcNow;
var allDataList = new List <DynamicTableEntity>(iPageSize);
TableQuery tq = new TableQuery();
tq.SelectColumns = new List <string>()
{
"PartitionKey", "RowKey", "Email"
};
string partitionFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, Constants.RowKeyConstants.PreFixIdentityUserName),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, "V_"));
string rowFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, Constants.RowKeyConstants.PreFixIdentityUserName),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "V_"));
tq.FilterString = TableQuery.CombineFilters(partitionFilter, TableOperators.And, rowFilter);
tq.TakeCount = iPageSize;
TableContinuationToken continueToken = new TableContinuationToken();
int iSkippedUserCount = 0;
int iSkippedPageCount = 0;
int iPageCounter = 0;
while (continueToken != null)
{
DateTime batchStart = DateTime.UtcNow;
var userResults = ic.UserTable.ExecuteQuerySegmentedAsync(tq, continueToken).Result;
continueToken = userResults.ContinuationToken;
List <Tuple <string, string> > userIds = userResults.Results
.Where(d => !string.IsNullOrWhiteSpace(d.Properties["Email"].StringValue))
.Select(d => new Tuple <string, string>(d.PartitionKey, d.Properties["Email"].StringValue))
.ToList();
int batchCount = userIds.Count();
iUserTotal += batchCount;
iPageCounter++;
bool includePage = (iStartPage == -1 || iPageCounter >= iStartPage) && (iFinishPage == -1 || iPageCounter <= iFinishPage);
if (includePage)
{
var result2 = Parallel.ForEach <Tuple <string, string> >(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = iMaxdegreesparallel
}, (userId) =>
{
if (migrateOption)
{
//Add the email index
try
{
IdentityUserIndex index = CreateEmailIndex(userId.Item1, userId.Item2);
var r = ic.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
Interlocked.Increment(ref iUserSuccessConvert);
}
catch (Exception ex)
{
userIdFailures.Add(string.Format("{0}\t{1}", userId.Item1, ex.Message));
Interlocked.Increment(ref iUserFailureConvert);
}
}
});
}
else
{
iSkippedPageCount++;
iSkippedUserCount += batchCount;
}
Console.WriteLine("Page: {2}{3}, Users Batch: {1}: {0} seconds", (DateTime.UtcNow - batchStart).TotalSeconds, batchCount, iPageCounter, includePage ? string.Empty : "(Skipped)");
//Are we done yet?
if (iFinishPage > 0 && iPageCounter >= iFinishPage)
{
break;
}
}
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total Users Skipped: {0}, Total Pages: {1}", iSkippedUserCount, iSkippedPageCount);
Console.WriteLine("Total Users To Convert: {0}, Total Pages: {1}", iUserTotal - iSkippedUserCount, iPageCounter - iSkippedPageCount);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total Users Successfully Converted: {0}", iUserSuccessConvert);
Console.WriteLine("Total Users Failed to Convert: {0}", iUserFailureConvert);
if (iUserFailureConvert > 0)
{
Console.WriteLine("User Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
DisplayAnyKeyToExit();
}
static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
using (IdentityCloudContext<IdentityUser> ic = new IdentityCloudContext<IdentityUser>())
{
using (UserStore<IdentityUser> store = new UserStore<IdentityUser>(ic))
{
userCollection = ic.Client.CreateDocumentCollectionQuery(ic.Database.CollectionsLink)
.Where(c => c.Id == Constants.DocumentCollectionIds.UsersCollection)
.ToList()
.FirstOrDefault();
if (userCollection != null)
{
List<Document> lroles = null;
List<Document> llogins = null;
List<Document> lclaims = null;
Dictionary<string, Document> lusers = null;
DateTime startLoad = DateTime.UtcNow;
var allDataList = new List<Document>(2000);
Task[] tasks = new Task[]{
new TaskFactory().StartNew(() =>
{
var qRoles = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.RoleName != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
lroles = qRoles.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return d; }).ToList();
allDataList.AddRange(lroles);
Console.WriteLine("Roles to convert: {0}", lroles.Count);
}),
new TaskFactory().StartNew(()=>
{
var qLogins = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.LoginProvider != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
llogins = qLogins.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return d; }).ToList();
allDataList.AddRange(llogins);
Console.WriteLine("Logins to convert: {0}", llogins.Count);
}),
new TaskFactory().StartNew(()=>
{
var qClaims = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.ClaimType != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
lclaims = qClaims.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return d; }).ToList();
allDataList.AddRange(lclaims);
Console.WriteLine("Claims to convert: {0}", lclaims.Count);
}),
new TaskFactory().StartNew(()=>
{
var qUser = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.id = r.UserId",
new Microsoft.Azure.Documents.Client.FeedOptions());
lusers = qUser.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return d; }).ToDictionary(d=> d.Id);
Console.WriteLine("Total Users: {0}", lusers.Count);
})
};
Task.WaitAll(tasks);
Console.WriteLine("Load Roles, Claims, Logins and Users: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
List<string> userIds = allDataList.Select(dl=> dl.GetPropertyValue<string>("UserId")).Distinct().ToList();
var result2 = Parallel.ForEach<string>(userIds, (userId) =>
{
Document user;
if (!lusers.TryGetValue(userId, out user))
{
Console.WriteLine("User document not found: {0}", userId);
return;
}
//Get all of the docs with the same UserId
Task<TempUser> tempTask = CreateTempUser(user, allDataList.Where(d => d.GetPropertyValue<string>("UserId") == user.Id).ToList());
tempTask.Wait();
TempUser temp = tempTask.Result;
if (temp.Roles.Count > 0
|| temp.Claims.Count > 0
|| temp.Logins.Count > 0)
{
if (migrateOption)
{
ConvertUser(temp, ic, deleteOption).ContinueWith((tu) =>
{
return ConfirmUserConvert(tu.Result, store);
}).Wait();
}
}
});
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total Users To Convert: {0}", userIds.Count);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total Users Successfully Converted: {0}", iUserSuccessConvert);
Console.WriteLine("Total Users Failed to Convert: {0}", iUserFailureConvert);
if (iUserFailureConvert > 0)
{
Console.WriteLine("User Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
else
{
Console.WriteLine("Cannot find UserCollection. Check app.config appSettings for correct DocumentDB connection. If correct, no migration needed.");
}
}
}
DisplayAnyKeyToExit();
}
public static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
var builder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false);
Configuration = builder.Build();
IdentityConfiguration sourceConfig = new IdentityConfiguration
{
TablePrefix = Configuration.GetSection("source:IdentityConfiguration:TablePrefix")?.Value,
StorageConnectionString =
Configuration.GetSection("source:IdentityConfiguration:StorageConnectionString")?.Value,
LocationMode = Configuration.GetSection("source:IdentityConfiguration:LocationMode")?.Value ??
string.Empty,
UserTableName = Configuration.GetSection("source:IdentityConfiguration:UserTableName")?.Value ??
string.Empty,
IndexTableName = Configuration.GetSection("source:IdentityConfiguration:IndexTableName")?.Value ??
string.Empty,
RoleTableName = Configuration.GetSection("source:IdentityConfiguration:RoleTableName")?.Value ??
string.Empty
};
IdentityConfiguration targetConfig = new IdentityConfiguration
{
TablePrefix = Configuration.GetSection("target:IdentityConfiguration:TablePrefix")?.Value,
StorageConnectionString =
Configuration.GetSection("target:IdentityConfiguration:StorageConnectionString")?.Value,
LocationMode = Configuration.GetSection("target:IdentityConfiguration:LocationMode")?.Value ??
string.Empty,
UserTableName = Configuration.GetSection("target:IdentityConfiguration:UserTableName")?.Value ??
string.Empty,
IndexTableName = Configuration.GetSection("target:IdentityConfiguration:IndexTableName")?.Value ??
string.Empty,
RoleTableName = Configuration.GetSection("target:IdentityConfiguration:RoleTableName")?.Value ??
string.Empty
};
Console.WriteLine("MaxDegreeOfParallelism: {0}", iMaxDegreesParallel);
Console.WriteLine("PageSize: {0}", iPageSize);
Console.WriteLine("MigrateCommand: {0}", MigrateCommand);
var migration = MigrationFactory.CreateMigration(MigrateCommand);
using (IdentityCloudContext targetContext = new IdentityCloudContext(targetConfig))
{
Task.WhenAll(targetContext.IndexTable.CreateIfNotExistsAsync(),
targetContext.UserTable.CreateIfNotExistsAsync(),
targetContext.RoleTable.CreateIfNotExistsAsync()).Wait();
Console.WriteLine($"Target IndexTable: {targetContext.IndexTable.Name}");
Console.WriteLine($"Target UserTable: {targetContext.UserTable.Name}");
Console.WriteLine($"Target RoleTable: {targetContext.RoleTable.Name}");
string entityRecordName = "Users";
using (IdentityCloudContext sourceContext = new IdentityCloudContext(sourceConfig))
{
Console.WriteLine($"Source IndexTable: {sourceContext.IndexTable.Name}");
Console.WriteLine($"Source UserTable: {sourceContext.UserTable.Name}");
Console.WriteLine($"Source RoleTable: {sourceContext.RoleTable.Name}");
DateTime startLoad = DateTime.UtcNow;
//var allDataList = new List<DynamicTableEntity>(iPageSize);
TableQuery tq = migration.GetSourceTableQuery();
tq.TakeCount = iPageSize;
TableContinuationToken continueToken = new TableContinuationToken();
int iSkippedUserCount = 0;
int iSkippedPageCount = 0;
int iPageCounter = 0;
while (continueToken != null)
{
DateTime batchStart = DateTime.UtcNow;
CloudTable sourceTable = sourceContext.UserTable;
if (MigrateCommand == MigrationFactory.Roles)
{
sourceTable = sourceContext.RoleTable;
entityRecordName = "Role and Role Claims";
}
var sourceResults = sourceTable.ExecuteQuerySegmentedAsync(tq, continueToken).Result;
continueToken = sourceResults.ContinuationToken;
int batchCount = sourceResults.Count(migration.UserWhereFilter);
iUserTotal += batchCount;
iPageCounter++;
bool includePage = (iStartPage == -1 || iPageCounter >= iStartPage) && (iFinishPage == -1 || iPageCounter <= iFinishPage);
if (includePage)
{
if (migrateOption)
{
var name = entityRecordName;
migration.ProcessMigrate(targetContext, sourceContext, sourceResults.Results, iMaxDegreesParallel,
() =>
{
Interlocked.Increment(ref iUserSuccessConvert);
Console.WriteLine($"{name}(s) Complete: {iUserSuccessConvert}");
},
(exMessage) =>
{
if (!string.IsNullOrWhiteSpace(exMessage))
{
userIdFailures.Add(exMessage);
}
Interlocked.Increment(ref iUserFailureConvert);
});
}
}
else
{
iSkippedPageCount++;
iSkippedUserCount += batchCount;
}
Console.WriteLine("Page: {2}{3}, {4} Batch: {1}: {0} seconds", (DateTime.UtcNow - batchStart).TotalSeconds, batchCount, iPageCounter, includePage ? string.Empty : "(Skipped)", entityRecordName);
//Are we done yet?
if (iFinishPage > 0 && iPageCounter >= iFinishPage)
{
break;
}
}
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total {2} Skipped: {0}, Total Pages: {1}", iSkippedUserCount, iSkippedPageCount, entityRecordName);
Console.WriteLine("Total {2} To Convert: {0}, Total Pages: {1}", iUserTotal - iSkippedUserCount, iPageCounter - iSkippedPageCount, entityRecordName);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total {1} Successfully Converted: {0}", iUserSuccessConvert, entityRecordName);
Console.WriteLine("Total {1} Failed to Convert: {0}", iUserFailureConvert, entityRecordName);
if (iUserFailureConvert > 0)
{
Console.WriteLine($"{entityRecordName} Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
}
DisplayAnyKeyToExit();
}
private List <DynamicTableEntity> GetUserEntitiesBySourceId(string userPartitionKey, IdentityCloudContext sourcesContext)
{
List <DynamicTableEntity> results = new List <DynamicTableEntity>(1000);
TableQuery tq = new TableQuery();
string partitionKeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, userPartitionKey);
string keyVersionFilter = TableQuery.GenerateFilterConditionForDouble("KeyVersion", QueryComparisons.LessThan, KeyHelper.KeyVersion);
tq.FilterString = TableQuery.CombineFilters(partitionKeyFilter, TableOperators.And, keyVersionFilter);
TableContinuationToken token = new TableContinuationToken();
while (token != null)
{
var r = sourcesContext.UserTable.ExecuteQuerySegmented(tq, token);
token = r.ContinuationToken;
results.AddRange(r.Results);
}
return(results);
}
private DynamicTableEntity ConvertToTargetRoleEntity(DynamicTableEntity sourceEntity, IdentityCloudContext sourcesContext)
{
DynamicTableEntity targetEntity = null;
//RoleClaim record
if (sourceEntity.PartitionKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityRole) &&
sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityRoleClaim))
{
sourceEntity.Properties.TryGetValue("ClaimType", out EntityProperty claimTypeProperty);
string claimType = claimTypeProperty.StringValue;
sourceEntity.Properties.TryGetValue("ClaimValue", out EntityProperty claimValueProperty);
string claimValue = claimValueProperty.StringValue;
string roleName = GetRoleNameBySourceId(sourceEntity.PartitionKey, sourcesContext);
targetEntity = new DynamicTableEntity(KeyHelper.GenerateRowKeyIdentityRole(roleName),
KeyHelper.GenerateRowKeyIdentityRoleClaim(claimType, claimValue), Constants.ETagWildcard, sourceEntity.Properties);
targetEntity.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
}
else if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityRole))
{
sourceEntity.Properties.TryGetValue("Name", out EntityProperty roleNameProperty);
string roleName = roleNameProperty.StringValue;
targetEntity = new DynamicTableEntity(KeyHelper.GeneratePartitionKeyIdentityRole(roleName), KeyHelper.GenerateRowKeyIdentityRole(roleName), Constants.ETagWildcard, sourceEntity.Properties);
targetEntity.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
}
return(targetEntity);
}
public RoleStore(IdentityCloudContext context)
: base(context)
{
}
private static async Task<TempUser> ConvertUser(TempUser user, IdentityCloudContext<IdentityUser> ic, bool deleteDocuments)
{
var response = await ic.Client.ReplaceDocumentAsync(user.User, new Microsoft.Azure.Documents.Client.RequestOptions()
{
SessionToken = ic.SessionToken
});
ic.SetSessionTokenIfEmpty(response.SessionToken);
if (deleteDocuments)
{
foreach (var r in user.Roles)
{
await ic.Client.DeleteDocumentAsync(r.SelfLink);
}
foreach (var c in user.Claims)
{
await ic.Client.DeleteDocumentAsync(c.SelfLink);
}
foreach (var l in user.Logins)
{
await ic.Client.DeleteDocumentAsync(l.SelfLink);
}
}
return user;
}
public void IdentityCloudContextCtors()
{
string strValidConnection = CloudConfigurationManager.GetSetting(
ElCamino.AspNet.Identity.AzureTable.Constants.AppSettingsKeys.DefaultStorageConnectionStringKey);
var currentConfig = ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None);
var section = currentConfig.Sections[IdentityConfigurationSection.Name];
if (section == null)
{
currentConfig.Sections.Add(IdentityConfigurationSection.Name,
new IdentityConfigurationSection()
{
TablePrefix = string.Empty,
StorageConnectionString = strValidConnection
});
currentConfig.Save(ConfigurationSaveMode.Modified);
}
var ic = new IdentityCloudContext();
Assert.NotNull(ic);
//Pass in valid connection string
var icc = new IdentityCloudContext(strValidConnection);
icc.Dispose();
ic = new IdentityCloudContext(new IdentityConfiguration()
{
TablePrefix = string.Empty,
StorageConnectionString = strValidConnection
});
using (UserStore <IdentityUser> store = new UserStore <IdentityUser>(
new IdentityCloudContext(new IdentityConfiguration()
{
StorageConnectionString = strValidConnection,
TablePrefix = "My"
})))
{
var task = store.CreateTablesIfNotExists();
task.Wait();
}
currentConfig.Sections.Remove(IdentityConfigurationSection.Name);
currentConfig.Save(ConfigurationSaveMode.Modified);
ConfigurationManager.RefreshSection(IdentityConfigurationSection.Name);
using (UserStore <IdentityUser> store = new UserStore <IdentityUser>(
new IdentityCloudContext()))
{
var task = store.CreateTablesIfNotExists();
task.Wait();
}
currentConfig.Sections.Add(IdentityConfigurationSection.Name,
new IdentityConfigurationSection()
{
TablePrefix = string.Empty,
StorageConnectionString = strValidConnection,
LocationMode = "PrimaryThenSecondary"
});
currentConfig.Save(ConfigurationSaveMode.Modified);
ConfigurationManager.RefreshSection(IdentityConfigurationSection.Name);
string strInvalidConnectionStringKey = Guid.NewGuid().ToString();
var testAppsettings = new IdentityCloudContext(ElCamino.AspNet.Identity.AzureTable.Constants.AppSettingsKeys.DefaultStorageConnectionStringKey);
testAppsettings.Dispose();
try
{
ic = new IdentityCloudContext(new IdentityConfiguration()
{
TablePrefix = string.Empty,
StorageConnectionString = strValidConnection,
LocationMode = "InvalidLocationMode"
});
}
catch (ArgumentException) { }
try
{
ic = new IdentityCloudContext(strInvalidConnectionStringKey);
}
catch (System.FormatException) { }
try
{
ic = new IdentityCloudContext(string.Empty);
}
catch (MissingManifestResourceException) { }
//----------------------------------------------
var iucc = new IdentityCloudContext();
Assert.NotNull(iucc);
try
{
iucc = new IdentityCloudContext(strInvalidConnectionStringKey);
}
catch (System.FormatException) { }
try
{
iucc = new IdentityCloudContext(string.Empty);
}
catch (MissingManifestResourceException) { }
//------------------------------------------
var i2 = new IdentityCloudContext();
Assert.NotNull(i2);
try
{
i2 = new IdentityCloudContext(Guid.NewGuid().ToString());
}
catch (System.FormatException) { }
try
{
i2 = new IdentityCloudContext(string.Empty);
}
catch (MissingManifestResourceException) { }
try
{
var i3 = new IdentityCloudContext();
i3.Dispose();
var table = i3.RoleTable;
}
catch (ObjectDisposedException) { }
try
{
var i4 = new IdentityCloudContext();
i4.Dispose();
var table = i4.UserTable;
}
catch (ObjectDisposedException) { }
try
{
IdentityConfiguration iconfig = null;
var i5 = new IdentityCloudContext(iconfig);
}
catch (ArgumentNullException) { }
}
public UserStore(IdentityCloudContext context)
: base(context)
{
}
public static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
var builder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false);
Configuration = builder.Build();
IdentityConfiguration idconfig = new IdentityConfiguration();
idconfig.TablePrefix = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:TablePrefix").Value;
idconfig.StorageConnectionString = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:StorageConnectionString").Value;
idconfig.LocationMode = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:LocationMode").Value;
Console.WriteLine("MaxDegreeOfParallelism: {0}", iMaxdegreesparallel);
Console.WriteLine("PageSize: {0}", iPageSize);
Console.WriteLine("MigrateCommand: {0}", MigrateCommand);
var migrateIndex = MigrateIndexFactory.CreateMigrateIndex(MigrateCommand);
using (IdentityCloudContext ic = new IdentityCloudContext(idconfig))
{
DateTime startLoad = DateTime.UtcNow;
var allDataList = new List <DynamicTableEntity>(iPageSize);
TableQuery tq = migrateIndex.GetUserTableQuery();
tq.TakeCount = iPageSize;
TableContinuationToken continueToken = new TableContinuationToken();
int iSkippedUserCount = 0;
int iSkippedPageCount = 0;
int iPageCounter = 0;
while (continueToken != null)
{
DateTime batchStart = DateTime.UtcNow;
var userResults = ic.UserTable.ExecuteQuerySegmentedAsync(tq, continueToken).Result;
continueToken = userResults.ContinuationToken;
int batchCount = userResults.Count(migrateIndex.UserWhereFilter);
iUserTotal += batchCount;
iPageCounter++;
bool includePage = (iStartPage == -1 || iPageCounter >= iStartPage) && (iFinishPage == -1 || iPageCounter <= iFinishPage);
if (includePage)
{
if (migrateOption)
{
migrateIndex.ProcessMigrate(ic, userResults.Results, iMaxdegreesparallel,
() =>
{
Interlocked.Increment(ref iUserSuccessConvert);
},
(exMessage) =>
{
if (!string.IsNullOrWhiteSpace(exMessage))
{
userIdFailures.Add(exMessage);
}
Interlocked.Increment(ref iUserFailureConvert);
});
}
}
else
{
iSkippedPageCount++;
iSkippedUserCount += batchCount;
}
Console.WriteLine("Page: {2}{3}, Users Batch: {1}: {0} seconds", (DateTime.UtcNow - batchStart).TotalSeconds, batchCount, iPageCounter, includePage ? string.Empty : "(Skipped)");
//Are we done yet?
if (iFinishPage > 0 && iPageCounter >= iFinishPage)
{
break;
}
}
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total Users Skipped: {0}, Total Pages: {1}", iSkippedUserCount, iSkippedPageCount);
Console.WriteLine("Total Users To Convert: {0}, Total Pages: {1}", iUserTotal - iSkippedUserCount, iPageCounter - iSkippedPageCount);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total Users Successfully Converted: {0}", iUserSuccessConvert);
Console.WriteLine("Total Users Failed to Convert: {0}", iUserFailureConvert);
if (iUserFailureConvert > 0)
{
Console.WriteLine("User Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
DisplayAnyKeyToExit();
}
static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
using (IdentityCloudContext <IdentityUser> ic = new IdentityCloudContext <IdentityUser>())
{
using (UserStore <IdentityUser> store = new UserStore <IdentityUser>(ic))
{
userCollection = ic.Client.CreateDocumentCollectionQuery(ic.Database.CollectionsLink)
.Where(c => c.Id == Constants.DocumentCollectionIds.UsersCollection)
.ToList()
.FirstOrDefault();
if (userCollection != null)
{
List <Document> lroles = null;
List <Document> llogins = null;
List <Document> lclaims = null;
Dictionary <string, Document> lusers = null;
DateTime startLoad = DateTime.UtcNow;
var allDataList = new List <Document>(2000);
Task[] tasks = new Task[] {
new TaskFactory().StartNew(() =>
{
var qRoles = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.RoleName != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
lroles = qRoles.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return(d); }).ToList();
allDataList.AddRange(lroles);
Console.WriteLine("Roles to convert: {0}", lroles.Count);
}),
new TaskFactory().StartNew(() =>
{
var qLogins = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.LoginProvider != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
llogins = qLogins.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return(d); }).ToList();
allDataList.AddRange(llogins);
Console.WriteLine("Logins to convert: {0}", llogins.Count);
}),
new TaskFactory().StartNew(() =>
{
var qClaims = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.ClaimType != '' ",
new Microsoft.Azure.Documents.Client.FeedOptions());
lclaims = qClaims.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return(d); }).ToList();
allDataList.AddRange(lclaims);
Console.WriteLine("Claims to convert: {0}", lclaims.Count);
}),
new TaskFactory().StartNew(() =>
{
var qUser = ic.Client.CreateDocumentQuery(userCollection.SelfLink,
"SELECT VALUE r FROM root r WHERE r.id = r.UserId",
new Microsoft.Azure.Documents.Client.FeedOptions());
lusers = qUser.ToList().Select(r => { Document d = ConvertDynamicToDoc(r); return(d); }).ToDictionary(d => d.Id);
Console.WriteLine("Total Users: {0}", lusers.Count);
})
};
Task.WaitAll(tasks);
Console.WriteLine("Load Roles, Claims, Logins and Users: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
List <string> userIds = allDataList.Select(dl => dl.GetPropertyValue <string>("UserId")).Distinct().ToList();
var result2 = Parallel.ForEach <string>(userIds, (userId) =>
{
Document user;
if (!lusers.TryGetValue(userId, out user))
{
Console.WriteLine("User document not found: {0}", userId);
return;
}
//Get all of the docs with the same UserId
Task <TempUser> tempTask = CreateTempUser(user, allDataList.Where(d => d.GetPropertyValue <string>("UserId") == user.Id).ToList());
tempTask.Wait();
TempUser temp = tempTask.Result;
if (temp.Roles.Count > 0 ||
temp.Claims.Count > 0 ||
temp.Logins.Count > 0)
{
if (migrateOption)
{
ConvertUser(temp, ic, deleteOption).ContinueWith((tu) =>
{
return(ConfirmUserConvert(tu.Result, store));
}).Wait();
}
}
});
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total Users To Convert: {0}", userIds.Count);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total Users Successfully Converted: {0}", iUserSuccessConvert);
Console.WriteLine("Total Users Failed to Convert: {0}", iUserFailureConvert);
if (iUserFailureConvert > 0)
{
Console.WriteLine("User Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
else
{
Console.WriteLine("Cannot find UserCollection. Check app.config appSettings for correct DocumentDB connection. If correct, no migration needed.");
}
}
}
DisplayAnyKeyToExit();
}