public void ProcessMigrate(IdentityCloudContext targetContext,
IdentityCloudContext sourceContext,
IList <DynamicTableEntity> userResults,
int maxDegreesParallel,
Action updateComplete = null,
Action <string> updateError = null)
{
var userIds = userResults
.Where(UserWhereFilter)
.Select(d => new { UserId = d.PartitionKey, Email = d.Properties["Email"].StringValue })
.ToList();
var result2 = Parallel.ForEach(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (userId) =>
{
//Add the email index
try
{
IdentityUserIndex index = CreateEmailIndex(userId.UserId, userId.Email);
var r = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}\t{1}", userId.UserId, ex.Message));
}
});
}
public void ProcessMigrate(IdentityCloudContext targetContext, IdentityCloudContext sourceContext, IList <DynamicTableEntity> sourceUserResults, int maxDegreesParallel, Action updateComplete = null, Action <string> updateError = null)
{
var rolesAndClaims = sourceUserResults
.Where(UserWhereFilter);
var result2 = Parallel.ForEach(rolesAndClaims, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (dte) =>
{
//Add the role or claim index
try
{
IdentityUserIndex index = new IdentityUserIndex()
{
Id = dte.PartitionKey,
PartitionKey = dte.RowKey,
RowKey = dte.PartitionKey,
KeyVersion = _keyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
var r = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke(string.Format("{0}\t{1}", dte.PartitionKey, ex.Message));
}
});
}
public void ProcessMigrate(IdentityCloudContext targetContext,
IdentityCloudContext sourceContext,
IList <DynamicTableEntity> sourceUserResults,
int maxDegreesParallel,
Action updateComplete = null,
Action <string> updateError = null)
{
var userIds = sourceUserResults
.Where(UserWhereFilter)
.Select(d => new
{
UserId = d.PartitionKey,
LoginProvider = d.Properties["LoginProvider"].StringValue,
ProviderKey = d.Properties["ProviderKey"].StringValue
})
.ToList();
Parallel.ForEach(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = maxDegreesParallel
}, (userId) =>
{
//Add the email index
try
{
IdentityUserIndex index = CreateLoginIndex(userId.UserId, userId.LoginProvider, userId.ProviderKey);
var unused = targetContext.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
updateComplete?.Invoke();
}
catch (Exception ex)
{
updateError?.Invoke($"{userId.UserId}\t{ex.Message}");
}
});
}
private (List <DynamicTableEntity> targetUserEntities, List <ITableEntity> targetUserIndexes) ConvertToTargetUserEntities(string userId, List <DynamicTableEntity> sourceUserEntities)
{
List <DynamicTableEntity> targetUserEntities = new List <DynamicTableEntity>(100);
List <ITableEntity> targetUserIndexes = new List <ITableEntity>(100);
foreach (DynamicTableEntity sourceEntity in sourceUserEntities)
{
if (sourceEntity.PartitionKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserId))
{
string targetUserPartitionKey = KeyHelper.GenerateRowKeyUserId(userId);
//User record
if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserId))
{
//New User
//Add UserName Index
//Add Email Index
DynamicTableEntity tgtDte = new DynamicTableEntity(targetUserPartitionKey, targetUserPartitionKey, Constants.ETagWildcard, sourceEntity.Properties);
tgtDte.Properties["Id"] = new EntityProperty(userId);
tgtDte.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
targetUserEntities.Add(tgtDte);
//UserName index
tgtDte.Properties.TryGetValue("UserName", out EntityProperty userNameProperty);
string userNameKey = KeyHelper.GenerateRowKeyUserName(userNameProperty.StringValue);
IdentityUserIndex userNameIndex = new IdentityUserIndex()
{
Id = targetUserPartitionKey,
PartitionKey = userNameKey,
RowKey = targetUserPartitionKey,
KeyVersion = KeyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
targetUserIndexes.Add(userNameIndex);
//Email index - only if email exists
if (tgtDte.Properties.TryGetValue("Email", out EntityProperty emailProperty))
{
string emailKey = KeyHelper.GenerateRowKeyUserEmail(emailProperty.StringValue);
IdentityUserIndex emailIndex = new IdentityUserIndex()
{
Id = targetUserPartitionKey,
PartitionKey = emailKey,
RowKey = targetUserPartitionKey,
KeyVersion = KeyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
targetUserIndexes.Add(emailIndex);
}
continue;
}
//User Claim record
if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserClaim))
{
//New User Claim
//Add Claim Index
sourceEntity.Properties.TryGetValue("ClaimType", out EntityProperty claimTypeProperty);
string claimType = claimTypeProperty.StringValue;
sourceEntity.Properties.TryGetValue("ClaimValue", out EntityProperty claimValueProperty);
string claimValue = claimValueProperty.StringValue;
string targetUserRowKey = KeyHelper.GenerateRowKeyIdentityUserClaim(claimType, claimValue);
DynamicTableEntity tgtDte = new DynamicTableEntity(targetUserPartitionKey, targetUserRowKey, Constants.ETagWildcard, sourceEntity.Properties);
tgtDte.Properties["UserId"] = new EntityProperty(userId);
tgtDte.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
targetUserEntities.Add(tgtDte);
//Claim index
IdentityUserIndex claimIndex = new IdentityUserIndex()
{
Id = targetUserPartitionKey,
PartitionKey = targetUserRowKey,
RowKey = targetUserPartitionKey,
KeyVersion = KeyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
targetUserIndexes.Add(claimIndex);
continue;
}
//User Logon record
if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserLogin))
{
//New User Logon
//Add Logon Index
sourceEntity.Properties.TryGetValue("LoginProvider", out EntityProperty loginProviderProperty);
string loginProvider = loginProviderProperty.StringValue;
sourceEntity.Properties.TryGetValue("ProviderKey", out EntityProperty providerKeyProperty);
string providerKey = providerKeyProperty.StringValue;
string targetUserRowKey = KeyHelper.GenerateRowKeyIdentityUserLogin(loginProvider, providerKey);
DynamicTableEntity tgtDte = new DynamicTableEntity(targetUserPartitionKey, targetUserRowKey, Constants.ETagWildcard, sourceEntity.Properties);
tgtDte.Properties["UserId"] = new EntityProperty(userId);
tgtDte.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
targetUserEntities.Add(tgtDte);
//Logon index
IdentityUserIndex logonIndex = new IdentityUserIndex()
{
Id = targetUserPartitionKey,
PartitionKey = KeyHelper.GeneratePartitionKeyIndexByLogin(loginProvider, providerKey),
RowKey = KeyHelper.GenerateRowKeyIdentityUserLogin(loginProvider, providerKey),
KeyVersion = KeyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
targetUserIndexes.Add(logonIndex);
continue;
}
//User Role record
if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserRole))
{
//New User Role
//Add Role Index
sourceEntity.Properties.TryGetValue("RoleName", out EntityProperty roleNameProperty);
string roleName = roleNameProperty.StringValue;
string targetUserRowKey = KeyHelper.GenerateRowKeyIdentityUserRole(roleName);
DynamicTableEntity tgtDte = new DynamicTableEntity(targetUserPartitionKey, targetUserRowKey, Constants.ETagWildcard, sourceEntity.Properties);
tgtDte.Properties["UserId"] = new EntityProperty(userId);
tgtDte.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
targetUserEntities.Add(tgtDte);
//Role index
IdentityUserIndex roleIndex = new IdentityUserIndex()
{
Id = targetUserPartitionKey,
PartitionKey = targetUserRowKey,
RowKey = targetUserPartitionKey,
KeyVersion = KeyHelper.KeyVersion,
ETag = Constants.ETagWildcard
};
targetUserIndexes.Add(roleIndex);
continue;
}
//User Token record
if (sourceEntity.RowKey.StartsWith(Constants.RowKeyConstants.PreFixIdentityUserToken))
{
//New User Token
sourceEntity.Properties.TryGetValue("LoginProvider", out EntityProperty loginProviderProperty);
string loginProvider = loginProviderProperty.StringValue;
sourceEntity.Properties.TryGetValue("TokenName", out EntityProperty tokenNameProperty);
string tokenName = tokenNameProperty.StringValue;
string targetUserRowKey = KeyHelper.GenerateRowKeyIdentityUserToken(loginProvider, tokenName);
DynamicTableEntity tgtDte = new DynamicTableEntity(targetUserPartitionKey, targetUserRowKey, Constants.ETagWildcard, sourceEntity.Properties);
tgtDte.Properties["UserId"] = new EntityProperty(userId);
tgtDte.Properties["KeyVersion"] = new EntityProperty(KeyHelper.KeyVersion);
targetUserEntities.Add(tgtDte);
continue;
}
}
}
return(targetUserEntities, targetUserIndexes);
}
public static void Main(string[] args)
{
if (!ValidateArgs(args))
{
return;
}
var builder = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false);
Configuration = builder.Build();
IdentityConfiguration idconfig = new IdentityConfiguration();
idconfig.TablePrefix = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:TablePrefix").Value;
idconfig.StorageConnectionString = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:StorageConnectionString").Value;
idconfig.LocationMode = Configuration.GetSection("IdentityAzureTable:IdentityConfiguration:LocationMode").Value;
Console.WriteLine("MaxDegreeOfParallelism: {0}", iMaxdegreesparallel);
Console.WriteLine("PageSize: {0}", iPageSize);
using (IdentityCloudContext ic = new IdentityCloudContext(idconfig))
{
DateTime startLoad = DateTime.UtcNow;
var allDataList = new List <DynamicTableEntity>(iPageSize);
TableQuery tq = new TableQuery();
tq.SelectColumns = new List <string>()
{
"PartitionKey", "RowKey", "Email"
};
string partitionFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThanOrEqual, Constants.RowKeyConstants.PreFixIdentityUserName),
TableOperators.And,
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, "V_"));
string rowFilter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, Constants.RowKeyConstants.PreFixIdentityUserName),
TableOperators.And,
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, "V_"));
tq.FilterString = TableQuery.CombineFilters(partitionFilter, TableOperators.And, rowFilter);
tq.TakeCount = iPageSize;
TableContinuationToken continueToken = new TableContinuationToken();
int iSkippedUserCount = 0;
int iSkippedPageCount = 0;
int iPageCounter = 0;
while (continueToken != null)
{
DateTime batchStart = DateTime.UtcNow;
var userResults = ic.UserTable.ExecuteQuerySegmentedAsync(tq, continueToken).Result;
continueToken = userResults.ContinuationToken;
List <Tuple <string, string> > userIds = userResults.Results
.Where(d => !string.IsNullOrWhiteSpace(d.Properties["Email"].StringValue))
.Select(d => new Tuple <string, string>(d.PartitionKey, d.Properties["Email"].StringValue))
.ToList();
int batchCount = userIds.Count();
iUserTotal += batchCount;
iPageCounter++;
bool includePage = (iStartPage == -1 || iPageCounter >= iStartPage) && (iFinishPage == -1 || iPageCounter <= iFinishPage);
if (includePage)
{
var result2 = Parallel.ForEach <Tuple <string, string> >(userIds, new ParallelOptions()
{
MaxDegreeOfParallelism = iMaxdegreesparallel
}, (userId) =>
{
if (migrateOption)
{
//Add the email index
try
{
IdentityUserIndex index = CreateEmailIndex(userId.Item1, userId.Item2);
var r = ic.IndexTable.ExecuteAsync(TableOperation.InsertOrReplace(index)).Result;
Interlocked.Increment(ref iUserSuccessConvert);
}
catch (Exception ex)
{
userIdFailures.Add(string.Format("{0}\t{1}", userId.Item1, ex.Message));
Interlocked.Increment(ref iUserFailureConvert);
}
}
});
}
else
{
iSkippedPageCount++;
iSkippedUserCount += batchCount;
}
Console.WriteLine("Page: {2}{3}, Users Batch: {1}: {0} seconds", (DateTime.UtcNow - batchStart).TotalSeconds, batchCount, iPageCounter, includePage ? string.Empty : "(Skipped)");
//Are we done yet?
if (iFinishPage > 0 && iPageCounter >= iFinishPage)
{
break;
}
}
Console.WriteLine("");
Console.WriteLine("Elapsed time: {0} seconds", (DateTime.UtcNow - startLoad).TotalSeconds);
Console.WriteLine("Total Users Skipped: {0}, Total Pages: {1}", iSkippedUserCount, iSkippedPageCount);
Console.WriteLine("Total Users To Convert: {0}, Total Pages: {1}", iUserTotal - iSkippedUserCount, iPageCounter - iSkippedPageCount);
Console.WriteLine("");
if (migrateOption)
{
Console.WriteLine("Total Users Successfully Converted: {0}", iUserSuccessConvert);
Console.WriteLine("Total Users Failed to Convert: {0}", iUserFailureConvert);
if (iUserFailureConvert > 0)
{
Console.WriteLine("User Ids Failed:");
foreach (string s in userIdFailures)
{
Console.WriteLine(s);
}
}
}
}
DisplayAnyKeyToExit();
}
