private async Task PopulateWorkItemMigrationState()
{
//dictionary of target workitem id to source id - these workitems have been migrated before
var existingWorkItems = ValidationContext.WorkItemsMigrationState.Where(wi => wi.MigrationState == WorkItemMigrationState.State.Existing);
var totalNumberOfBatches = ClientHelpers.GetBatchCount(existingWorkItems.Count(), Constants.BatchSize);
await existingWorkItems.Batch(Constants.BatchSize).ForEachAsync(ValidationContext.Config.Parallelism, async(batchWorkItemMigrationState, batchId) =>
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Started");
Dictionary <int, WorkItemMigrationState> targetToWorkItemMigrationState = batchWorkItemMigrationState.ToDictionary(k => k.TargetId.Value, v => v);
//read the target work items
IList <WorkItem> targetWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(ValidationContext.TargetClient.WorkItemTrackingHttpClient, batchWorkItemMigrationState.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IDictionary <int, WorkItemRelation> targetIdToHyperlinkToSourceRelationMapping = GetTargetIdToHyperlinkToSourceRelationMapping(targetWorkItems, targetToWorkItemMigrationState);
ProcessUpdatedSourceWorkItems(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
StoreWorkItemBatchRelationInformationOnContext(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Completed in {stopwatch.Elapsed.TotalSeconds}s");
});
}
private async Task MigratePhase2()
{
Logger.LogInformation("Starting migration phase 2");
IEnumerable <WorkItemMigrationState> successfulWorkItemMigrationStates;
// when skip existing config flag is on and this work item was existing, continue to next work item.
if (context.Config.SkipExisting)
{
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create);
}
else
{
// allow any Create, OR Existing with UpdatePhase2
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create || (a.MigrationState == WorkItemMigrationState.State.Existing && a.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase2)));
}
var phase2WorkItemsToUpdateCount = successfulWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase2WorkItemsToUpdateCount, Constants.BatchSize);
if (phase2WorkItemsToUpdateCount == 0)
{
Logger.LogInformation(LogDestination.File, "No work items to process for phase 2");
return;
}
await successfulWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
Logger.LogTrace(LogDestination.File, $"Reading Phase 2 source and target work items for batch {batchId} of {totalNumberOfBatches}");
// make web call to get source and target work items
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
IList <WorkItem> targetWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.TargetClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
batchContext.SourceWorkItemIdToTargetWorkItemIdMapping = workItemMigrationStateBatch.ToDictionary(key => key.SourceId, value => value.TargetId.Value);
foreach (var sourceWorkItem in sourceWorkItemsInBatch)
{
int targetId = Migrator.GetTargetId(sourceWorkItem.Id.Value, workItemMigrationStateBatch);
batchContext.TargetIdToSourceWorkItemMapping.Add(targetId, sourceWorkItem);
}
Logger.LogTrace(LogDestination.File, $"Generating Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var sourceIdToWitBatchRequests = await GenerateWitBatchRequestsForPhase2Batch(batchContext, batchId, workItemMigrationStateBatch, sourceWorkItemsInBatch, targetWorkItemsInBatch);
Logger.LogTrace(LogDestination.File, $"Saving Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var phase2ApiWrapper = new Phase2ApiWrapper();
await phase2ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
Logger.LogTrace(LogDestination.File, $"Completed Phase 2 for batch {batchId} of {totalNumberOfBatches}");
});
Logger.LogInformation("Completed migration phase 2");
}
private async Task MigratePhase3()
{
IEnumerable <IPhase3Processor> phase3Processors = ClientHelpers.GetProcessorInstances <IPhase3Processor>(context.Config);
if (phase3Processors != null && !phase3Processors.Any())
{
// nothing to do if no phase 3 processors are enabled
return;
}
// Phase1 or Phase2 have completed, and FailureReason == None
IEnumerable <WorkItemMigrationState> successfullyMigratedWorkItemMigrationStates = context.WorkItemsMigrationState.Where(w => (w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) || w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2)) && w.FailureReason == FailureReason.None);
var phase3WorkItemsToUpdateCount = successfullyMigratedWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase3WorkItemsToUpdateCount, Constants.BatchSize);
if (phase3WorkItemsToUpdateCount == 0)
{
return;
}
await successfullyMigratedWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
IList <(int SourceId, WitBatchRequest WitBatchRequest)> sourceIdToWitBatchRequests = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
foreach (WorkItem sourceWorkItem in sourceWorkItemsInBatch)
{
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
foreach (IPhase3Processor processor in phase3Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, null, sourceWorkItem, null);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, sourceWorkItem.Id.Value);
sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest));
}
}
var phase3ApiWrapper = new Phase3ApiWrapper();
await phase3ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
});
}
private async Task ValidateWorkItemMetadata()
{
Logger.LogInformation("Starting work item metadata validation");
var validators = ClientHelpers.GetInstances <IWorkItemValidator>();
foreach (var validator in validators)
{
await validator.Prepare(context);
}
var totalNumberOfBatches = ClientHelpers.GetBatchCount(context.WorkItemIdsUris.Count, Constants.BatchSize);
await context.WorkItemIdsUris.Keys.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemIds, batchId) =>
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: Starting");
var workItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(
context.SourceClient.WorkItemTrackingHttpClient,
workItemIds,
context.RequestedFields);
foreach (var validator in validators)
{
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: {validator.Name}");
foreach (var workItem in workItems)
{
await validator.Validate(context, workItem);
}
}
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: Completed in {stopwatch.Elapsed.TotalSeconds}s");
});
Logger.LogInformation("Completed work item metadata validation");
}
/// <summary>
/// Populates batchContext.WorkItems
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="workItemIds"></param>
/// <param name="batchContext"></param>
/// <param name="expand"></param>
/// <returns></returns>
public static async Task ReadSourceWorkItems(IMigrationContext migrationContext, IEnumerable <int> workItemIds, IBatchMigrationContext batchContext, WorkItemExpand?expand = WorkItemExpand.All)
{
batchContext.SourceWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, workItemIds, expand : expand);
}