private async Task <IList <(int SourceId, WitBatchRequest WitBatchRequest)> > GenerateWitBatchRequestsForPhase2Batch(IBatchMigrationContext batchContext, int batchId, IList <WorkItemMigrationState> workItemMigrationState, IList <WorkItem> sourceWorkItems, IList <WorkItem> targetWorkItems)
{
IList <(int SourceId, WitBatchRequest WitBatchRequest)> result = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(context.Config);
foreach (IPhase2Processor processor in phase2Processors)
{
Logger.LogInformation(LogDestination.File, $"Starting preprocessing of phase 2 step {processor.Name} for batch {batchId}");
await processor.Preprocess(context, batchContext, sourceWorkItems, targetWorkItems);
Logger.LogInformation(LogDestination.File, $"Completed preprocessing of phase 2 step {processor.Name} for batch {batchId}");
}
foreach (var sourceToTarget in batchContext.SourceWorkItemIdToTargetWorkItemIdMapping)
{
int sourceId = sourceToTarget.Key;
int targetId = sourceToTarget.Value;
WorkItem sourceWorkItem = sourceWorkItems.First(a => a.Id == sourceId);
WorkItem targetWorkItem = targetWorkItems.First(a => a.Id == targetId);
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
WorkItemMigrationState state = workItemMigrationState.First(a => a.SourceId == sourceId);
state.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, sourceId);
ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(context.Config)).Select(b => b.Name));
enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations);
foreach (IPhase2Processor processor in phase2Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, batchContext, sourceWorkItem, targetWorkItem);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
jsonPatchOperations.Add(GetAddHyperlinkWithCommentOperation(targetWorkItems, state, sourceId, targetId, sourceWorkItem, enabledPhaseStatuses));
if (this.context.Config.IncludeWebLink)
{
var link = (ReferenceLink)sourceWorkItem.Links.Links["html"];
var addWebLinkOperation = MigrationHelpers.GetHyperlinkAddOperation(link.Href);
jsonPatchOperations.Add(addWebLinkOperation);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, targetId);
result.Add((sourceId, witBatchRequest));
}
}
return(result);
}
private async Task MigratePhase1()
{
Logger.LogInformation("Starting migration phase 1");
foreach (IPhase1Processor migrator in ClientHelpers.GetProcessorInstances <IPhase1Processor>(context.Config))
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Starting migration phase for: {migrator.Name}");
await migrator.Process(context);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Completed migration phase for: {migrator.Name} in {stopwatch.Elapsed.TotalSeconds}s");
}
Logger.LogInformation("Completed migration phase 1");
}
private async Task MigratePhase3()
{
IEnumerable <IPhase3Processor> phase3Processors = ClientHelpers.GetProcessorInstances <IPhase3Processor>(context.Config);
if (phase3Processors != null && !phase3Processors.Any())
{
// nothing to do if no phase 3 processors are enabled
return;
}
// Phase1 or Phase2 have completed, and FailureReason == None
IEnumerable <WorkItemMigrationState> successfullyMigratedWorkItemMigrationStates = context.WorkItemsMigrationState.Where(w => (w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) || w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2)) && w.FailureReason == FailureReason.None);
var phase3WorkItemsToUpdateCount = successfullyMigratedWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase3WorkItemsToUpdateCount, Constants.BatchSize);
if (phase3WorkItemsToUpdateCount == 0)
{
return;
}
await successfullyMigratedWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
IList <(int SourceId, WitBatchRequest WitBatchRequest)> sourceIdToWitBatchRequests = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
foreach (WorkItem sourceWorkItem in sourceWorkItemsInBatch)
{
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
foreach (IPhase3Processor processor in phase3Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, null, sourceWorkItem, null);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, sourceWorkItem.Id.Value);
sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest));
}
}
var phase3ApiWrapper = new Phase3ApiWrapper();
await phase3ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
});
}
private bool IsPhase2UpdateRequired(WorkItemMigrationState workItemMigrationState, WorkItem targetWorkItem)
{
IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(ValidationContext.Config);
workItemMigrationState.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, workItemMigrationState.SourceId);
// find out if Enabled, see if matches comment from target
ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(ValidationContext.Config)).Select(b => b.Name));
enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations);
if (enabledPhaseStatuses.IsSubsetOf(workItemMigrationState.RevAndPhaseStatus.PhaseStatus)) // enabled relation phases are already complete for current work item
{
return(false);
}
return(true);
}
public async Task Process(IMigrationContext context)
{
var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context);
var totalNumberOfBatches = ClientHelpers.GetBatchCount(workItemsAndStateToMigrate.Count, Constants.BatchSize);
if (!workItemsAndStateToMigrate.Any())
{
Logger.LogInformation(LogDestination.File, $"No work items to process for {this.Name}");
return;
}
Logger.LogInformation(LogDestination.All, $"{this.Name} will process {workItemsAndStateToMigrate.Count} work items on the target");
var preprocessors = ClientHelpers.GetProcessorInstances <IPhase1PreProcessor>(context.Config);
foreach (var preprocessor in preprocessors)
{
await preprocessor.Prepare(context);
}
await workItemsAndStateToMigrate.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(batchWorkItemsAndState, batchId) =>
{
var batchStopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting");
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, batchWorkItemsAndState);
//read the work items
var stepStopwatch = Stopwatch.StartNew();
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Reading source work items");
await Migrator.ReadSourceWorkItems(context, batchWorkItemsAndState.Select(w => w.SourceId), batchContext);
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed reading source work items in {stepStopwatch.Elapsed.Seconds}s");
this.PrepareBatchContext(batchContext, batchWorkItemsAndState);
foreach (var preprocessor in preprocessors)
{
stepStopwatch.Restart();
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting {preprocessor.Name}");
await preprocessor.Process(batchContext);
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed {preprocessor.Name} in {stepStopwatch.Elapsed.Seconds}s");
}
var workItemsToUpdateCount = this.GetWorkItemsToProcessCount(batchContext);
Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Number of work items to migrate: {workItemsToUpdateCount}");
//migrate the batch of work items
if (workItemsToUpdateCount == 0)
{
batchStopwatch.Stop();
Logger.LogWarning(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: No work items to migrate");
}
else
{
stepStopwatch.Restart();
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Saving the target work items");
var witBatchRequestGenerator = this.GetWitBatchRequestGenerator(context, batchContext);
await witBatchRequestGenerator.Write();
Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed saving the target work items in {stepStopwatch.Elapsed.Seconds}s");
batchStopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed in {batchStopwatch.Elapsed.TotalSeconds}s");
}
});
}