private async Task <IList <(int SourceId, WitBatchRequest WitBatchRequest)> > GenerateWitBatchRequestsForPhase2Batch(IBatchMigrationContext batchContext, int batchId, IList <WorkItemMigrationState> workItemMigrationState, IList <WorkItem> sourceWorkItems, IList <WorkItem> targetWorkItems) { IList <(int SourceId, WitBatchRequest WitBatchRequest)> result = new List <(int SourceId, WitBatchRequest WitBatchRequest)>(); IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(context.Config); foreach (IPhase2Processor processor in phase2Processors) { Logger.LogInformation(LogDestination.File, $"Starting preprocessing of phase 2 step {processor.Name} for batch {batchId}"); await processor.Preprocess(context, batchContext, sourceWorkItems, targetWorkItems); Logger.LogInformation(LogDestination.File, $"Completed preprocessing of phase 2 step {processor.Name} for batch {batchId}"); } foreach (var sourceToTarget in batchContext.SourceWorkItemIdToTargetWorkItemIdMapping) { int sourceId = sourceToTarget.Key; int targetId = sourceToTarget.Value; WorkItem sourceWorkItem = sourceWorkItems.First(a => a.Id == sourceId); WorkItem targetWorkItem = targetWorkItems.First(a => a.Id == targetId); IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>(); WorkItemMigrationState state = workItemMigrationState.First(a => a.SourceId == sourceId); state.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, sourceId); ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(context.Config)).Select(b => b.Name)); enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations); foreach (IPhase2Processor processor in phase2Processors) { IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, batchContext, sourceWorkItem, targetWorkItem); jsonPatchOperations.AddRange(processorJsonPatchOperations); } jsonPatchOperations.Add(GetAddHyperlinkWithCommentOperation(targetWorkItems, state, sourceId, targetId, sourceWorkItem, enabledPhaseStatuses)); if (this.context.Config.IncludeWebLink) { var link = (ReferenceLink)sourceWorkItem.Links.Links["html"]; var addWebLinkOperation = MigrationHelpers.GetHyperlinkAddOperation(link.Href); jsonPatchOperations.Add(addWebLinkOperation); } if (jsonPatchOperations.Any()) { WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, targetId); result.Add((sourceId, witBatchRequest)); } } return(result); }
private async Task MigratePhase1() { Logger.LogInformation("Starting migration phase 1"); foreach (IPhase1Processor migrator in ClientHelpers.GetProcessorInstances <IPhase1Processor>(context.Config)) { var stopwatch = Stopwatch.StartNew(); Logger.LogInformation(LogDestination.File, $"Starting migration phase for: {migrator.Name}"); await migrator.Process(context); stopwatch.Stop(); Logger.LogInformation(LogDestination.File, $"Completed migration phase for: {migrator.Name} in {stopwatch.Elapsed.TotalSeconds}s"); } Logger.LogInformation("Completed migration phase 1"); }
private async Task MigratePhase3() { IEnumerable <IPhase3Processor> phase3Processors = ClientHelpers.GetProcessorInstances <IPhase3Processor>(context.Config); if (phase3Processors != null && !phase3Processors.Any()) { // nothing to do if no phase 3 processors are enabled return; } // Phase1 or Phase2 have completed, and FailureReason == None IEnumerable <WorkItemMigrationState> successfullyMigratedWorkItemMigrationStates = context.WorkItemsMigrationState.Where(w => (w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) || w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2)) && w.FailureReason == FailureReason.None); var phase3WorkItemsToUpdateCount = successfullyMigratedWorkItemMigrationStates.Count(); var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase3WorkItemsToUpdateCount, Constants.BatchSize); if (phase3WorkItemsToUpdateCount == 0) { return; } await successfullyMigratedWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) => { IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch); IList <(int SourceId, WitBatchRequest WitBatchRequest)> sourceIdToWitBatchRequests = new List <(int SourceId, WitBatchRequest WitBatchRequest)>(); IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All); foreach (WorkItem sourceWorkItem in sourceWorkItemsInBatch) { IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>(); foreach (IPhase3Processor processor in phase3Processors) { IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, null, sourceWorkItem, null); jsonPatchOperations.AddRange(processorJsonPatchOperations); } if (jsonPatchOperations.Any()) { WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, sourceWorkItem.Id.Value); sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest)); } } var phase3ApiWrapper = new Phase3ApiWrapper(); await phase3ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext); }); }
private bool IsPhase2UpdateRequired(WorkItemMigrationState workItemMigrationState, WorkItem targetWorkItem) { IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(ValidationContext.Config); workItemMigrationState.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, workItemMigrationState.SourceId); // find out if Enabled, see if matches comment from target ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(ValidationContext.Config)).Select(b => b.Name)); enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations); if (enabledPhaseStatuses.IsSubsetOf(workItemMigrationState.RevAndPhaseStatus.PhaseStatus)) // enabled relation phases are already complete for current work item { return(false); } return(true); }
public async Task Process(IMigrationContext context) { var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context); var totalNumberOfBatches = ClientHelpers.GetBatchCount(workItemsAndStateToMigrate.Count, Constants.BatchSize); if (!workItemsAndStateToMigrate.Any()) { Logger.LogInformation(LogDestination.File, $"No work items to process for {this.Name}"); return; } Logger.LogInformation(LogDestination.All, $"{this.Name} will process {workItemsAndStateToMigrate.Count} work items on the target"); var preprocessors = ClientHelpers.GetProcessorInstances <IPhase1PreProcessor>(context.Config); foreach (var preprocessor in preprocessors) { await preprocessor.Prepare(context); } await workItemsAndStateToMigrate.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(batchWorkItemsAndState, batchId) => { var batchStopwatch = Stopwatch.StartNew(); Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting"); IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, batchWorkItemsAndState); //read the work items var stepStopwatch = Stopwatch.StartNew(); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Reading source work items"); await Migrator.ReadSourceWorkItems(context, batchWorkItemsAndState.Select(w => w.SourceId), batchContext); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed reading source work items in {stepStopwatch.Elapsed.Seconds}s"); this.PrepareBatchContext(batchContext, batchWorkItemsAndState); foreach (var preprocessor in preprocessors) { stepStopwatch.Restart(); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Starting {preprocessor.Name}"); await preprocessor.Process(batchContext); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed {preprocessor.Name} in {stepStopwatch.Elapsed.Seconds}s"); } var workItemsToUpdateCount = this.GetWorkItemsToProcessCount(batchContext); Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Number of work items to migrate: {workItemsToUpdateCount}"); //migrate the batch of work items if (workItemsToUpdateCount == 0) { batchStopwatch.Stop(); Logger.LogWarning(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: No work items to migrate"); } else { stepStopwatch.Restart(); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Saving the target work items"); var witBatchRequestGenerator = this.GetWitBatchRequestGenerator(context, batchContext); await witBatchRequestGenerator.Write(); Logger.LogTrace(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed saving the target work items in {stepStopwatch.Elapsed.Seconds}s"); batchStopwatch.Stop(); Logger.LogInformation(LogDestination.File, $"{this.Name} batch {batchId} of {totalNumberOfBatches}: Completed in {batchStopwatch.Elapsed.TotalSeconds}s"); } }); }