private async Task VerifyRelationTypesExistsOnTarget(IValidationContext context)
{
var sourceRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.SourceClient.WorkItemTrackingHttpClient);
var targetRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.TargetClient.WorkItemTrackingHttpClient);
foreach (var relationType in sourceRelationTypes)
{
//retrieve relations which are of type workitemlink defined by attribute kvp {"usage", "workitemlink"}
//exclude remote link types because they need to be converted into hyperlinks
if (IsWorkItemLinkType(relationType))
{
if (RelationHelpers.IsRemoteLinkType(relationType))
{
context.RemoteLinkRelationTypes.Add(relationType.ReferenceName);
}
else
{
if (TargetHasRelationType(relationType, targetRelationTypes))
{
context.ValidatedWorkItemLinkRelationTypes.Add(relationType.ReferenceName);
}
else
{
Logger.LogWarning(LogDestination.File, $"Target: Relation type {relationType.ReferenceName} does not exist");
}
}
}
}
}
private async Task PopulateWorkItemMigrationState()
{
//dictionary of target workitem id to source id - these workitems have been migrated before
var existingWorkItems = ValidationContext.WorkItemsMigrationState.Where(wi => wi.MigrationState == WorkItemMigrationState.State.Existing);
var totalNumberOfBatches = ClientHelpers.GetBatchCount(existingWorkItems.Count(), Constants.BatchSize);
await existingWorkItems.Batch(Constants.BatchSize).ForEachAsync(ValidationContext.Config.Parallelism, async(batchWorkItemMigrationState, batchId) =>
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Started");
Dictionary <int, WorkItemMigrationState> targetToWorkItemMigrationState = batchWorkItemMigrationState.ToDictionary(k => k.TargetId.Value, v => v);
//read the target work items
IList <WorkItem> targetWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(ValidationContext.TargetClient.WorkItemTrackingHttpClient, batchWorkItemMigrationState.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IDictionary <int, WorkItemRelation> targetIdToHyperlinkToSourceRelationMapping = GetTargetIdToHyperlinkToSourceRelationMapping(targetWorkItems, targetToWorkItemMigrationState);
ProcessUpdatedSourceWorkItems(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
StoreWorkItemBatchRelationInformationOnContext(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Completed in {stopwatch.Elapsed.TotalSeconds}s");
});
}
private async Task VerifyRelationTypesExistsOnTarget(IValidationContext context)
{
var sourceRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.SourceClient.WorkItemTrackingHttpClient);
var targetRelationTypes = await WorkItemTrackingHelpers.GetRelationTypesAsync(context.TargetClient.WorkItemTrackingHttpClient);
var targetRelationNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (var relationType in sourceRelationTypes)
{
//retrieve relations which are of type workitemlink defined by attribute kvp {"usage", "workitemlink"}
if (relationType.Attributes.ContainsKeyIgnoringCase(Constants.WorkItemLinkAttributeKey) &&
String.Equals(relationType.Attributes[Constants.WorkItemLinkAttributeKey].ToString(), Constants.WorkItemLinkAttributeValue, StringComparison.OrdinalIgnoreCase))
{
if (TargetHasRelationType(relationType, targetRelationTypes))
{
context.ValidatedWorkItemLinkRelationTypes.Add(relationType.ReferenceName);
}
else
{
Logger.LogWarning(LogDestination.File, $"Target: Relation type {relationType.ReferenceName} does not exist");
}
}
}
}
/// <summary>
/// Populates migrationContext.TargetAreaAndIterationTree, migrationContext.TargetAreaPaths, and migrationContext.TargetIterationPaths
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="projectId"></param>
/// <returns></returns>
public static async Task ReadTargetNodes(IMigrationContext migrationContext, string projectId)
{
var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.TargetClient.WorkItemTrackingHttpClient, projectId);
migrationContext.TargetAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
migrationContext.TargetAreaPaths = migrationContext.TargetAreaAndIterationTree.AreaPathList;
migrationContext.TargetIterationPaths = migrationContext.TargetAreaAndIterationTree.IterationPathList;
}
public async virtual Task CreateIterationPath(string iterationPath, WorkItemClassificationNode sourceIterationNode)
{
await WorkItemTrackingHelpers.CreateIterationAsync(
context.TargetClient.WorkItemTrackingHttpClient,
context.Config.TargetConnection.Project,
iterationPath,
sourceIterationNode.Attributes == null?null : (DateTime?)sourceIterationNode.Attributes?["startDate"],
sourceIterationNode.Attributes == null?null : (DateTime?)sourceIterationNode.Attributes["finishDate"]);
}
private async Task <RevisionHistoryAttachments> GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem)
{
IList <RevisionHistoryAttachments> revisionHistoryAttachments = new List <RevisionHistoryAttachments>();
var wiUpdates = await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value);
return(new RevisionHistoryAttachments {
Workitem = sourceWorkItem, Updates = wiUpdates
});
}
private async Task MigratePhase2()
{
Logger.LogInformation("Starting migration phase 2");
IEnumerable <WorkItemMigrationState> successfulWorkItemMigrationStates;
// when skip existing config flag is on and this work item was existing, continue to next work item.
if (context.Config.SkipExisting)
{
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create);
}
else
{
// allow any Create, OR Existing with UpdatePhase2
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create || (a.MigrationState == WorkItemMigrationState.State.Existing && a.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase2)));
}
var phase2WorkItemsToUpdateCount = successfulWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase2WorkItemsToUpdateCount, Constants.BatchSize);
if (phase2WorkItemsToUpdateCount == 0)
{
Logger.LogInformation(LogDestination.File, "No work items to process for phase 2");
return;
}
await successfulWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
Logger.LogTrace(LogDestination.File, $"Reading Phase 2 source and target work items for batch {batchId} of {totalNumberOfBatches}");
// make web call to get source and target work items
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
IList <WorkItem> targetWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.TargetClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
batchContext.SourceWorkItemIdToTargetWorkItemIdMapping = workItemMigrationStateBatch.ToDictionary(key => key.SourceId, value => value.TargetId.Value);
foreach (var sourceWorkItem in sourceWorkItemsInBatch)
{
int targetId = Migrator.GetTargetId(sourceWorkItem.Id.Value, workItemMigrationStateBatch);
batchContext.TargetIdToSourceWorkItemMapping.Add(targetId, sourceWorkItem);
}
Logger.LogTrace(LogDestination.File, $"Generating Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var sourceIdToWitBatchRequests = await GenerateWitBatchRequestsForPhase2Batch(batchContext, batchId, workItemMigrationStateBatch, sourceWorkItemsInBatch, targetWorkItemsInBatch);
Logger.LogTrace(LogDestination.File, $"Saving Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var phase2ApiWrapper = new Phase2ApiWrapper();
await phase2ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
Logger.LogTrace(LogDestination.File, $"Completed Phase 2 for batch {batchId} of {totalNumberOfBatches}");
});
Logger.LogInformation("Completed migration phase 2");
}
public async static Task CheckConnection(WorkItemClientConnection client, string project, int requestedPermission)
{
Logger.LogInformation($"Checking security permissions for {client.Connection.AuthorizedIdentity.DisplayName} in {project}");
bool hasPermission = false;
SecurityHttpClient securityHttpClient = null;
WorkItemClassificationNode result = null;
try
{
securityHttpClient = client.Connection.GetClient <SecurityHttpClient>();
result = await WorkItemTrackingHelpers.GetClassificationNode(client.WorkItemTrackingHttpClient, project, TreeStructureGroup.Areas);
}
catch (Exception e) when(e.InnerException is VssUnauthorizedException)
{
throw new ValidationException(client.Connection.Uri.ToString(), (VssUnauthorizedException)e.InnerException);
}
catch (Exception e)
{
throw new ValidationException("An unexpected error occurred while reading the classification nodes to validate project permissions", e);
}
//construct the token by appending the id
string token = $"vstfs:///Classification/Node/{result.Identifier}";
//WORK_ITEM guid is hardcoded below
//securityNameSpaceId for WORK_ITEM is 83e28ad4-2d72-4ceb-97b0-c7726d5502c3
try
{
hasPermission = await securityHttpClient.HasPermissionAsync(
new Guid("83e28ad4-2d72-4ceb-97b0-c7726d5502c3"),
token,
requestedPermission,
false);
}
catch (Exception e)
{
throw new ValidationException($"An unexpected error occurred while trying to check permissions for project {project}", e);
}
if (hasPermission)
{
Logger.LogSuccess(LogDestination.All, $"Verified security permissions for {project} project");
}
else
{
throw new ValidationException($"You do not have the necessary security permissions for {project}, work item {(requestedPermission == WritePermission ? "write" : "read")} permissions are required.");
}
}
private async static Task CheckPermission(WorkItemClientConnection client, string project, Guid securityNamespace, int requestedPermission)
{
Logger.LogInformation($"Checking security permissions for {client.Connection.AuthorizedIdentity.DisplayName} in {project}");
bool hasPermission = false;
SecurityHttpClient securityHttpClient = null;
WorkItemClassificationNode result = null;
try
{
securityHttpClient = client.Connection.GetClient <SecurityHttpClient>();
result = await WorkItemTrackingHelpers.GetClassificationNode(client.WorkItemTrackingHttpClient, project, TreeStructureGroup.Areas);
}
catch (Exception e) when(e.InnerException is VssUnauthorizedException)
{
throw new ValidationException(client.Connection.Uri.ToString(), (VssUnauthorizedException)e.InnerException);
}
catch (Exception e)
{
throw new ValidationException("An unexpected error occurred while reading the classification nodes to validate project permissions", e);
}
//construct the token by appending the id
string token = $"vstfs:///Classification/Node/{result.Identifier}";
try
{
hasPermission = await securityHttpClient.HasPermissionAsync(
securityNamespace,
token,
requestedPermission,
false);
}
catch (Exception e)
{
throw new ValidationException($"An unexpected error occurred while trying to check permissions for project {project} in namespace {securityNamespace}", e);
}
if (hasPermission)
{
Logger.LogSuccess(LogDestination.All, $"Verified security permissions for {project} project");
}
else
{
throw new ValidationException($"You do not have the necessary security permissions for {project}, work item permission: {requestedPermission} is required.");
}
}
private async Task MigratePhase3()
{
IEnumerable <IPhase3Processor> phase3Processors = ClientHelpers.GetProcessorInstances <IPhase3Processor>(context.Config);
if (phase3Processors != null && !phase3Processors.Any())
{
// nothing to do if no phase 3 processors are enabled
return;
}
// Phase1 or Phase2 have completed, and FailureReason == None
IEnumerable <WorkItemMigrationState> successfullyMigratedWorkItemMigrationStates = context.WorkItemsMigrationState.Where(w => (w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) || w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2)) && w.FailureReason == FailureReason.None);
var phase3WorkItemsToUpdateCount = successfullyMigratedWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase3WorkItemsToUpdateCount, Constants.BatchSize);
if (phase3WorkItemsToUpdateCount == 0)
{
return;
}
await successfullyMigratedWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
IList <(int SourceId, WitBatchRequest WitBatchRequest)> sourceIdToWitBatchRequests = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
foreach (WorkItem sourceWorkItem in sourceWorkItemsInBatch)
{
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
foreach (IPhase3Processor processor in phase3Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, null, sourceWorkItem, null);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, sourceWorkItem.Id.Value);
sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest));
}
}
var phase3ApiWrapper = new Phase3ApiWrapper();
await phase3ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
});
}
private async Task VerifyQueryExistsAndIsValid(IValidationContext context)
{
Logger.LogInformation(LogDestination.File, "Checking if the migration query exists in the source project");
QueryHierarchyItem query;
try
{
query = await WorkItemTrackingHelpers.GetQueryAsync(context.SourceClient.WorkItemTrackingHttpClient, context.Config.SourceConnection.Project, context.Config.Query);
}
catch (Exception e)
{
throw new ValidationException("Unable to read the migration query", e);
}
if (query.QueryType != QueryType.Flat)
{
throw new ValidationException("Only flat queries are supported for migration");
}
}
private async Task <AttachmentReference> UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem)
{
RevisionHistoryAttachments revisionHistoryAttachmentsItem = await GetWorkItemUpdates(migrationContext, sourceWorkItem);
string attachment = JsonConvert.SerializeObject(revisionHistoryAttachmentsItem.Updates);
AttachmentReference aRef;
using (MemoryStream stream = new MemoryStream())
{
var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachment);
await stream.WriteAsync(stringBytes, 0, stringBytes.Length);
stream.Position = 0;
//upload the attachment to the target for each workitem
aRef = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream);
}
return(aRef);
}
private async Task RunQuery(IValidationContext context)
{
Logger.LogInformation(LogDestination.File, "Running the migration query in the source project");
try
{
var workItemUris = await WorkItemTrackingHelpers.GetWorkItemIdAndReferenceLinksAsync(
context.SourceClient.WorkItemTrackingHttpClient,
context.Config.SourceConnection.Project,
context.Config.Query,
context.Config.TargetPostMoveTag,
context.Config.QueryPageSize - 1 /* Have to subtract -1 from the page size due to a bug in how query interprets page size */);
context.WorkItemIdsUris = new ConcurrentDictionary <int, string>(workItemUris);
}
catch (Exception e)
{
throw new ValidationException("Unable to run the migration query", e);
}
}
public async Task Prepare(IValidationContext context)
{
context.RequestedFields.Add(FieldNames.AreaPath);
context.RequestedFields.Add(FieldNames.IterationPath);
Logger.LogInformation("Reading the area and iteration paths from the source and target accounts");
try
{
var classificationNodes = await WorkItemTrackingHelpers.GetClassificationNodes(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project);
var nodes = new AreaAndIterationPathTree(classificationNodes);
context.TargetAreaPaths = nodes.AreaPathList;
context.TargetIterationPaths = nodes.IterationPathList;
}
catch (Exception e)
{
throw new ValidationException("Unable to read the classification nodes on the source", e);
}
}
private async Task <IList <AttachmentLink> > UploadAttachmentsToTarget(IMigrationContext migrationContext, WorkItem sourceWorkItem)
{
var attachmentLinks = new List <AttachmentLink>();
int updateLimit = migrationContext.Config.MoveHistoryLimit;
int updateCount = 0;
while (updateCount < updateLimit)
{
var updates = await GetWorkItemUpdates(migrationContext, sourceWorkItem, skip : updateCount);
string attachmentContent = JsonConvert.SerializeObject(updates);
AttachmentReference attachmentReference;
using (MemoryStream stream = new MemoryStream())
{
var stringBytes = System.Text.Encoding.UTF8.GetBytes(attachmentContent);
await stream.WriteAsync(stringBytes, 0, stringBytes.Length);
stream.Position = 0;
//upload the attachment to the target for each batch of workitem updates
attachmentReference = await WorkItemTrackingHelpers.CreateAttachmentAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, stream);
attachmentLinks.Add(
new AttachmentLink(
$"{Constants.WorkItemHistory}-{sourceWorkItem.Id}-{updateCount}.json",
attachmentReference,
stringBytes.Length,
comment: $"Update range from {updateCount} to {updateCount + updates.Count}"));
}
updateCount += updates.Count;
// if we got less than a page size, that means we're on the last
// page and shouldn't try and read another page.
if (updates.Count < Constants.PageSize)
{
break;
}
}
return(attachmentLinks);
}
private async Task ValidateWorkItemMetadata()
{
Logger.LogInformation("Starting work item metadata validation");
var validators = ClientHelpers.GetInstances <IWorkItemValidator>();
foreach (var validator in validators)
{
await validator.Prepare(context);
}
var totalNumberOfBatches = ClientHelpers.GetBatchCount(context.WorkItemIdsUris.Count, Constants.BatchSize);
await context.WorkItemIdsUris.Keys.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemIds, batchId) =>
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: Starting");
var workItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(
context.SourceClient.WorkItemTrackingHttpClient,
workItemIds,
context.RequestedFields);
foreach (var validator in validators)
{
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: {validator.Name}");
foreach (var workItem in workItems)
{
await validator.Validate(context, workItem);
}
}
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Work item metadata validation batch {batchId} of {totalNumberOfBatches}: Completed in {stopwatch.Elapsed.TotalSeconds}s");
});
Logger.LogInformation("Completed work item metadata validation");
}
private async static Task CheckCssPermission(WorkItemClientConnection client, string project, int requestedPermission)
{
Logger.LogInformation($"Checking css security permissions for {client.Connection.AuthorizedIdentity.DisplayName} in {project}");
SecurityHttpClient securityHttpClient = null;
WorkItemClassificationNode result = null;
try
{
securityHttpClient = client.Connection.GetClient <SecurityHttpClient>();
result = await WorkItemTrackingHelpers.GetClassificationNode(client.WorkItemTrackingHttpClient, project, TreeStructureGroup.Areas);
}
catch (Exception e) when(e.InnerException is VssUnauthorizedException)
{
throw new ValidationException(client.Connection.Uri.ToString(), (VssUnauthorizedException)e.InnerException);
}
catch (Exception e)
{
throw new ValidationException("An unexpected error occurred while reading the classification nodes to validate project permissions", e);
}
await HasPermission(securityHttpClient, project, $"vstfs:///Classification/Node/{result.Identifier}", CssSecurityNamespace, requestedPermission);
}
public void ParseQueryForPaging_InjectsPostMoveTagWithWhereClause()
{
var query = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems WHERE System.Id = 1 order BY System.Id", "Migrated");
Assert.AreEqual("SELECT * FROM WorkItems WHERE (System.Id = 1) AND System.Tags NOT CONTAINS 'Migrated'", query);
}
public void GetPageableQuery_NoWhereClause()
{
var query = WorkItemTrackingHelpers.GetPageableQuery("SELECT * FROM WorkItems", 1, 1);
Assert.AreEqual("SELECT * FROM WorkItems WHERE ((System.Watermark > 1) OR (System.Watermark = 1 AND System.Id > 1)) ORDER BY System.Watermark, System.Id", query);
}
/// <summary>
/// Populates batchContext.WorkItems
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="workItemIds"></param>
/// <param name="batchContext"></param>
/// <param name="expand"></param>
/// <returns></returns>
public static async Task ReadSourceWorkItems(IMigrationContext migrationContext, IEnumerable <int> workItemIds, IBatchMigrationContext batchContext, WorkItemExpand?expand = WorkItemExpand.All)
{
batchContext.SourceWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, workItemIds, expand : expand);
}
/// <summary>
/// Populates migrationContext.SourceAreaAndIterationTree
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="projectId"></param>
/// <returns></returns>
public static async Task ReadSourceNodes(IMigrationContext migrationContext, string projectId)
{
var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.SourceClient.WorkItemTrackingHttpClient, projectId);
migrationContext.SourceAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
}
private async Task <IList <WorkItemUpdate> > GetWorkItemUpdates(IMigrationContext migrationContext, WorkItem sourceWorkItem, int skip = 0)
{
return(await WorkItemTrackingHelpers.GetWorkItemUpdatesAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, sourceWorkItem.Id.Value, skip));
}
public void ParseQueryForPaging_StripsOrderByClause()
{
var query = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems order BY System.Id", null);
Assert.AreEqual("SELECT * FROM WorkItems", query);
}
public async Task Prepare(IValidationContext context)
{
Logger.LogInformation("Reading the work item types from the source and target accounts");
// When reading the work item, we need the type field for this validator
context.RequestedFields.Add(FieldNames.WorkItemType);
try
{
// We need all fields to validate the field types
var sourceFields = (await WorkItemTrackingHelpers.GetFields(context.SourceClient.WorkItemTrackingHttpClient)).ToDictionary(key => key.ReferenceName);
context.SourceFields = new ConcurrentDictionary <string, WorkItemField>(sourceFields, StringComparer.OrdinalIgnoreCase);
}
catch (Exception e)
{
throw new ValidationException("Unable to read fields on the source", e);
}
try
{
// We need all fields to validate the field types
var targetFields = (await WorkItemTrackingHelpers.GetFields(context.TargetClient.WorkItemTrackingHttpClient)).ToDictionary(key => key.ReferenceName);
context.TargetFields = new ConcurrentDictionary <string, WorkItemField>(targetFields, StringComparer.OrdinalIgnoreCase);
context.IdentityFields = new HashSet <string>(targetFields.Where(f => f.Value.IsIdentity).Select(f => f.Key), StringComparer.OrdinalIgnoreCase);
}
catch (Exception e)
{
throw new ValidationException("Unable to read fields on the target", e);
}
// handle condition of activated/not activated:
ValidateFieldsMapping(context);
try
{
var workItemTypes = await WorkItemTrackingHelpers.GetWorkItemTypes(context.SourceClient.WorkItemTrackingHttpClient, context.Config.SourceConnection.Project);
foreach (var workItemType in workItemTypes)
{
context.SourceTypesAndFields[workItemType.Name] = new HashSet <string>(workItemType.Fields.Select(f => f.ReferenceName), StringComparer.OrdinalIgnoreCase);
}
}
catch (Exception e)
{
throw new ValidationException("Unable to read work item types on the source", e);
}
try
{
var workItemTypes = await WorkItemTrackingHelpers.GetWorkItemTypes(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project);
foreach (var workItemType in workItemTypes)
{
context.TargetTypesAndFields[workItemType.Name] = new HashSet <string>(workItemType.Fields.Select(f => f.ReferenceName), StringComparer.OrdinalIgnoreCase);
}
}
catch (Exception e)
{
throw new ValidationException("Unable to read work item types on the target", e);
}
// A very edge case, but we should fail this scenario.
if (context.SourceTypesAndFields.Count == 0 || context.TargetTypesAndFields.Count == 0)
{
throw new ValidationException("Source or target does not have any work item types");
}
}
public void ParseQueryForPaging_NoOrderByClause()
{
var parsedQuery = WorkItemTrackingHelpers.ParseQueryForPaging("SELECT * FROM WorkItems", null);
Assert.AreEqual("SELECT * FROM WorkItems", parsedQuery);
}
public async virtual Task CreateAreaPath(string areaPathInTarget)
{
await WorkItemTrackingHelpers.CreateAreaPathAsync(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project, areaPathInTarget);
}
/// <summary>
/// Uploads inline image attachment from source to target and returns target inline image AttachmentReference Guid.
/// </summary>
/// <param name="inlineImageUrl"></param>
/// <returns></returns>
private async Task <string> UploadInlineImageAttachmentFromSourceWorkItemToTarget(IBatchMigrationContext batchContext, string inlineImageUrl, int sourceWorkItemId, int maxAttachmentSize)
{
Guid sourceGuid = MigrationHelpers.GetAttachmentUrlGuid(inlineImageUrl);
string targetGuid = null;
if (Guid.Empty.Equals(sourceGuid))
{
Logger.LogWarning(LogDestination.File, $"Unexpected format for inline image url {inlineImageUrl} for source work item {sourceWorkItemId}");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItemId, FailureReason.InlineImageUrlFormatError, batchContext.WorkItemMigrationState);
// just return the null guid since there is nothing we can do with an invalid url
return(null);
}
Stream stream = null;
try
{
Logger.LogTrace(LogDestination.File, $"Reading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
stream = await WorkItemTrackingHelpers.GetAttachmentAsync(this.context.SourceClient.WorkItemTrackingHttpClient, sourceGuid);
Logger.LogTrace(LogDestination.File, $"Completed reading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to download inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItemId, FailureReason.InlineImageDownloadError, batchContext.WorkItemMigrationState);
// just return the null guid since there is nothing we can do if we couldn't download the inline image
return(null);
}
if (stream != null)
{
using (MemoryStream memstream = new MemoryStream())
{
bool copiedStream = false;
using (stream)
{
try
{
Logger.LogTrace(LogDestination.File, $"Downloading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
await ClientHelpers.CopyStreamAsync(stream, memstream);
Logger.LogTrace(LogDestination.File, $"Completed downloading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
copiedStream = true;
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to download inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItemId, FailureReason.InlineImageDownloadError, batchContext.WorkItemMigrationState);
}
}
if (memstream.Length > maxAttachmentSize)
{
Logger.LogWarning(LogDestination.File, $"Inline image attachment of source work item with id {sourceWorkItemId} and url {inlineImageUrl} exceeded the maximum attachment size of {maxAttachmentSize} bytes." +
$" Skipping creating the inline image attachment in target account.");
return(null);
}
if (copiedStream)
{
memstream.Position = 0;
//upload the attachment to target
try
{
Logger.LogTrace(LogDestination.File, $"Uploading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
var aRef = await WorkItemTrackingHelpers.CreateAttachmentChunkedAsync(this.context.TargetClient.WorkItemTrackingHttpClient, this.context.TargetClient.Connection, memstream, this.context.Config.AttachmentUploadChunkSize);
targetGuid = aRef.Id.ToString();
Logger.LogTrace(LogDestination.File, $"Completed uploading inline image {inlineImageUrl} for source work item {sourceWorkItemId} from the source account");
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to upload inline image for source work item {sourceWorkItemId} in the target account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItemId, FailureReason.InlineImageUploadError, batchContext.WorkItemMigrationState);
}
}
}
}
return(targetGuid);
}
private async Task <AttachmentLink> UploadAttachmentFromSourceRelation(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItemRelation sourceRelation, int maxAttachmentSize)
{
//Attachments are of type Rel = "AttachedFile"
if (sourceRelation.Rel == Constants.AttachedFile)
{
string filename = null;
string comment = null;
long resourceSize = 0;
//get the file name and comment
if (sourceRelation.Attributes.ContainsKey(Constants.RelationAttributeName))
{
filename = sourceRelation.Attributes[Constants.RelationAttributeName].ToString();
}
if (sourceRelation.Attributes.ContainsKey(Constants.RelationAttributeComment))
{
comment = sourceRelation.Attributes[Constants.RelationAttributeComment].ToString();
}
//get the guid from the url
Guid attachmentId;
if (Guid.TryParse(sourceRelation.Url.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).Last(), out attachmentId))
{
Stream stream = null;
try
{
Logger.LogTrace(LogDestination.File, $"Reading attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
stream = await WorkItemTrackingHelpers.GetAttachmentAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, attachmentId);
Logger.LogTrace(LogDestination.File, $"Completed reading attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to download attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItem.Id.Value, FailureReason.AttachmentDownloadError, batchContext.WorkItemMigrationState);
return(null);
}
AttachmentReference aRef = null;
using (MemoryStream memstream = new MemoryStream())
{
using (stream)
{
try
{
Logger.LogTrace(LogDestination.File, $"Downloading attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
await ClientHelpers.CopyStreamAsync(stream, memstream);
Logger.LogTrace(LogDestination.File, $"Completed downloading attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to read downloaded attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItem.Id.Value, FailureReason.AttachmentDownloadError, batchContext.WorkItemMigrationState);
return(null);
}
}
resourceSize = memstream.Length;
if (resourceSize > maxAttachmentSize)
{
Logger.LogWarning(LogDestination.File, $"Attachment of source work item with id {sourceWorkItem.Id} and url {sourceRelation.Url} exceeded the maximum attachment size of {maxAttachmentSize} bytes." +
$" Skipping creating the attachment in target account.");
return(null);
}
memstream.Position = 0;
//upload the attachment to the target
try
{
Logger.LogTrace(LogDestination.File, $"Uploading attachment {filename} of {resourceSize} bytes for source work item {sourceWorkItem.Id} from the source account");
aRef = await WorkItemTrackingHelpers.CreateAttachmentChunkedAsync(migrationContext.TargetClient.WorkItemTrackingHttpClient, migrationContext.TargetClient.Connection, memstream, migrationContext.Config.AttachmentUploadChunkSize);
Logger.LogTrace(LogDestination.File, $"Completed uploading attachment {filename} for source work item {sourceWorkItem.Id} from the source account");
}
catch (Exception e)
{
Logger.LogError(LogDestination.File, e, $"Unable to upload attachment {filename} for source work item {sourceWorkItem.Id} to the target account");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItem.Id.Value, FailureReason.AttachmentUploadError, batchContext.WorkItemMigrationState);
}
}
if (aRef != null)
{
return(new AttachmentLink(filename, aRef, resourceSize, comment));
}
}
else
{
Logger.LogError(LogDestination.File, $"Attachment link is incorrect for {sourceWorkItem.Id} {sourceRelation.Url}. Skipping creating the attachment in target account.");
ClientHelpers.AddFailureReasonToWorkItemMigrationState(sourceWorkItem.Id.Value, FailureReason.AttachmentUploadError, batchContext.WorkItemMigrationState);
}
}
return(null);
}
