public override int Execute(string[] commandLineArguments)
{
Options.Parse(commandLineArguments);
var environment = AwsEnvironmentGeneration.Create(log, variables).GetAwaiter().GetResult();
var stackEventLogger = new StackEventLogger(log);
IAmazonCloudFormation ClientFactory() => ClientHelpers.CreateCloudFormationClient(environment);
StackArn StackProvider(RunningDeployment x) => new StackArn(x.Variables.Get(AwsSpecialVariables.CloudFormation.StackName));
ChangeSetArn ChangesetProvider(RunningDeployment x) => new ChangeSetArn(x.Variables[AwsSpecialVariables.CloudFormation.Changesets.Arn]);
var conventions = new List <IConvention>
{
new LogAwsUserInfoConvention(environment),
new ExecuteCloudFormationChangeSetConvention(ClientFactory, stackEventLogger, StackProvider, ChangesetProvider, waitForComplete),
new CloudFormationOutputsAsVariablesConvention(ClientFactory, stackEventLogger, StackProvider)
};
var deployment = new RunningDeployment(packageFile, variables);
var conventionRunner = new ConventionProcessor(deployment, conventions, log);
conventionRunner.RunConventions();
return(0);
}
public async Task <IEnumerable <JsonPatchOperation> > Process(IMigrationContext migrationContext, IBatchMigrationContext batchContext, WorkItem sourceWorkItem, WorkItem targetWorkItem)
{
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
if (sourceWorkItem.Relations == null)
{
return(jsonPatchOperations);
}
IList <WorkItemRelation> sourceWorkItemLinkRelations = GetWorkItemLinkRelations(migrationContext, sourceWorkItem.Relations);
if (sourceWorkItemLinkRelations.Any())
{
foreach (WorkItemRelation sourceWorkItemLinkRelation in sourceWorkItemLinkRelations)
{
int linkedSourceId = ClientHelpers.GetWorkItemIdFromApiEndpoint(sourceWorkItemLinkRelation.Url);
int targetWorkItemId = targetWorkItem.Id.Value;
int linkedTargetId;
if (!migrationContext.SourceToTargetIds.TryGetValue(linkedSourceId, out linkedTargetId))
{
continue;
}
string comment = MigrationHelpers.GetCommentFromAttributes(sourceWorkItemLinkRelation);
WorkItemLink newWorkItemLink = new WorkItemLink(linkedTargetId, sourceWorkItemLinkRelation.Rel, false, false, comment, 0);
JsonPatchOperation workItemLinkAddOperation = MigrationHelpers.GetWorkItemLinkAddOperation(migrationContext, newWorkItemLink);
jsonPatchOperations.Add(workItemLinkAddOperation);
}
}
return(jsonPatchOperations);
}
public void Notify(string key, object queryString = null, object body = null)
{
try
{
if (!ConfigSection.Webhook.Hooks.Enable)
{
return;
}
var qs = ClientHelpers.GetQueryString(queryString);
var json = ClientHelpers.GetJsonBody(body);
var data = ConfigSection.Webhook.Data[key];
if (data.Method == "GET")
{
var url = data.Url + (data.Url.Contains("?") ? "&" : "?") + qs;
_client.httpGetRequest(url);
}
else if (data.Method == "POST")
{
_client.httpPostRequest(data.Url, json);
}
else
{
throw new NotImplementedException(String.Format("Http method {0} is not implemented yet", data.Method));
}
}
catch (Exception ex)
{
if (OnError != null)
{
OnError(ex);
}
}
}
private async Task ClassifyWorkItemIds(IValidationContext context)
{
// Remove all skipped work items to reduce the load of what we need to query
var workItemIdsUrisToClassify = context.WorkItemIdsUris.Where(wi => !context.SkippedWorkItems.Contains(wi.Key)).ToList();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(workItemIdsUrisToClassify.Count(), Constants.BatchSize);
if (workItemIdsUrisToClassify.Any())
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, "Started querying target account to find previously migrated work items");
await workItemIdsUrisToClassify.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemIdsUris, batchId) =>
{
var batchStopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Started");
//check if the workitems have already been migrated and add the classified work items to the context
var migrationStates = await FilterWorkItemIds(context, context.TargetClient.WorkItemTrackingHttpClient, workItemIdsUris.ToDictionary(k => k.Key, v => v.Value));
if (migrationStates.Any())
{
foreach (var migrationState in migrationStates)
{
context.WorkItemsMigrationState.Add(migrationState);
}
}
batchStopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Completed in {batchStopwatch.Elapsed.TotalSeconds}s");
});
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Completed querying target account to find previously migrated work items in {stopwatch.Elapsed.TotalSeconds}s");
}
}
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
HttpContext httpContext = filterContext.HttpContext;
Controller control = filterContext.Controller as Controller;
string BRE_NO = CheckBRE_NO != null ? CheckBRE_NO : control.RouteData.Values["Controller"].ToString();
string ACTION_ID = CheckACTION_ID != null ? CheckACTION_ID : control.RouteData.Values["action"].ToString();
if (!CheckDbPermissson(httpContext.User.Identity.GetClaimValue(ClaimTypes.NameIdentifier)))
{
if (ClientHelpers.IsAjaxRequest(httpContext.Request) ||
ClientHelpers.IsApiRequest(httpContext.Request))
{
var data = "{ \"Success\" : \"false\" , \"Error\" : \"您無權異動資料\" }";
var JsonResult = new JsonResult(data);
filterContext.Result = JsonResult;
}
else
{
filterContext.Result = new ForbidResult();
}
}
base.OnActionExecuting(filterContext);
}
private async Task PopulateWorkItemMigrationState()
{
//dictionary of target workitem id to source id - these workitems have been migrated before
var existingWorkItems = ValidationContext.WorkItemsMigrationState.Where(wi => wi.MigrationState == WorkItemMigrationState.State.Existing);
var totalNumberOfBatches = ClientHelpers.GetBatchCount(existingWorkItems.Count(), Constants.BatchSize);
await existingWorkItems.Batch(Constants.BatchSize).ForEachAsync(ValidationContext.Config.Parallelism, async(batchWorkItemMigrationState, batchId) =>
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Started");
Dictionary <int, WorkItemMigrationState> targetToWorkItemMigrationState = batchWorkItemMigrationState.ToDictionary(k => k.TargetId.Value, v => v);
//read the target work items
IList <WorkItem> targetWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(ValidationContext.TargetClient.WorkItemTrackingHttpClient, batchWorkItemMigrationState.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IDictionary <int, WorkItemRelation> targetIdToHyperlinkToSourceRelationMapping = GetTargetIdToHyperlinkToSourceRelationMapping(targetWorkItems, targetToWorkItemMigrationState);
ProcessUpdatedSourceWorkItems(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
StoreWorkItemBatchRelationInformationOnContext(targetWorkItems, targetToWorkItemMigrationState, targetIdToHyperlinkToSourceRelationMapping);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"{Name} batch {batchId} of {totalNumberOfBatches}: Completed in {stopwatch.Elapsed.TotalSeconds}s");
});
}
public AuthenticationHeaderValue GetLiveSessionHeaders(string token, string url, string verificationCode)
{
DiffieBytes ??= File.ReadAllBytes(@"dhparam.pem");
Prime ??= new Org.BouncyCastle.Math.BigInteger(1, DiffieBytes);
var g = new Org.BouncyCastle.Math.BigInteger("2", 10);
var dh = new DHParameters(Prime, g);
//var dhPrivateKeyParameters = keyPair.Private as DHPrivateKeyParameters;
var challenge = dh.G.ModPow(DhRandom, dh.P);
var diffie_hellman_challenge = BitConverter.ToString(challenge.ToByteArray()).Replace("-", "").ToLower();
var nonce = ClientHelpers.GetNonce();
var timestamp = ClientHelpers.GetTimestamp();
var data = new Dictionary <string, string>
{
{ "diffie_hellman_challenge", diffie_hellman_challenge },
{ "oauth_consumer_key", s_clientId },
{ "oauth_timestamp", timestamp.ToString() },
{ "oauth_nonce", nonce },
{ "oauth_signature_method", "RSA-SHA256" },
{ "oauth_token", token },
};
data.Add("oauth_signature", ClientHelpers.GetSignatureBaseString(_httpClient.BaseAddress.AbsoluteUri + url, data, verificationCode));
return(ClientHelpers.GetDefaultAuthHeader(data, false));
}
public override int Execute(string[] commandLineArguments)
{
Options.Parse(commandLineArguments);
if (variablesFile != null && !File.Exists(variablesFile))
{
throw new CommandException("Could not find variables file: " + variablesFile);
}
var variables = new CalamariVariableDictionary(variablesFile, sensitiveVariablesFile, sensitiveVariablesPassword);
var environment = new AwsEnvironmentGeneration(variables);
var stackEventLogger = new StackEventLogger(new LogWrapper());
IAmazonCloudFormation ClientFactory() => ClientHelpers.CreateCloudFormationClient(environment);
StackArn StackProvider(RunningDeployment x) => new StackArn(x.Variables.Get(AwsSpecialVariables.CloudFormation.StackName));
var conventions = new List <IConvention>
{
new LogAwsUserInfoConvention(environment),
new ContributeEnvironmentVariablesConvention(),
new LogVariablesConvention(),
new DeleteCloudFormationStackConvention(environment, stackEventLogger, ClientFactory, StackProvider, waitForComplete)
};
var deployment = new RunningDeployment(packageFile, variables);
var conventionRunner = new ConventionProcessor(deployment, conventions);
conventionRunner.RunConventions();
return(0);
}
private WitBatchRequest GenerateWitBatchRequestFromWorkItem(WorkItem sourceWorkItem)
{
Dictionary <string, string> headers = new Dictionary <string, string>();
headers.Add("Content-Type", "application/json-patch+json");
JsonPatchDocument jsonPatchDocument = CreateJsonPatchDocumentFromWorkItemFields(sourceWorkItem);
JsonPatchOperation insertIdAddOperation = GetInsertBatchIdAddOperation();
jsonPatchDocument.Add(insertIdAddOperation);
// add hyperlink to source WorkItem
string sourceWorkItemApiEndpoint = ClientHelpers.GetWorkItemApiEndpoint(this.migrationContext.Config.SourceConnection.Account, sourceWorkItem.Id.Value);
JsonPatchOperation addHyperlinkAddOperation = MigrationHelpers.GetHyperlinkAddOperation(sourceWorkItemApiEndpoint, sourceWorkItem.Rev.ToString());
jsonPatchDocument.Add(addHyperlinkAddOperation);
string json = JsonConvert.SerializeObject(jsonPatchDocument);
string workItemType = jsonPatchDocument.Find(a => a.Path.Contains(FieldNames.WorkItemType)).Value as string;
var witBatchRequest = new WitBatchRequest();
witBatchRequest.Method = "PATCH";
witBatchRequest.Headers = headers;
witBatchRequest.Uri = $"/{this.migrationContext.Config.TargetConnection.Project}/_apis/wit/workItems/${workItemType}?{this.QueryString}";
witBatchRequest.Body = json;
return(witBatchRequest);
}
public async Task Preprocess(IMigrationContext migrationContext, IBatchMigrationContext batchContext, IList <WorkItem> sourceWorkItems, IList <WorkItem> targetWorkItems)
{
var linkedWorkItemArtifactUrls = new HashSet <string>();
foreach (WorkItem sourceWorkItem in sourceWorkItems)
{
var relations = GetWorkItemLinkRelations(migrationContext, sourceWorkItem.Relations);
var linkedIds = relations.Select(r => ClientHelpers.GetWorkItemIdFromApiEndpoint(r.Url));
var uris = linkedIds.Where(id => !migrationContext.SourceToTargetIds.ContainsKey(id)).Select(id => ClientHelpers.GetWorkItemApiEndpoint(migrationContext.Config.SourceConnection.Account, id));
linkedWorkItemArtifactUrls.AddRange(uris);
}
await linkedWorkItemArtifactUrls.Batch(Constants.BatchSize).ForEachAsync(migrationContext.Config.Parallelism, async(workItemArtifactUris, batchId) =>
{
Logger.LogTrace(LogDestination.File, $"Finding linked work items on target for batch {batchId}");
var results = await ClientHelpers.QueryArtifactUriToGetIdsFromUris(migrationContext.TargetClient.WorkItemTrackingHttpClient, workItemArtifactUris);
foreach (var result in results.ArtifactUrisQueryResult)
{
if (result.Value != null)
{
if (result.Value.Count() == 1)
{
var sourceId = ClientHelpers.GetWorkItemIdFromApiEndpoint(result.Key);
var targetId = result.Value.First().Id;
migrationContext.SourceToTargetIds[sourceId] = targetId;
}
}
}
Logger.LogTrace(LogDestination.File, $"Finished finding linked work items on target for batch {batchId}");
});
}
private Task InstallAsync(RunningDeployment deployment)
{
Guard.NotNull(deployment, "Deployment should not be null");
Guard.NotNull(bucketFactory, "Bucket factory should not be null");
AmazonS3Client ClientFactory() => ClientHelpers.CreateS3Client(awsEnvironmentGeneration);
return(EnsureBucketExists(ClientFactory, bucketFactory(deployment)));
}
private void LogFinalStatus()
{
var createdWorkItems = this.context.WorkItemsMigrationState.Where(w => w.MigrationState == WorkItemMigrationState.State.Create);
if (createdWorkItems.Any())
{
Logger.LogSuccess(LogDestination.All, $"Created {createdWorkItems.Count()} work item(s)");
Logger.LogInformation(LogDestination.File, "Created WorkItems");
Logger.LogInformation(LogDestination.File, "Source Id :: Target Id");
foreach (var item in createdWorkItems)
{
Logger.LogInformation(LogDestination.File, $"{item.SourceId} :: {item.TargetId}");
}
}
var updatedWorkItems = this.context.WorkItemsMigrationState.Where(w => w.MigrationState == WorkItemMigrationState.State.Existing && w.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase1));
if (updatedWorkItems.Any())
{
Logger.LogSuccess(LogDestination.All, $"Updated {updatedWorkItems.Count()} work item(s)");
Logger.LogInformation(LogDestination.File, "Updated WorkItems");
Logger.LogInformation(LogDestination.File, "Source Id :: Target Id");
foreach (var item in updatedWorkItems)
{
Logger.LogInformation(LogDestination.File, $"{item.SourceId} :: {item.TargetId}");
}
}
Dictionary <int, FailureReason> notMigratedWorkItems = ClientHelpers.GetNotMigratedWorkItemsFromWorkItemsMigrationState(context.WorkItemsMigrationState);
if (notMigratedWorkItems.Any())
{
//Log breakdown of not migrated work items by FailureReason
Logger.LogError(LogDestination.All, $"{notMigratedWorkItems.Count} total work item(s) failed.");
FailureReason[] failureReasons = (FailureReason[])Enum.GetValues(typeof(FailureReason));
FailureReason[] failureReasonsWithoutNone = failureReasons.SubArray(1, failureReasons.Length - 1);
foreach (FailureReason failureReason in failureReasonsWithoutNone)
{
int failureCount = notMigratedWorkItems.Where(a => a.Value.HasFlag(failureReason)).Count();
if (failureCount > 0)
{
Logger.LogError(LogDestination.All, $" {failureCount} work item(s) failed for this reason: {failureReason}.");
}
}
//Log all the not migrated work items to both console and file
foreach (var item in notMigratedWorkItems)
{
Logger.LogInformation(LogDestination.File, $"{item.Key} :: {item.Value}");
}
}
Logger.LogInformation(LogDestination.All, "Migration complete");
}
public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsIdAtEnd()
{
string endpointUri = "https://aaaaaa.visualstudio.com/_apis/wit/workItems/3543";
int expected = 3543;
int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri);
Assert.AreEqual(expected, actual);
}
public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsIdFollowedBySlashAtEnd()
{
string endpointUri = "https://dev.azure.com/account/_apis/wit/workItems/3543/";
int expected = 3543;
int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri);
Assert.AreEqual(expected, actual);
}
public void GetWorkItemIdFromApiEndpoint_ReturnsCorrectResultWhenEndpointContainsQueryString()
{
string endpointUri = "https://dev.azure.com/account/_apis/wit/workItems/3543?bypassRules=True&suppressNotifications=True&api-version=4.0";
int expected = 3543;
int actual = ClientHelpers.GetWorkItemIdFromApiEndpoint(endpointUri);
Assert.AreEqual(expected, actual);
}
private async Task MigratePhase2()
{
Logger.LogInformation("Starting migration phase 2");
IEnumerable <WorkItemMigrationState> successfulWorkItemMigrationStates;
// when skip existing config flag is on and this work item was existing, continue to next work item.
if (context.Config.SkipExisting)
{
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create);
}
else
{
// allow any Create, OR Existing with UpdatePhase2
successfulWorkItemMigrationStates = context.WorkItemsMigrationState.Where(a => a.MigrationState == WorkItemMigrationState.State.Create || (a.MigrationState == WorkItemMigrationState.State.Existing && a.Requirement.HasFlag(WorkItemMigrationState.RequirementForExisting.UpdatePhase2)));
}
var phase2WorkItemsToUpdateCount = successfulWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase2WorkItemsToUpdateCount, Constants.BatchSize);
if (phase2WorkItemsToUpdateCount == 0)
{
Logger.LogInformation(LogDestination.File, "No work items to process for phase 2");
return;
}
await successfulWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
Logger.LogTrace(LogDestination.File, $"Reading Phase 2 source and target work items for batch {batchId} of {totalNumberOfBatches}");
// make web call to get source and target work items
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
IList <WorkItem> targetWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.TargetClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.TargetId.Value).ToList(), expand: WorkItemExpand.Relations);
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
batchContext.SourceWorkItemIdToTargetWorkItemIdMapping = workItemMigrationStateBatch.ToDictionary(key => key.SourceId, value => value.TargetId.Value);
foreach (var sourceWorkItem in sourceWorkItemsInBatch)
{
int targetId = Migrator.GetTargetId(sourceWorkItem.Id.Value, workItemMigrationStateBatch);
batchContext.TargetIdToSourceWorkItemMapping.Add(targetId, sourceWorkItem);
}
Logger.LogTrace(LogDestination.File, $"Generating Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var sourceIdToWitBatchRequests = await GenerateWitBatchRequestsForPhase2Batch(batchContext, batchId, workItemMigrationStateBatch, sourceWorkItemsInBatch, targetWorkItemsInBatch);
Logger.LogTrace(LogDestination.File, $"Saving Phase 2 json patch operations for batch {batchId} of {totalNumberOfBatches}");
var phase2ApiWrapper = new Phase2ApiWrapper();
await phase2ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
Logger.LogTrace(LogDestination.File, $"Completed Phase 2 for batch {batchId} of {totalNumberOfBatches}");
});
Logger.LogInformation("Completed migration phase 2");
}
public void GetSourceWorkItemApiEndpoint_AccountWithoutSlashReturnsCorrectValue()
{
string account = "accountWithoutSlash";
int workItemId = 777;
string expected = "accountWithoutSlash/_apis/wit/workItems/777";
string actual = ClientHelpers.GetWorkItemApiEndpoint(account, workItemId);
Assert.AreEqual(expected, actual);
}
/// <summary>
/// Updates the stack and returns the stack ID
/// </summary>
/// <param name="stack">The stack name or id</param>
/// <param name="deployment">The current deployment</param>
/// <param name="template">The CloudFormation template</param>
/// <returns>stackId</returns>
private async Task <string> UpdateCloudFormation(
RunningDeployment deployment,
StackArn stack,
CloudFormationTemplate template)
{
Guard.NotNull(deployment, "deployment can not be null");
try
{
var result = await ClientHelpers.CreateCloudFormationClient(awsEnvironmentGeneration).UpdateStackAsync(new UpdateStackRequest
{
StackName = stackName,
TemplateBody = template.Content,
Parameters = template.Inputs.ToList(),
Capabilities = capabilities,
RoleARN = roleArnProvider(deployment)
});
Log.Info(
$"Updated stack with id {result.StackId} in region {awsEnvironmentGeneration.AwsRegion.SystemName}");
return(result.StackId);
}
catch (AmazonCloudFormationException ex)
{
// Some stack states indicate that we can delete the stack and start again. Otherwise we have some other
// exception that needs to be dealt with.
if (!(await StackMustBeDeleted(stack)).SelectValueOrDefault(x => x))
{
// Is this an unrecoverable state, or just a stack that has nothing to update?
if (DealWithUpdateException(ex))
{
// There was nothing to update, but we return the id for consistency anyway
var result = await QueryStackAsync(clientFactory, stack);
return(result.StackId);
}
}
// If the stack exists, is in a ROLLBACK_COMPLETE state, and was never successfully
// created in the first place, we can end up here. In this case we try to create
// the stack from scratch.
await DeleteCloudFormation(stack);
await clientFactory.WaitForStackToComplete(CloudFormationDefaults.StatusWaitPeriod, stack, LogAndThrowRollbacks(clientFactory, stack, false));
return(await CreateCloudFormation(deployment, template));
}
catch (AmazonServiceException ex)
{
LogAmazonServiceException(ex);
throw ex;
}
}
private async Task <IList <(int SourceId, WitBatchRequest WitBatchRequest)> > GenerateWitBatchRequestsForPhase2Batch(IBatchMigrationContext batchContext, int batchId, IList <WorkItemMigrationState> workItemMigrationState, IList <WorkItem> sourceWorkItems, IList <WorkItem> targetWorkItems)
{
IList <(int SourceId, WitBatchRequest WitBatchRequest)> result = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(context.Config);
foreach (IPhase2Processor processor in phase2Processors)
{
Logger.LogInformation(LogDestination.File, $"Starting preprocessing of phase 2 step {processor.Name} for batch {batchId}");
await processor.Preprocess(context, batchContext, sourceWorkItems, targetWorkItems);
Logger.LogInformation(LogDestination.File, $"Completed preprocessing of phase 2 step {processor.Name} for batch {batchId}");
}
foreach (var sourceToTarget in batchContext.SourceWorkItemIdToTargetWorkItemIdMapping)
{
int sourceId = sourceToTarget.Key;
int targetId = sourceToTarget.Value;
WorkItem sourceWorkItem = sourceWorkItems.First(a => a.Id == sourceId);
WorkItem targetWorkItem = targetWorkItems.First(a => a.Id == targetId);
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
WorkItemMigrationState state = workItemMigrationState.First(a => a.SourceId == sourceId);
state.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, sourceId);
ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(context.Config)).Select(b => b.Name));
enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations);
foreach (IPhase2Processor processor in phase2Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, batchContext, sourceWorkItem, targetWorkItem);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
jsonPatchOperations.Add(GetAddHyperlinkWithCommentOperation(targetWorkItems, state, sourceId, targetId, sourceWorkItem, enabledPhaseStatuses));
if (this.context.Config.IncludeWebLink)
{
var link = (ReferenceLink)sourceWorkItem.Links.Links["html"];
var addWebLinkOperation = MigrationHelpers.GetHyperlinkAddOperation(link.Href);
jsonPatchOperations.Add(addWebLinkOperation);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, targetId);
result.Add((sourceId, witBatchRequest));
}
}
return(result);
}
private async Task InstallAsync(RunningDeployment deployment)
{
//The bucket should exist at this point
Guard.NotNull(deployment, "deployment can not be null");
if (!md5HashSupported)
{
Log.Info("MD5 hashes are not supported in executing environment. Files will always be uploaded.");
}
var options = optionsProvider.GetOptions(targetMode);
AmazonS3Client Factory() => ClientHelpers.CreateS3Client(awsEnvironmentGeneration);
try
{
(await UploadAll(options, Factory, deployment)).Tee(responses =>
{
var results = responses.Where(z => z.IsSuccess()).ToArray();
if (targetMode == S3TargetMode.EntirePackage && results.FirstOrDefault() != null)
{
SetOutputVariables(deployment, results.FirstOrDefault());
}
else if (targetMode == S3TargetMode.FileSelections)
{
foreach (var result in results)
{
var fileName = Path.GetFileName(result.BucketKey);
SetOutputVariables(deployment, result, fileName);
}
}
});
}
catch (AmazonS3Exception exception)
{
if (exception.ErrorCode == "AccessDenied")
{
throw new PermissionException("The AWS account used to perform the operation does not have the " +
$"the required permissions to upload to bucket {bucket}");
}
throw new UnknownException(
$"An unrecognized {exception.ErrorCode} error was thrown while uploading to bucket {bucket}");
}
catch (AmazonServiceException exception)
{
HandleAmazonServiceException(exception);
throw;
}
}
/// <summary>
/// Wraps the call to CreateClient, handling any specific errors that could be thrown
/// around invalid PAT and account name and wraps them in a ValidationException with
/// a helpful message.
/// </summary>
private WorkItemClientConnection CreateValidationClient(ConfigConnection connection)
{
try
{
return(ClientHelpers.CreateClient(connection));
}
catch (Exception e) when(e is VssServiceResponseException && e.Message == "The resource cannot be found.")
{
throw new ValidationException(connection.Account, (VssServiceResponseException)e);
}
catch (Exception e) when(e is VssUnauthorizedException)
{
throw new ValidationException(connection.Account, (VssUnauthorizedException)e);
}
}
public async Task <string> GetGitHubFileContentsAsync(
string path,
GitHubBranch branch)
{
try
{
GitHubContents file = await GetGitHubFileAsync(path, branch.Project, $"heads/{branch.Name}");
return(ClientHelpers.FromBase64(file.Content));
}
catch (HttpFailureResponseException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
return(null);
}
}
private async Task ValidateConfiguration()
{
Logger.LogInformation("Starting configuration validation");
foreach (IConfigurationValidator validator in ClientHelpers.GetInstances <IConfigurationValidator>())
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Starting configuration validation for: {validator.Name}");
await validator.Validate(context);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Completed configuration validation for: {validator.Name} in {stopwatch.Elapsed.TotalSeconds}s");
}
Logger.LogInformation("Completed configuration validation");
}
public async Task <string> GetGitHubFileContentsAsync(
string path,
GitHubProject project,
string @ref)
{
try
{
GitHubContents file = await GetGitHubFileAsync(path, project, @ref);
return(ClientHelpers.FromBase64(file.Content));
}
catch (HttpFailureResponseException ex) when(ex.HttpStatusCode == HttpStatusCode.NotFound)
{
return(null);
}
}
private async Task <AuthenticationHeaderValue> GetHeadersAsync(string token, string url, HttpMethod method, string liveSessionToken, object payload)
{
var nonce = ClientHelpers.GetNonce();
var timestamp = ClientHelpers.GetTimestamp();
var data = await payload.ToKeyValueAsync() ?? new Dictionary <string, string>();
data.Add("oauth_consumer_key", s_clientId);
data.Add("oauth_timestamp", timestamp.ToString());
data.Add("oauth_nonce", nonce);
data.Add("oauth_signature_method", "HMAC-SHA256");
data.Add("oauth_token", token);
data.Add("oauth_signature", ClientHelpers.GetSignatureBaseString(_httpClient.BaseAddress.AbsoluteUri + url, data, liveSessionToken, sigMethod: "HMAC-SHA256", httpMethod: method.Method));
return(ClientHelpers.GetDefaultAuthHeader(data, true));
}
private async Task MigratePhase1()
{
Logger.LogInformation("Starting migration phase 1");
foreach (IPhase1Processor migrator in ClientHelpers.GetProcessorInstances <IPhase1Processor>(context.Config))
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Starting migration phase for: {migrator.Name}");
await migrator.Process(context);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Completed migration phase for: {migrator.Name} in {stopwatch.Elapsed.TotalSeconds}s");
}
Logger.LogInformation("Completed migration phase 1");
}
private async Task TargetWorkItemValidation()
{
Logger.LogInformation("Starting target work item migration status");
foreach (ITargetValidator validator in ClientHelpers.GetInstances <ITargetValidator>())
{
var stopwatch = Stopwatch.StartNew();
Logger.LogInformation(LogDestination.File, $"Starting target work item migration status for: {validator.Name}");
await validator.Validate(context);
stopwatch.Stop();
Logger.LogInformation(LogDestination.File, $"Completed target work item migration status for: {validator.Name} in {stopwatch.Elapsed.TotalSeconds}s");
}
Logger.LogInformation("Completed target work item migration status");
}
public AuthenticationHeaderValue GetRequestTokenHeaders(string url)
{
var nonce = ClientHelpers.GetNonce();
var timestamp = ClientHelpers.GetTimestamp();
var data = new Dictionary <string, string>
{
{ "oauth_consumer_key", s_clientId },
{ "oauth_timestamp", timestamp.ToString() },
{ "oauth_nonce", nonce },
{ "oauth_signature_method", "RSA-SHA256" },
{ "oauth_callback", "oob" },
{ "oauth_version", "1.0" }
};
data.Add("oauth_signature", ClientHelpers.GetSignatureBaseString(_httpClient.BaseAddress.AbsoluteUri + url, data));
return(ClientHelpers.GetDefaultAuthHeader(data));
}
private async Task MigratePhase3()
{
IEnumerable <IPhase3Processor> phase3Processors = ClientHelpers.GetProcessorInstances <IPhase3Processor>(context.Config);
if (phase3Processors != null && !phase3Processors.Any())
{
// nothing to do if no phase 3 processors are enabled
return;
}
// Phase1 or Phase2 have completed, and FailureReason == None
IEnumerable <WorkItemMigrationState> successfullyMigratedWorkItemMigrationStates = context.WorkItemsMigrationState.Where(w => (w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase1) || w.MigrationCompleted.HasFlag(WorkItemMigrationState.MigrationCompletionStatus.Phase2)) && w.FailureReason == FailureReason.None);
var phase3WorkItemsToUpdateCount = successfullyMigratedWorkItemMigrationStates.Count();
var totalNumberOfBatches = ClientHelpers.GetBatchCount(phase3WorkItemsToUpdateCount, Constants.BatchSize);
if (phase3WorkItemsToUpdateCount == 0)
{
return;
}
await successfullyMigratedWorkItemMigrationStates.Batch(Constants.BatchSize).ForEachAsync(context.Config.Parallelism, async(workItemMigrationStateBatch, batchId) =>
{
IBatchMigrationContext batchContext = new BatchMigrationContext(batchId, workItemMigrationStateBatch);
IList <(int SourceId, WitBatchRequest WitBatchRequest)> sourceIdToWitBatchRequests = new List <(int SourceId, WitBatchRequest WitBatchRequest)>();
IList <WorkItem> sourceWorkItemsInBatch = await WorkItemTrackingHelpers.GetWorkItemsAsync(context.SourceClient.WorkItemTrackingHttpClient, workItemMigrationStateBatch.Select(a => a.SourceId).ToList(), expand: WorkItemExpand.All);
foreach (WorkItem sourceWorkItem in sourceWorkItemsInBatch)
{
IList <JsonPatchOperation> jsonPatchOperations = new List <JsonPatchOperation>();
foreach (IPhase3Processor processor in phase3Processors)
{
IEnumerable <JsonPatchOperation> processorJsonPatchOperations = await processor.Process(context, null, sourceWorkItem, null);
jsonPatchOperations.AddRange(processorJsonPatchOperations);
}
if (jsonPatchOperations.Any())
{
WitBatchRequest witBatchRequest = GenerateWitBatchRequestFromJsonPatchOperations(jsonPatchOperations, sourceWorkItem.Id.Value);
sourceIdToWitBatchRequests.Add((sourceWorkItem.Id.Value, witBatchRequest));
}
}
var phase3ApiWrapper = new Phase3ApiWrapper();
await phase3ApiWrapper.ExecuteWitBatchRequests(sourceIdToWitBatchRequests, context, batchContext);
});
}
private bool IsPhase2UpdateRequired(WorkItemMigrationState workItemMigrationState, WorkItem targetWorkItem)
{
IEnumerable <IPhase2Processor> phase2Processors = ClientHelpers.GetProcessorInstances <IPhase2Processor>(ValidationContext.Config);
workItemMigrationState.RevAndPhaseStatus = GetRevAndPhaseStatus(targetWorkItem, workItemMigrationState.SourceId);
// find out if Enabled, see if matches comment from target
ISet <string> enabledPhaseStatuses = System.Linq.Enumerable.ToHashSet(phase2Processors.Where(a => a.IsEnabled(ValidationContext.Config)).Select(b => b.Name));
enabledPhaseStatuses.Remove(Constants.RelationPhaseClearAllRelations);
if (enabledPhaseStatuses.IsSubsetOf(workItemMigrationState.RevAndPhaseStatus.PhaseStatus)) // enabled relation phases are already complete for current work item
{
return(false);
}
return(true);
}
