protected async override Task ExecuteCoreAsync(BaseSchema baseSchema, string manifestContent, string filePath)
{
var manifest = Deserialize <ProjectManifest>(manifestContent);
var projectService = GetProjectService();
var repoService = GetRepositoryService();
using var op = Insights.BeginOperation($"Validating Manifest file [{filePath}]...", "Validation");
if (manifest.Validate())
{
var outcome = await EnsureProjectExistsAsync(manifest, projectService);
await ProcessPermissionsAsync(manifest, projectService, outcome);
await EnsureTeamProvisionedAsync(manifest, projectService, outcome);
await EnsureRepositoriesExistsAsync(manifest, repoService, outcome.Item1, outcome.Item2);
var seOutcome = await EnsureServiceEndpointExistsAsync(manifest, projectService, outcome);
await EnsureEnvironmentExistsAsync(manifest, outcome.Item1, seOutcome);
await EnsureBuildFoldersAsync(manifest, outcome.Item1);
await EnsureReleaseFoldersAsync(manifest, outcome.Item1);
}
else
{
op.EndWithFailure("Invalid manifest file!");
}
}
protected async override Task ExecuteCoreAsync()
{
using var migrationSession = Insights.BeginOperation("WorkItem-Migration");
await PrepareMigrationAsync();
var successItemCount = 0;
var failedItemCount = 0;
foreach (var migrationItem in await ListMigrationItemsAsync())
{
await MapMigrationFieldsAsync(migrationItem);
await EnrichMigrationFieldsAsync(migrationItem);
var status = await MigrateItemAsync(migrationItem);
if (status)
{
++successItemCount;
}
else
{
++failedItemCount;
}
}
Insights.TrackMetric("TotalWorkItemCount", successItemCount + failedItemCount);
Insights.TrackMetric("SuceededWorkItemCount", successItemCount);
Insights.TrackMetric("FailedWorkItemCount", failedItemCount);
}
protected async Task ProvisionEnvironmentPermissionsAsync(
Kdoctl.CliServices.AzDoServices.Dtos.Project project,
PipelineEnvironmentService peService,
EnvironmentManifest pe, PipelineEnvironment envObject)
{
if (envObject != null && pe.Permissions != null && pe.Permissions.Any())
{
foreach (var permissionObject in pe.Permissions)
{
using var op = Insights.BeginOperation($"Configuring Environment ({pe.Name}) permissions: AAD object ({permissionObject.Group}) ...", "Envrionment");
var group = await GetGroupByNameAsync(
permissionObject.Origin, permissionObject.Group, permissionObject.Id);
if (group != null)
{
var legacyIdentity = await GetGraphService()
.GetLegacyIdentitiesBySidAsync(group.Sid);
if (legacyIdentity != null && legacyIdentity.Value.Any())
{
var localId = legacyIdentity.Value.First().Id;
foreach (var role in permissionObject.Roles)
{
await peService.SetPermissionAsync(project.Id, envObject.Id, localId, role);
}
}
}
else
{
op.EndWithFailure("Failed (Not found in AAD)");
}
}
}
}
protected async Task EnsureServiceEndpointForKubernetesAsync(
ProjectManifest manifest,
ServiceEndpointManifest seManifest,
ProjectService projectService,
k8s.Models.V1Secret secret,
Tuple <Kdoctl.CliServices.AzDoServices.Dtos.Project, bool> outcome)
{
var seService = GetServiceEndpointService();
var project = outcome.Item1;
var eps = await seService.ListServiceEndpointsAsync(project.Id);
if (eps != null && eps.Value != null)
{
var endpoint = eps.Value.FirstOrDefault(ep => ep.Name.Equals(seManifest.Name, StringComparison.OrdinalIgnoreCase));
if (endpoint == null)
{
using var op = Insights.BeginOperation($"Creating Kubernetes Service Endpoint '{seManifest.Name}' ...");
var newEndpoint = await seService.CreateKubernetesEndpointAsync(
project.Id, project.Name,
seManifest.Name, seManifest.Description,
GetK8sService().GetClusterAPIUri().ToString(),
Convert.ToBase64String(secret.Data["ca.crt"]),
Convert.ToBase64String(secret.Data["token"]));
if (newEndpoint != null)
{
op.EndWithSuccess();
}
else
{
op.EndWithFailure();
}
}
else
{
using var op = Insights.BeginOperation($"Kubernetes Service Endpoint '{seManifest.Name}' ...");
var updatedEndpoint = await seService.UpdateKubernetesEndpointAsync(
endpoint.Id,
project.Id, project.Name,
seManifest.Name, seManifest.Description,
GetK8sService().GetClusterAPIUri().ToString(),
Convert.ToBase64String(secret.Data["ca.crt"]),
Convert.ToBase64String(secret.Data["token"]));
if (updatedEndpoint != null)
{
op.EndWithSuccess();
}
else
{
op.EndWithFailure();
}
}
}
}
private async Task ApplyGroupPermissionsAsync(
GraphService gService,
GroupCollection allUsers,
Guid projectId,
PermissionSchemaManifest permissionEntry,
VstsGroup targetGroup)
{
using var op = Insights.BeginOperation($"Updating membership of [{targetGroup.PrincipalName}]...", "GroupPermissions");
// get existing members - so later we can remove the unwanted members (no longer in yaml)
var outdatedMembership = await gService.GetGroupMembersAsync(targetGroup.Descriptor);
var survivorDescriptors = new List <string>();
if (permissionEntry.Membership.Groups != null && permissionEntry.Membership.Groups.Any())
{
foreach (var gp in permissionEntry.Membership.Groups)
{
var groupObject = await GetGroupByNameAsync(IdentityOrigin.Aad.ToString(), gp.Name, gp.Id);
if (groupObject != null)
{
await gService.AddMemberAsync(projectId, targetGroup.Descriptor, groupObject.Descriptor);
survivorDescriptors.Add(groupObject.Descriptor);
}
}
}
if (permissionEntry.Membership.Users != null && permissionEntry.Membership.Users.Any())
{
foreach (var user in permissionEntry.Membership.Users)
{
var userInfo = allUsers.Value.FirstOrDefault(u => u.OriginId.Equals(user.Id));
if (userInfo != null)
{
await gService.AddMemberAsync(projectId, targetGroup.Descriptor, userInfo.Descriptor);
survivorDescriptors.Add(userInfo.Descriptor);
}
}
}
if (outdatedMembership != null && outdatedMembership.Members != null && outdatedMembership.Members.Any())
{
foreach (var potentialOutdatedMember in outdatedMembership.Members)
{
var remainValid = survivorDescriptors
.Exists(s => s.Contains(potentialOutdatedMember.MemberDescriptor, StringComparison.OrdinalIgnoreCase));
if (!remainValid)
{
await gService.RemoveMembershipAsync(projectId, targetGroup.Descriptor, potentialOutdatedMember.MemberDescriptor);
}
}
}
}
protected async Task <Tuple <Kdoctl.CliServices.AzDoServices.Dtos.Project, bool> > EnsureProjectExistsAsync(
ProjectManifest manifest, ProjectService projectService)
{
var projectCreatedJIT = false;
var projects = await projectService.ListProjectsAsync();
var project = projects.Value.FirstOrDefault(p => p.Name.Equals(manifest.Metadata.Name,
StringComparison.OrdinalIgnoreCase));
if (project == null)
{
using var op = Insights.BeginOperation("Creating project, reading Process templates...", "Project");
var templates = await projectService.ListProcessAsync();
var tempalte = templates.Value.FirstOrDefault(t => t.Name.Equals(manifest.Template.Name, StringComparison.InvariantCulture));
if (tempalte == null)
{
op.EndWithFailure($"Process template {manifest.Template.Name} is not valid! Good example: Agile, CMMI, Basic, Scrum etc.");
throw new InvalidOperationException($"Process template {manifest.Template.Name} is not valid! Good example: Agile, CMMI, Basic, Scrum etc.");
}
await projectService.CreateProjectAsync(
manifest.Metadata.Name, tempalte,
manifest.Template.SourceControlType, manifest.Metadata.Description);
projectCreatedJIT = true;
while (project == null)
{
projects = await projectService.ListProjectsAsync();
project = projects.Value.FirstOrDefault(p => p.Name.Equals(manifest.Metadata.Name,
StringComparison.OrdinalIgnoreCase));
}
}
await projectService.UpdateRetentionAsync(project.Id, new ProjectRetentionSetting
{
ArtifactsRetention = new UpdateRetentionSettingSchema {
Value = 15
},
PullRequestRunRetention = new UpdateRetentionSettingSchema {
Value = 12
},
RetainRunsPerProtectedBranch = new UpdateRetentionSettingSchema {
Value = 4
},
RunRetention = new UpdateRetentionSettingSchema {
Value = 12
},
});
return(new Tuple <Kdoctl.CliServices.AzDoServices.Dtos.Project, bool>(project, projectCreatedJIT));
}
protected async Task EnsureReleaseFoldersAsync(ProjectManifest manifest,
Kdoctl.CliServices.AzDoServices.Dtos.Project project)
{
if (manifest.ReleaseFolders != null && manifest.ReleaseFolders.Any())
{
var releaseService = GetReleaseService();
var releasePaths = await releaseService.ListFoldersAsync(project.Id);
foreach (var rp in manifest.ReleaseFolders)
{
var existingItem = releasePaths.Value
.FirstOrDefault(p => p.Path.Replace("\\", "/").Equals(rp.Path, StringComparison.OrdinalIgnoreCase));
if (existingItem == null)
{
existingItem = await releaseService.CreateFolderAsync(project.Id, rp.Path);
}
using var op = Insights.BeginOperation($"Creating permissions {rp.Path}...", "ReleaseFolderPermissions");
await ProvisionReleasePathPermissionsAsync(project, rp, existingItem);
}
}
}
protected async Task EnsureBuildFoldersAsync(
ProjectManifest manifest,
Kdoctl.CliServices.AzDoServices.Dtos.Project project)
{
if (manifest.BuildFolders != null && manifest.BuildFolders.Any())
{
var buildService = this.GetBuildService();
var buildPaths = await buildService.ListFoldersAsync(project.Id);
foreach (var bp in manifest.BuildFolders)
{
var existingItem = buildPaths.Value
.FirstOrDefault(p => p.Path.Replace("\\", "/").Equals(bp.Path, StringComparison.OrdinalIgnoreCase));
if (existingItem == null)
{
existingItem = await buildService.CreateFolderAsync(project.Id, bp.Path);
}
using var op = Insights.BeginOperation($"Creating permissions {bp.Path}...", "Build-Folder-Permissions");
await ProvisionBuildPathPermissionsAsync(project, bp, existingItem);
}
}
}
protected async Task EnsureRepositoriesExistsAsync(
ProjectManifest manifest,
RepositoryService repoService,
Kdoctl.CliServices.AzDoServices.Dtos.Project project,
bool projectWasAbsent)
{
if (project != null && manifest.Repositories != null && manifest.Repositories.Any())
{
foreach (var repo in manifest.Repositories)
{
if (!string.IsNullOrWhiteSpace(repo.Name))
{
var reposCollection = await repoService.GetRepositoryListAsync(project.Id);
var repository = reposCollection
.FirstOrDefault(r => r.Name.Equals(repo.Name, StringComparison.OrdinalIgnoreCase));
if (repository == null)
{
using var op = Insights.BeginOperation($"Creating Repository {repo.Name}...", "Repository");
await ExecutionSupports.Retry(async() =>
{
repository = await repoService.CreateAsync(project.Id, repo.Name);
},
exception => { Insights.TrackException(exception); });
op.EndWithSuccess("Succeed");
}
using var opPermissions = Insights.BeginOperation($"Setting up permissions for repository {repo.Name}...", "RepoPermissions");
await EnsureRepositoryPermissionsAsync(project, repo, repository);
}
}
await DeleteDefaultRepoAsync(repoService, project, projectWasAbsent);
}
}
protected async Task EnsureTeamProvisionedAsync(
ProjectManifest manifest,
ProjectService projectService,
Tuple <Kdoctl.CliServices.AzDoServices.Dtos.Project, bool> outcome)
{
if (manifest.Teams != null && manifest.Teams.Any())
{
var gService = GetGraphService();
var secService = GetSecurityNamespaceService();
var aclService = GetAclListService();
var allUsers = await gService.ListUsersAsync();
foreach (var teamManifest in manifest.Teams)
{
var tc = await projectService.GetTeamsAsync();
var eteam = tc.Value
.FirstOrDefault(tc => tc.Name.Equals(teamManifest.Name,
StringComparison.OrdinalIgnoreCase));
if (eteam == null)
{
using var op = Insights.BeginOperation($"Creating team [{teamManifest.Name}]...", "Team");
var team = await projectService.CreateTeamAsync(
new Microsoft.TeamFoundation.Core.WebApi.WebApiTeam
{
Name = teamManifest.Name,
Description = teamManifest.Description,
ProjectId = outcome.Item1.Id,
ProjectName = outcome.Item1.Name
},
outcome.Item1.Id);
var breakOut = 0;
while (eteam == null)
{
tc = await projectService.GetTeamsAsync();
eteam = tc.Value
.FirstOrDefault(tc => tc.Name.Equals(teamManifest.Name,
StringComparison.OrdinalIgnoreCase));
if (++breakOut > 10)
{
op.EndWithFailure($"Team [{teamManifest.Name}] was not retrieved on time.");
throw new InvalidOperationException($"Team [{teamManifest.Name}] was not retrieved on time.");
}
}
}
if (eteam != null && teamManifest.Membership != null &&
(teamManifest.Membership.Groups != null && teamManifest.Membership.Groups.Any()))
{
var teamGroup = await GetGroupByNameAsync(IdentityOrigin.Vsts.ToString(), eteam.Name);
if (teamGroup != null)
{
foreach (var gp in teamManifest.Membership.Groups)
{
var groupObject = await GetGroupByNameAsync(IdentityOrigin.Aad.ToString(), gp.Name, gp.Id);
if (groupObject != null)
{
await gService.AddMemberAsync(eteam.ProjectId, teamGroup.Descriptor, groupObject.Descriptor);
}
}
foreach (var user in teamManifest.Membership.Users)
{
var userInfo = allUsers.Value.FirstOrDefault(u => u.OriginId.Equals(user.Id));
if (userInfo != null)
{
await gService.AddMemberAsync(eteam.ProjectId, teamGroup.Descriptor, userInfo.Descriptor);
}
}
}
}
if (eteam != null &&
teamManifest.Admins != null &&
teamManifest.Admins.Any())
{
var token = $"{eteam.ProjectId}\\{eteam.Id}";
var releaseNamespace = await secService.GetNamespaceAsync(SecurityNamespaceConstants.Identity);
var secNamespaceId = releaseNamespace.NamespaceId;
var aclDictioanry = new Dictionary <string, VstsAcesDictionaryEntry>();
foreach (var adminUserName in teamManifest.Admins)
{
var matches = await gService.GetLegacyIdentitiesByNameAsync(adminUserName.Name);
if (matches != null && matches.Count > 0)
{
var adminUserInfo = matches.Value.First();
aclDictioanry.Add(adminUserInfo.Descriptor, new VstsAcesDictionaryEntry
{
Allow = 31,
Deny = 0,
Descriptor = adminUserInfo.Descriptor
});
}
}
await aclService.SetAclsAsync(secNamespaceId, token, aclDictioanry, false);
}
}
}
}
