public async Task ConfigureNotifications( string projectName, string projectPath, GitHubToAADConverter gitHubToAADConverter, bool persistChanges = true, PipelineSelectionStrategy strategy = PipelineSelectionStrategy.Scheduled) { var pipelines = await GetPipelinesAsync(projectName, projectPath, strategy); var teams = await service.GetAllTeamsAsync(projectName); foreach (var pipeline in pipelines) { using (logger.BeginScope("Evaluate Pipeline: Name = {0}, Path = {1}, Id = {2}", pipeline.Name, pipeline.Path, pipeline.Id)) { var parentTeam = await EnsureTeamExists(pipeline, TeamPurpose.ParentNotificationTeam, teams, gitHubToAADConverter, persistChanges); var childTeam = await EnsureTeamExists(pipeline, TeamPurpose.SynchronizedNotificationTeam, teams, gitHubToAADConverter, persistChanges); if (!persistChanges && (parentTeam == default || childTeam == default)) { // Skip team nesting and notification work if logger.LogInformation("Skipping Teams and Notifications because parent or child team does not exist"); continue; } await EnsureSynchronizedNotificationTeamIsChild(parentTeam, childTeam, persistChanges); } } }
/// <summary> /// Create notification groups for failures in scheduled builds /// </summary> /// <param name="organization">Azure DevOps Organization</param> /// <param name="project">Name of the DevOps project</param> /// <param name="pathPrefix">Path prefix to include pipelines (e.g. "\net")</param> /// <param name="tokenVariableName">Environment variable token name (e.g. "SYSTEM_ACCESSTOKEN")</param> /// <param name="aadAppIdVar">AAD App ID environment variable name (OpensourceAPI access)</param> /// <param name="aadAppSecretVar">AAD App Secret environment variable name (OpensourceAPI access)</param> /// <param name="aadTenantVar">AAD Tenant environment variable name (OpensourceAPI access)</param> /// <param name="selectionStrategy">Pipeline selection strategy</param> /// <param name="dryRun">Prints changes but does not alter any objects</param> /// <returns></returns> static async Task Main( string organization, string project, string pathPrefix, string tokenVariableName, string aadAppIdVar, string aadAppSecretVar, string aadTenantVar, PipelineSelectionStrategy selectionStrategy = PipelineSelectionStrategy.Scheduled, bool dryRun = false) { var devOpsToken = Environment.GetEnvironmentVariable(tokenVariableName); var devOpsCreds = new VssBasicCredential("nobody", devOpsToken); var devOpsConnection = new VssConnection(new Uri($"https://dev.azure.com/{organization}/"), devOpsCreds); #pragma warning disable CS0618 // Type or member is obsolete var loggerFactory = LoggerFactory.Create(builder => { builder.AddConsole(config => { config.IncludeScopes = true; }); }); #pragma warning restore CS0618 // Type or member is obsolete var devOpsServiceLogger = loggerFactory.CreateLogger <AzureDevOpsService>(); var notificationConfiguratorLogger = loggerFactory.CreateLogger <NotificationConfigurator>(); var devOpsService = new AzureDevOpsService(devOpsConnection, devOpsServiceLogger); var gitHubService = new GitHubService(loggerFactory.CreateLogger <GitHubService>()); var credential = new ClientSecretCredential( Environment.GetEnvironmentVariable(aadTenantVar), Environment.GetEnvironmentVariable(aadAppIdVar), Environment.GetEnvironmentVariable(aadAppSecretVar)); var githubToAadResolver = new GitHubToAADConverter( credential, loggerFactory.CreateLogger <GitHubToAADConverter>() ); var configurator = new NotificationConfigurator(devOpsService, gitHubService, notificationConfiguratorLogger); await configurator.ConfigureNotifications( project, pathPrefix, githubToAadResolver, persistChanges : !dryRun, strategy : selectionStrategy); }
/// <summary> /// Synchronizes CODEOWNERS contacts to appropriate DevOps groups /// </summary> /// <param name="organization">Azure DevOps organization name</param> /// <param name="project">Azure DevOps project name</param> /// <param name="devOpsTokenVar">Personal Access Token environment variable name</param> /// <param name="aadAppIdVar">AAD App ID environment variable name (OpensourceAPI access)</param> /// <param name="aadAppSecretVar">AAD App Secret environment variable name (OpensourceAPI access)</param> /// <param name="aadTenantVar">AAD Tenant environment variable name (OpensourceAPI access)</param> /// <param name="pathPrefix">Azure DevOps path prefix (e.g. "\net")</param> /// <param name="dryRun">Do not persist changes</param> /// <returns></returns> public static async Task Main( string organization, string project, string devOpsTokenVar, string aadAppIdVar, string aadAppSecretVar, string aadTenantVar, string pathPrefix = "", bool dryRun = false ) { #pragma warning disable CS0618 // Type or member is obsolete using (var loggerFactory = LoggerFactory.Create(builder => { builder.AddConsole(config => { config.IncludeScopes = true; }); })) #pragma warning restore CS0618 // Type or member is obsolete { var devOpsService = AzureDevOpsService.CreateAzureDevOpsService( Environment.GetEnvironmentVariable(devOpsTokenVar), $"https://dev.azure.com/{organization}/", loggerFactory.CreateLogger <AzureDevOpsService>() ); var gitHubServiceLogger = loggerFactory.CreateLogger <GitHubService>(); var gitHubService = new GitHubService(gitHubServiceLogger); var credential = new ClientSecretCredential( Environment.GetEnvironmentVariable(aadTenantVar), Environment.GetEnvironmentVariable(aadAppIdVar), Environment.GetEnvironmentVariable(aadAppSecretVar)); var githubToAadResolver = new GitHubToAADConverter( credential, loggerFactory.CreateLogger <GitHubToAADConverter>() ); var logger = loggerFactory.CreateLogger <Program>(); var pipelines = (await devOpsService.GetPipelinesAsync(project)).ToDictionary(p => p.Id); var pipelineGroupTasks = (await devOpsService.GetAllTeamsAsync(project)) .Where(team => YamlHelper.Deserialize <TeamMetadata>(team.Description, swallowExceptions: true)?.Purpose == TeamPurpose.SynchronizedNotificationTeam ).Select(async team => { BuildDefinition pipeline; var pipelineId = YamlHelper.Deserialize <TeamMetadata>(team.Description).PipelineId; if (!pipelines.ContainsKey(pipelineId)) { pipeline = await devOpsService.GetPipelineAsync(project, pipelineId); } else { pipeline = pipelines[pipelineId]; } if (pipeline == default) { logger.LogWarning($"Could not find pipeline with id {pipelineId} referenced by team {team.Name}."); } return(new { Pipeline = pipeline, Team = team }); }); var pipelineGroups = await Task.WhenAll(pipelineGroupTasks); var filteredGroups = pipelineGroups.Where(group => group.Pipeline != default && group.Pipeline.Path.StartsWith(pathPrefix)); foreach (var group in filteredGroups) { using (logger.BeginScope("Team Name = {0}", group.Team.Name)) { if (group.Pipeline.Process.Type != PipelineYamlProcessType) { continue; } // Get contents of CODEOWNERS logger.LogInformation("Fetching CODEOWNERS file"); var managementUrl = new Uri(group.Pipeline.Repository.Properties["manageUrl"]); var codeOwnerEntries = await gitHubService.GetCodeownersFile(managementUrl); if (codeOwnerEntries == default) { logger.LogInformation("CODEOWNERS file not found, skipping sync"); continue; } var process = group.Pipeline.Process as YamlProcess; logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename); var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename); codeOwnerEntry.FilterOutNonUserAliases(); logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count); // Get set of team members in the CODEOWNERS file var codeownerPrincipals = codeOwnerEntry.Owners .Select(contact => githubToAadResolver.GetUserPrincipalNameFromGithub(contact)); var codeownersDescriptorsTasks = codeownerPrincipals .Where(userPrincipal => !string.IsNullOrEmpty(userPrincipal)) .Select(userPrincipal => devOpsService.GetDescriptorForPrincipal(userPrincipal)); var codeownersDescriptors = await Task.WhenAll(codeownersDescriptorsTasks); var codeownersSet = new HashSet <string>(codeownersDescriptors); // Get set of existing team members var teamMembers = await devOpsService.GetMembersAsync(group.Team); var teamContactTasks = teamMembers .Select(async member => await devOpsService.GetUserFromId(new Guid(member.Identity.Id))); var teamContacts = await Task.WhenAll(teamContactTasks); var teamDescriptors = teamContacts.Select(contact => contact.SubjectDescriptor.ToString()); var teamSet = new HashSet <string>(teamDescriptors); // Synchronize contacts var contactsToRemove = teamSet.Except(codeownersSet); var contactsToAdd = codeownersSet.Except(teamSet); var teamDescriptor = await devOpsService.GetDescriptorAsync(group.Team.Id); foreach (var descriptor in contactsToRemove) { logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor); if (!dryRun) { await devOpsService.RemoveMember(teamDescriptor, descriptor); } } foreach (var descriptor in contactsToAdd) { logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor); if (!dryRun) { await devOpsService.AddToTeamAsync(teamDescriptor, descriptor); } } } } } }
private async Task <WebApiTeam> EnsureTeamExists( BuildDefinition pipeline, TeamPurpose purpose, IEnumerable <WebApiTeam> teams, GitHubToAADConverter gitHubToAADConverter, bool persistChanges) { string teamName = $"{pipeline.Id} "; if (purpose == TeamPurpose.ParentNotificationTeam) { // Ensure team name fits within maximum 64 character limit // https://docs.microsoft.com/en-us/azure/devops/organizations/settings/naming-restrictions?view=azure-devops#teams string fullTeamName = teamName + $"{pipeline.Name}"; teamName = StringHelper.MaxLength(fullTeamName, MaxTeamNameLength); if (fullTeamName.Length > teamName.Length) { logger.LogWarning($"Notification team name (length {fullTeamName.Length}) will be truncated to {teamName}"); } } else if (purpose == TeamPurpose.SynchronizedNotificationTeam) { teamName += $"Code owners sync notifications"; } bool updateMetadataAndName = false; var result = teams.FirstOrDefault( team => { // Swallowing exceptions because parse errors on // free form text fields which might be non-yaml text // are not exceptional var metadata = YamlHelper.Deserialize <TeamMetadata>(team.Description, swallowExceptions: true); bool metadataMatches = (metadata?.PipelineId == pipeline.Id && metadata?.Purpose == purpose); bool nameMatches = (team.Name == teamName); if (metadataMatches && nameMatches) { return(true); } if (metadataMatches) { logger.LogInformation("Found team with matching pipeline id {0} but different name '{1}', expected '{2}'. Purpose = '{3}'", metadata?.PipelineId, team.Name, teamName, metadata?.Purpose); updateMetadataAndName = true; return(true); } if (nameMatches) { logger.LogInformation("Found team with matching name {0} but different pipeline id {1}, expected {2}. Purpose = '{3}'", team.Name, metadata?.PipelineId, pipeline.Id, metadata?.Purpose); updateMetadataAndName = true; return(true); } return(false); }); if (result == default) { logger.LogInformation("Team Not Found purpose = {0}", purpose); var teamMetadata = new TeamMetadata { PipelineId = pipeline.Id, Purpose = purpose, PipelineName = pipeline.Name, }; var newTeam = new WebApiTeam { Description = YamlHelper.Serialize(teamMetadata), Name = teamName }; logger.LogInformation("Create Team for Pipeline PipelineId = {0} Purpose = {1} Name = '{2}'", pipeline.Id, purpose, teamName); if (persistChanges) { result = await service.CreateTeamForProjectAsync(pipeline.Project.Id.ToString(), newTeam); if (purpose == TeamPurpose.ParentNotificationTeam) { await EnsureScheduledBuildFailSubscriptionExists(pipeline, result, true); } } } else if (updateMetadataAndName) { var teamMetadata = new TeamMetadata { PipelineId = pipeline.Id, Purpose = purpose, }; result.Description = YamlHelper.Serialize(teamMetadata); result.Name = teamName; logger.LogInformation("Update Team for Pipeline PipelineId = {0} Purpose = {1} Name = '{2}'", pipeline.Id, purpose, teamName); if (persistChanges) { result = await service.UpdateTeamForProjectAsync(pipeline.Project.Id.ToString(), result); if (purpose == TeamPurpose.ParentNotificationTeam) { await EnsureScheduledBuildFailSubscriptionExists(pipeline, result, true); } } } if (purpose == TeamPurpose.SynchronizedNotificationTeam) { await SyncTeamWithCodeOwnerFile(pipeline, result, gitHubToAADConverter, gitHubService, persistChanges); } return(result); }
private async Task SyncTeamWithCodeOwnerFile(BuildDefinition pipeline, WebApiTeam team, GitHubToAADConverter gitHubToAADConverter, GitHubService gitHubService, bool persistChanges) { using (logger.BeginScope("Team Name = {0}", team.Name)) { if (pipeline.Process.Type != PipelineYamlProcessType) { return; } // Get contents of CODEOWNERS logger.LogInformation("Fetching CODEOWNERS file"); Uri repoUrl = pipeline.Repository.Url; if (repoUrl != null) { repoUrl = new Uri(Regex.Replace(repoUrl.ToString(), @"\.git$", String.Empty)); } else { logger.LogError("No repository url returned from pipeline. Repo id: {0}", pipeline.Repository.Id); return; } var codeOwnerEntries = await gitHubService.GetCodeownersFile(repoUrl); if (codeOwnerEntries == default) { logger.LogInformation("CODEOWNERS file not found, skipping sync"); return; } var process = pipeline.Process as YamlProcess; logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename); var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename); codeOwnerEntry.FilterOutNonUserAliases(); logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count); // Get set of team members in the CODEOWNERS file var codeownersDescriptors = new List <String>(); foreach (var contact in codeOwnerEntry.Owners) { if (!codeOwnerCache.ContainsKey(contact)) { // TODO: Better to have retry if no success on this call. var userPrincipal = gitHubToAADConverter.GetUserPrincipalNameFromGithub(contact); if (!string.IsNullOrEmpty(userPrincipal)) { codeOwnerCache[contact] = await service.GetDescriptorForPrincipal(userPrincipal); } else { logger.LogInformation("Cannot find the user principal for github {0}", contact); codeOwnerCache[contact] = null; } } codeownersDescriptors.Add(codeOwnerCache[contact]); } var codeownersSet = new HashSet <string>(codeownersDescriptors); // Get set of team members in the DevOps teams var teamMembers = await service.GetMembersAsync(team); var teamDescriptors = new List <String>(); foreach (var member in teamMembers) { if (!teamMemberCache.ContainsKey(member.Identity.Id)) { var teamMemberDescriptor = (await service.GetUserFromId(new Guid(member.Identity.Id))).SubjectDescriptor.ToString(); teamMemberCache[member.Identity.Id] = teamMemberDescriptor; } teamDescriptors.Add(teamMemberCache[member.Identity.Id]); } var teamSet = new HashSet <string>(teamDescriptors); var contactsToRemove = teamSet.Except(codeownersSet); var contactsToAdd = codeownersSet.Except(teamSet); foreach (var descriptor in contactsToRemove) { if (persistChanges && descriptor != null) { var teamDescriptor = await service.GetDescriptorAsync(team.Id); logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor); await service.RemoveMember(teamDescriptor, descriptor); } } foreach (var descriptor in contactsToAdd) { if (persistChanges && descriptor != null) { var teamDescriptor = await service.GetDescriptorAsync(team.Id); logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor); await service.AddToTeamAsync(teamDescriptor, descriptor); } } } }