public async Task ConfigureNotifications(
string projectName,
string projectPath,
GitHubToAADConverter gitHubToAADConverter,
bool persistChanges = true,
PipelineSelectionStrategy strategy = PipelineSelectionStrategy.Scheduled)
{
var pipelines = await GetPipelinesAsync(projectName, projectPath, strategy);
var teams = await service.GetAllTeamsAsync(projectName);
foreach (var pipeline in pipelines)
{
using (logger.BeginScope("Evaluate Pipeline: Name = {0}, Path = {1}, Id = {2}", pipeline.Name, pipeline.Path, pipeline.Id))
{
var parentTeam = await EnsureTeamExists(pipeline, TeamPurpose.ParentNotificationTeam, teams, gitHubToAADConverter, persistChanges);
var childTeam = await EnsureTeamExists(pipeline, TeamPurpose.SynchronizedNotificationTeam, teams, gitHubToAADConverter, persistChanges);
if (!persistChanges && (parentTeam == default || childTeam == default))
{
// Skip team nesting and notification work if
logger.LogInformation("Skipping Teams and Notifications because parent or child team does not exist");
continue;
}
await EnsureSynchronizedNotificationTeamIsChild(parentTeam, childTeam, persistChanges);
}
}
}
/// <summary>
/// Create notification groups for failures in scheduled builds
/// </summary>
/// <param name="organization">Azure DevOps Organization</param>
/// <param name="project">Name of the DevOps project</param>
/// <param name="pathPrefix">Path prefix to include pipelines (e.g. "\net")</param>
/// <param name="tokenVariableName">Environment variable token name (e.g. "SYSTEM_ACCESSTOKEN")</param>
/// <param name="aadAppIdVar">AAD App ID environment variable name (OpensourceAPI access)</param>
/// <param name="aadAppSecretVar">AAD App Secret environment variable name (OpensourceAPI access)</param>
/// <param name="aadTenantVar">AAD Tenant environment variable name (OpensourceAPI access)</param>
/// <param name="selectionStrategy">Pipeline selection strategy</param>
/// <param name="dryRun">Prints changes but does not alter any objects</param>
/// <returns></returns>
static async Task Main(
string organization,
string project,
string pathPrefix,
string tokenVariableName,
string aadAppIdVar,
string aadAppSecretVar,
string aadTenantVar,
PipelineSelectionStrategy selectionStrategy = PipelineSelectionStrategy.Scheduled,
bool dryRun = false)
{
var devOpsToken = Environment.GetEnvironmentVariable(tokenVariableName);
var devOpsCreds = new VssBasicCredential("nobody", devOpsToken);
var devOpsConnection = new VssConnection(new Uri($"https://dev.azure.com/{organization}/"), devOpsCreds);
#pragma warning disable CS0618 // Type or member is obsolete
var loggerFactory = LoggerFactory.Create(builder =>
{
builder.AddConsole(config => { config.IncludeScopes = true; });
});
#pragma warning restore CS0618 // Type or member is obsolete
var devOpsServiceLogger = loggerFactory.CreateLogger <AzureDevOpsService>();
var notificationConfiguratorLogger = loggerFactory.CreateLogger <NotificationConfigurator>();
var devOpsService = new AzureDevOpsService(devOpsConnection, devOpsServiceLogger);
var gitHubService = new GitHubService(loggerFactory.CreateLogger <GitHubService>());
var credential = new ClientSecretCredential(
Environment.GetEnvironmentVariable(aadTenantVar),
Environment.GetEnvironmentVariable(aadAppIdVar),
Environment.GetEnvironmentVariable(aadAppSecretVar));
var githubToAadResolver = new GitHubToAADConverter(
credential,
loggerFactory.CreateLogger <GitHubToAADConverter>()
);
var configurator = new NotificationConfigurator(devOpsService,
gitHubService, notificationConfiguratorLogger);
await configurator.ConfigureNotifications(
project,
pathPrefix,
githubToAadResolver,
persistChanges : !dryRun,
strategy : selectionStrategy);
}
/// <summary>
/// Synchronizes CODEOWNERS contacts to appropriate DevOps groups
/// </summary>
/// <param name="organization">Azure DevOps organization name</param>
/// <param name="project">Azure DevOps project name</param>
/// <param name="devOpsTokenVar">Personal Access Token environment variable name</param>
/// <param name="aadAppIdVar">AAD App ID environment variable name (OpensourceAPI access)</param>
/// <param name="aadAppSecretVar">AAD App Secret environment variable name (OpensourceAPI access)</param>
/// <param name="aadTenantVar">AAD Tenant environment variable name (OpensourceAPI access)</param>
/// <param name="pathPrefix">Azure DevOps path prefix (e.g. "\net")</param>
/// <param name="dryRun">Do not persist changes</param>
/// <returns></returns>
public static async Task Main(
string organization,
string project,
string devOpsTokenVar,
string aadAppIdVar,
string aadAppSecretVar,
string aadTenantVar,
string pathPrefix = "",
bool dryRun = false
)
{
#pragma warning disable CS0618 // Type or member is obsolete
using (var loggerFactory = LoggerFactory.Create(builder => {
builder.AddConsole(config => { config.IncludeScopes = true; });
}))
#pragma warning restore CS0618 // Type or member is obsolete
{
var devOpsService = AzureDevOpsService.CreateAzureDevOpsService(
Environment.GetEnvironmentVariable(devOpsTokenVar),
$"https://dev.azure.com/{organization}/",
loggerFactory.CreateLogger <AzureDevOpsService>()
);
var gitHubServiceLogger = loggerFactory.CreateLogger <GitHubService>();
var gitHubService = new GitHubService(gitHubServiceLogger);
var credential = new ClientSecretCredential(
Environment.GetEnvironmentVariable(aadTenantVar),
Environment.GetEnvironmentVariable(aadAppIdVar),
Environment.GetEnvironmentVariable(aadAppSecretVar));
var githubToAadResolver = new GitHubToAADConverter(
credential,
loggerFactory.CreateLogger <GitHubToAADConverter>()
);
var logger = loggerFactory.CreateLogger <Program>();
var pipelines = (await devOpsService.GetPipelinesAsync(project)).ToDictionary(p => p.Id);
var pipelineGroupTasks = (await devOpsService.GetAllTeamsAsync(project))
.Where(team =>
YamlHelper.Deserialize <TeamMetadata>(team.Description, swallowExceptions: true)?.Purpose == TeamPurpose.SynchronizedNotificationTeam
).Select(async team =>
{
BuildDefinition pipeline;
var pipelineId = YamlHelper.Deserialize <TeamMetadata>(team.Description).PipelineId;
if (!pipelines.ContainsKey(pipelineId))
{
pipeline = await devOpsService.GetPipelineAsync(project, pipelineId);
}
else
{
pipeline = pipelines[pipelineId];
}
if (pipeline == default)
{
logger.LogWarning($"Could not find pipeline with id {pipelineId} referenced by team {team.Name}.");
}
return(new
{
Pipeline = pipeline,
Team = team
});
});
var pipelineGroups = await Task.WhenAll(pipelineGroupTasks);
var filteredGroups = pipelineGroups.Where(group => group.Pipeline != default && group.Pipeline.Path.StartsWith(pathPrefix));
foreach (var group in filteredGroups)
{
using (logger.BeginScope("Team Name = {0}", group.Team.Name))
{
if (group.Pipeline.Process.Type != PipelineYamlProcessType)
{
continue;
}
// Get contents of CODEOWNERS
logger.LogInformation("Fetching CODEOWNERS file");
var managementUrl = new Uri(group.Pipeline.Repository.Properties["manageUrl"]);
var codeOwnerEntries = await gitHubService.GetCodeownersFile(managementUrl);
if (codeOwnerEntries == default)
{
logger.LogInformation("CODEOWNERS file not found, skipping sync");
continue;
}
var process = group.Pipeline.Process as YamlProcess;
logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename);
var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename);
codeOwnerEntry.FilterOutNonUserAliases();
logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count);
// Get set of team members in the CODEOWNERS file
var codeownerPrincipals = codeOwnerEntry.Owners
.Select(contact => githubToAadResolver.GetUserPrincipalNameFromGithub(contact));
var codeownersDescriptorsTasks = codeownerPrincipals
.Where(userPrincipal => !string.IsNullOrEmpty(userPrincipal))
.Select(userPrincipal => devOpsService.GetDescriptorForPrincipal(userPrincipal));
var codeownersDescriptors = await Task.WhenAll(codeownersDescriptorsTasks);
var codeownersSet = new HashSet <string>(codeownersDescriptors);
// Get set of existing team members
var teamMembers = await devOpsService.GetMembersAsync(group.Team);
var teamContactTasks = teamMembers
.Select(async member => await devOpsService.GetUserFromId(new Guid(member.Identity.Id)));
var teamContacts = await Task.WhenAll(teamContactTasks);
var teamDescriptors = teamContacts.Select(contact => contact.SubjectDescriptor.ToString());
var teamSet = new HashSet <string>(teamDescriptors);
// Synchronize contacts
var contactsToRemove = teamSet.Except(codeownersSet);
var contactsToAdd = codeownersSet.Except(teamSet);
var teamDescriptor = await devOpsService.GetDescriptorAsync(group.Team.Id);
foreach (var descriptor in contactsToRemove)
{
logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
if (!dryRun)
{
await devOpsService.RemoveMember(teamDescriptor, descriptor);
}
}
foreach (var descriptor in contactsToAdd)
{
logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
if (!dryRun)
{
await devOpsService.AddToTeamAsync(teamDescriptor, descriptor);
}
}
}
}
}
}
private async Task <WebApiTeam> EnsureTeamExists(
BuildDefinition pipeline,
TeamPurpose purpose,
IEnumerable <WebApiTeam> teams,
GitHubToAADConverter gitHubToAADConverter,
bool persistChanges)
{
string teamName = $"{pipeline.Id} ";
if (purpose == TeamPurpose.ParentNotificationTeam)
{
// Ensure team name fits within maximum 64 character limit
// https://docs.microsoft.com/en-us/azure/devops/organizations/settings/naming-restrictions?view=azure-devops#teams
string fullTeamName = teamName + $"{pipeline.Name}";
teamName = StringHelper.MaxLength(fullTeamName, MaxTeamNameLength);
if (fullTeamName.Length > teamName.Length)
{
logger.LogWarning($"Notification team name (length {fullTeamName.Length}) will be truncated to {teamName}");
}
}
else if (purpose == TeamPurpose.SynchronizedNotificationTeam)
{
teamName += $"Code owners sync notifications";
}
bool updateMetadataAndName = false;
var result = teams.FirstOrDefault(
team =>
{
// Swallowing exceptions because parse errors on
// free form text fields which might be non-yaml text
// are not exceptional
var metadata = YamlHelper.Deserialize <TeamMetadata>(team.Description, swallowExceptions: true);
bool metadataMatches = (metadata?.PipelineId == pipeline.Id && metadata?.Purpose == purpose);
bool nameMatches = (team.Name == teamName);
if (metadataMatches && nameMatches)
{
return(true);
}
if (metadataMatches)
{
logger.LogInformation("Found team with matching pipeline id {0} but different name '{1}', expected '{2}'. Purpose = '{3}'", metadata?.PipelineId, team.Name, teamName, metadata?.Purpose);
updateMetadataAndName = true;
return(true);
}
if (nameMatches)
{
logger.LogInformation("Found team with matching name {0} but different pipeline id {1}, expected {2}. Purpose = '{3}'", team.Name, metadata?.PipelineId, pipeline.Id, metadata?.Purpose);
updateMetadataAndName = true;
return(true);
}
return(false);
});
if (result == default)
{
logger.LogInformation("Team Not Found purpose = {0}", purpose);
var teamMetadata = new TeamMetadata
{
PipelineId = pipeline.Id,
Purpose = purpose,
PipelineName = pipeline.Name,
};
var newTeam = new WebApiTeam
{
Description = YamlHelper.Serialize(teamMetadata),
Name = teamName
};
logger.LogInformation("Create Team for Pipeline PipelineId = {0} Purpose = {1} Name = '{2}'", pipeline.Id, purpose, teamName);
if (persistChanges)
{
result = await service.CreateTeamForProjectAsync(pipeline.Project.Id.ToString(), newTeam);
if (purpose == TeamPurpose.ParentNotificationTeam)
{
await EnsureScheduledBuildFailSubscriptionExists(pipeline, result, true);
}
}
}
else if (updateMetadataAndName)
{
var teamMetadata = new TeamMetadata
{
PipelineId = pipeline.Id,
Purpose = purpose,
};
result.Description = YamlHelper.Serialize(teamMetadata);
result.Name = teamName;
logger.LogInformation("Update Team for Pipeline PipelineId = {0} Purpose = {1} Name = '{2}'", pipeline.Id, purpose, teamName);
if (persistChanges)
{
result = await service.UpdateTeamForProjectAsync(pipeline.Project.Id.ToString(), result);
if (purpose == TeamPurpose.ParentNotificationTeam)
{
await EnsureScheduledBuildFailSubscriptionExists(pipeline, result, true);
}
}
}
if (purpose == TeamPurpose.SynchronizedNotificationTeam)
{
await SyncTeamWithCodeOwnerFile(pipeline, result, gitHubToAADConverter, gitHubService, persistChanges);
}
return(result);
}
private async Task SyncTeamWithCodeOwnerFile(BuildDefinition pipeline, WebApiTeam team, GitHubToAADConverter gitHubToAADConverter, GitHubService gitHubService, bool persistChanges)
{
using (logger.BeginScope("Team Name = {0}", team.Name))
{
if (pipeline.Process.Type != PipelineYamlProcessType)
{
return;
}
// Get contents of CODEOWNERS
logger.LogInformation("Fetching CODEOWNERS file");
Uri repoUrl = pipeline.Repository.Url;
if (repoUrl != null)
{
repoUrl = new Uri(Regex.Replace(repoUrl.ToString(), @"\.git$", String.Empty));
}
else
{
logger.LogError("No repository url returned from pipeline. Repo id: {0}", pipeline.Repository.Id);
return;
}
var codeOwnerEntries = await gitHubService.GetCodeownersFile(repoUrl);
if (codeOwnerEntries == default)
{
logger.LogInformation("CODEOWNERS file not found, skipping sync");
return;
}
var process = pipeline.Process as YamlProcess;
logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename);
var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename);
codeOwnerEntry.FilterOutNonUserAliases();
logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count);
// Get set of team members in the CODEOWNERS file
var codeownersDescriptors = new List <String>();
foreach (var contact in codeOwnerEntry.Owners)
{
if (!codeOwnerCache.ContainsKey(contact))
{
// TODO: Better to have retry if no success on this call.
var userPrincipal = gitHubToAADConverter.GetUserPrincipalNameFromGithub(contact);
if (!string.IsNullOrEmpty(userPrincipal))
{
codeOwnerCache[contact] = await service.GetDescriptorForPrincipal(userPrincipal);
}
else
{
logger.LogInformation("Cannot find the user principal for github {0}", contact);
codeOwnerCache[contact] = null;
}
}
codeownersDescriptors.Add(codeOwnerCache[contact]);
}
var codeownersSet = new HashSet <string>(codeownersDescriptors);
// Get set of team members in the DevOps teams
var teamMembers = await service.GetMembersAsync(team);
var teamDescriptors = new List <String>();
foreach (var member in teamMembers)
{
if (!teamMemberCache.ContainsKey(member.Identity.Id))
{
var teamMemberDescriptor = (await service.GetUserFromId(new Guid(member.Identity.Id))).SubjectDescriptor.ToString();
teamMemberCache[member.Identity.Id] = teamMemberDescriptor;
}
teamDescriptors.Add(teamMemberCache[member.Identity.Id]);
}
var teamSet = new HashSet <string>(teamDescriptors);
var contactsToRemove = teamSet.Except(codeownersSet);
var contactsToAdd = codeownersSet.Except(teamSet);
foreach (var descriptor in contactsToRemove)
{
if (persistChanges && descriptor != null)
{
var teamDescriptor = await service.GetDescriptorAsync(team.Id);
logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
await service.RemoveMember(teamDescriptor, descriptor);
}
}
foreach (var descriptor in contactsToAdd)
{
if (persistChanges && descriptor != null)
{
var teamDescriptor = await service.GetDescriptorAsync(team.Id);
logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
await service.AddToTeamAsync(teamDescriptor, descriptor);
}
}
}
}
