public override string GetPayload()
{
List <CodeOwnerEntry> entries = CodeOwnersFile.ParseFile(_codeownersFile);
foreach (CodeOwnerEntry entry in entries)
{
// If we have labels for the specific codeowners entry, add that to the triage list
if (entry.ServiceLabels.Any())
{
// Remove the '@' from the owners handle
IEnumerable <string> mentionees = entry.Owners.Select(x => x.Replace("@", "").Trim());
//add the service
AddService(entry.ServiceLabels, mentionees);
}
}
Colorizer.WriteLine("Found [Yellow!{0}] service routes.", RouteCount);
foreach (TriageConfig triage in _triageConfig)
{
Colorizer.WriteLine("Labels:[Yellow!{0}], Owners:[Yellow!{1}]", string.Join(',', triage.Labels), string.Join(',', triage.Mentionee));
}
return(s_template
.Replace("###repo###", GetTaskId())
.Replace("###labelsAndMentions###", string.Join(",", _triageConfig)));
}
/// <summary>
/// Retrieves codeowners information for specific section of the repo
/// </summary>
/// <param name="codeOwnerFilePath">The path of CODEOWNERS file in repo</param>
/// <param name="targetDirectory">The directory whose information is to be retrieved</param>
/// <param name="filterOutNonUserAliases">The option to filter out code owner team alias.</param>
/// <returns>Exit code</returns>
public static int Main(
string codeOwnerFilePath,
string targetDirectory,
bool filterOutNonUserAliases = false
)
{
var target = targetDirectory.ToLower().Trim();
try {
var codeOwnerEntry = CodeOwnersFile.ParseAndFindOwnersForClosestMatch(codeOwnerFilePath, target);
if (filterOutNonUserAliases)
{
codeOwnerEntry.FilterOutNonUserAliases();
}
var codeOwnerJson = JsonSerializer.Serialize <CodeOwnerEntry>(codeOwnerEntry, new JsonSerializerOptions {
WriteIndented = true
});
Console.WriteLine(codeOwnerJson);
return(0);
}
catch (Exception e) {
Console.Error.WriteLine(e.Message);
return(1);
}
}
public void ValidateOwnersLines(TestAndExpected entry)
{
// create the content
string content = $"{entry.LabelsLine}\r\n{entry.PathAndOwners}";
CodeOwnerEntry coe = CodeOwnersFile.ParseContent(content).First();
Assert.AreEqual(entry.ExpectedLabels, coe.PRLabels);
Assert.AreEqual(entry.ExpectedOwners, coe.Owners);
Assert.AreEqual(entry.ExpectedPath, coe.PathExpression);
}
public void ParseInvalidEntry()
{
// no path and owners
var entry = new TestAndExpected("", "# PRLabel: %label1 %label2", string.Empty, new string[] { }, new string[] { "label1", "label2" });
// create the content
string content = $"{entry.LabelsLine}\r\n{entry.PathAndOwners}";
CodeOwnerEntry coe = CodeOwnersFile.ParseContent(content).FirstOrDefault();
Assert.IsNull(coe);
}
/// <summary>
/// Looks for CODEOWNERS in the main branch of the given repo URL
/// </summary>
/// <param name="repoUrl"></param>
/// <returns></returns>
private async Task <List <CodeOwnerEntry> > GetCodeownersFileImpl(Uri repoUrl)
{
// Gets the repo path from the URL
var relevantPathParts = repoUrl.Segments.Skip(1).Take(2);
var repoPath = string.Join("", relevantPathParts);
var codeOwnersUrl = $"https://raw.githubusercontent.com/{repoPath}/main/.github/CODEOWNERS";
var result = await httpClient.GetAsync(codeOwnersUrl);
if (result.IsSuccessStatusCode)
{
logger.LogInformation("Retrieved CODEOWNERS file URL = {0}", codeOwnersUrl);
return(CodeOwnersFile.ParseContent(await result.Content.ReadAsStringAsync()));
}
logger.LogWarning("Could not retrieve CODEOWNERS file URL = {0} ResponseCode = {1}", codeOwnersUrl, result.StatusCode);
return(default);
internal override void ReadConfigurationFromFile(string configurationFile)
{
List <CodeOwnerEntry> entries = CodeOwnersFile.ParseFile(configurationFile);
foreach (CodeOwnerEntry entry in entries)
{
// If we have labels for the specific codeowners entry, add that to the triage list
if (entry.ServiceLabels.Any())
{
// Remove the '@' from the owners handle
IEnumerable <string> mentionees = entry.Owners.Select(x => x.Replace("@", "").Trim());
//add the service
AddRoute(entry.ServiceLabels, mentionees);
}
}
}
internal override void ReadConfigurationFromFile(string configurationFile)
{
List <CodeOwnerEntry> entries = CodeOwnersFile.ParseFile(configurationFile);
// Filter our the list of entries that we want to create.
for (int i = 0; i < entries.Count; i++)
{
// Entries with wildcards are not yet supported
if (entries[i].ContainsWildcard)
{
// log a warning there
if (entries[i].PRLabels.Any())
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' contains a wildcard and a label '[Magenta!{1}]' which is not supported!", entries[i].PathExpression, string.Join(',', entries[i].PRLabels));
}
continue; //TODO: regex expressions are not yet supported
}
if (entries[i].PathExpression.IndexOf(CodeOwnerEntry.MissingFolder, StringComparison.OrdinalIgnoreCase) != -1)
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' is marked with the non-existing path marker.", entries[i].PathExpression);
continue;
}
// Entries with more than one label are not yet supported.
if (entries[i].PRLabels.Count > 1)
{
Colorizer.WriteLine("[Yellow!Warning]: Multiple labels for the same path '[Cyan!{0}]' are not yet supported", entries[i].PathExpression);
continue;
}
if (entries[i].PRLabels.Count == 0)
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' does not contain a label.", entries[i].PathExpression, string.Join(',', entries[i].PRLabels));
continue;
}
AddEntry(entries[i].PathExpression, entries[i].PRLabels.First());
}
}
public void ValidateCodeOwnersContent()
{
string content = @"#Comment
# ServiceLabel: %F1 %Service Attention
# PRLabel: %F1
/folder1/ @user1
# ServiceLabel: %F2 %Service Attention
# PRLabel: %F2
/folder2/ @user2
# ServiceLabel: %Service Attention %F3
/folder3/ @user3 @user1
# PRLabel: %F4 %Service Attention
/folder4/ @user4
/folder5/ @user5
# ServiceLabel: %MyService
#/<NotInRepo>/ @user6
# ServiceLabel: %MyService
# CommentedLine @user7
/folder6 @user7
# ServiceLabel: %MyService
/folder8 @user6 #This has comment at the end
";
List <CodeOwnerEntry> entries = CodeOwnersFile.ParseContent(content);
Assert.AreEqual(8, entries.Count);
Assert.AreEqual("F1", entries[0].PRLabels[0]);
Assert.AreEqual("F1", entries[0].ServiceLabels[0]);
Assert.AreEqual("Service Attention", entries[0].ServiceLabels[1]);
Assert.AreEqual("F2", entries[1].PRLabels[0]);
Assert.AreEqual("F2", entries[1].ServiceLabels[0]);
Assert.AreEqual("Service Attention", entries[1].ServiceLabels[1]);
Assert.AreEqual("/folder2/", entries[1].PathExpression);
Assert.AreEqual("Service Attention", entries[2].ServiceLabels[0]);
Assert.AreEqual(0, entries[2].PRLabels.Count);
Assert.AreEqual("F3", entries[2].ServiceLabels[1]);
Assert.AreEqual("F4", entries[3].PRLabels[0]);
Assert.AreEqual(0, entries[3].ServiceLabels.Count);
Assert.AreEqual("Service Attention", entries[3].PRLabels[1]);
Assert.AreEqual(0, entries[4].ServiceLabels.Count);
Assert.AreEqual(0, entries[4].PRLabels.Count);
Assert.AreEqual("/folder5/", entries[4].PathExpression);
Assert.AreEqual(1, entries[5].ServiceLabels.Count);
Assert.AreEqual(0, entries[5].PRLabels.Count);
Assert.AreEqual("#/<NotInRepo>/", entries[5].PathExpression);
Assert.AreEqual(1, entries[6].ServiceLabels.Count);
Assert.AreEqual(0, entries[6].PRLabels.Count);
Assert.AreEqual("/folder6", entries[6].PathExpression);
Assert.AreEqual(1, entries[7].ServiceLabels.Count);
Assert.AreEqual(0, entries[7].PRLabels.Count);
Assert.AreEqual("/folder8", entries[7].PathExpression);
Assert.AreEqual("user6", entries[7].Owners[0]);
}
/// <summary>
/// Synchronizes CODEOWNERS contacts to appropriate DevOps groups
/// </summary>
/// <param name="organization">Azure DevOps organization name</param>
/// <param name="project">Azure DevOps project name</param>
/// <param name="devOpsTokenVar">Personal Access Token environment variable name</param>
/// <param name="aadAppIdVar">AAD App ID environment variable name (OpensourceAPI access)</param>
/// <param name="aadAppSecretVar">AAD App Secret environment variable name (OpensourceAPI access)</param>
/// <param name="aadTenantVar">AAD Tenant environment variable name (OpensourceAPI access)</param>
/// <param name="pathPrefix">Azure DevOps path prefix (e.g. "\net")</param>
/// <param name="dryRun">Do not persist changes</param>
/// <returns></returns>
public static async Task Main(
string organization,
string project,
string devOpsTokenVar,
string aadAppIdVar,
string aadAppSecretVar,
string aadTenantVar,
string pathPrefix = "",
bool dryRun = false
)
{
#pragma warning disable CS0618 // Type or member is obsolete
using (var loggerFactory = LoggerFactory.Create(builder => {
builder.AddConsole(config => { config.IncludeScopes = true; });
}))
#pragma warning restore CS0618 // Type or member is obsolete
{
var devOpsService = AzureDevOpsService.CreateAzureDevOpsService(
Environment.GetEnvironmentVariable(devOpsTokenVar),
$"https://dev.azure.com/{organization}/",
loggerFactory.CreateLogger <AzureDevOpsService>()
);
var gitHubServiceLogger = loggerFactory.CreateLogger <GitHubService>();
var gitHubService = new GitHubService(gitHubServiceLogger);
var credential = new ClientSecretCredential(
Environment.GetEnvironmentVariable(aadTenantVar),
Environment.GetEnvironmentVariable(aadAppIdVar),
Environment.GetEnvironmentVariable(aadAppSecretVar));
var githubToAadResolver = new GitHubToAADConverter(
credential,
loggerFactory.CreateLogger <GitHubToAADConverter>()
);
var logger = loggerFactory.CreateLogger <Program>();
var pipelines = (await devOpsService.GetPipelinesAsync(project)).ToDictionary(p => p.Id);
var pipelineGroupTasks = (await devOpsService.GetAllTeamsAsync(project))
.Where(team =>
YamlHelper.Deserialize <TeamMetadata>(team.Description, swallowExceptions: true)?.Purpose == TeamPurpose.SynchronizedNotificationTeam
).Select(async team =>
{
BuildDefinition pipeline;
var pipelineId = YamlHelper.Deserialize <TeamMetadata>(team.Description).PipelineId;
if (!pipelines.ContainsKey(pipelineId))
{
pipeline = await devOpsService.GetPipelineAsync(project, pipelineId);
}
else
{
pipeline = pipelines[pipelineId];
}
if (pipeline == default)
{
logger.LogWarning($"Could not find pipeline with id {pipelineId} referenced by team {team.Name}.");
}
return(new
{
Pipeline = pipeline,
Team = team
});
});
var pipelineGroups = await Task.WhenAll(pipelineGroupTasks);
var filteredGroups = pipelineGroups.Where(group => group.Pipeline != default && group.Pipeline.Path.StartsWith(pathPrefix));
foreach (var group in filteredGroups)
{
using (logger.BeginScope("Team Name = {0}", group.Team.Name))
{
if (group.Pipeline.Process.Type != PipelineYamlProcessType)
{
continue;
}
// Get contents of CODEOWNERS
logger.LogInformation("Fetching CODEOWNERS file");
var managementUrl = new Uri(group.Pipeline.Repository.Properties["manageUrl"]);
var codeOwnerEntries = await gitHubService.GetCodeownersFile(managementUrl);
if (codeOwnerEntries == default)
{
logger.LogInformation("CODEOWNERS file not found, skipping sync");
continue;
}
var process = group.Pipeline.Process as YamlProcess;
logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename);
var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename);
codeOwnerEntry.FilterOutNonUserAliases();
logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count);
// Get set of team members in the CODEOWNERS file
var codeownerPrincipals = codeOwnerEntry.Owners
.Select(contact => githubToAadResolver.GetUserPrincipalNameFromGithub(contact));
var codeownersDescriptorsTasks = codeownerPrincipals
.Where(userPrincipal => !string.IsNullOrEmpty(userPrincipal))
.Select(userPrincipal => devOpsService.GetDescriptorForPrincipal(userPrincipal));
var codeownersDescriptors = await Task.WhenAll(codeownersDescriptorsTasks);
var codeownersSet = new HashSet <string>(codeownersDescriptors);
// Get set of existing team members
var teamMembers = await devOpsService.GetMembersAsync(group.Team);
var teamContactTasks = teamMembers
.Select(async member => await devOpsService.GetUserFromId(new Guid(member.Identity.Id)));
var teamContacts = await Task.WhenAll(teamContactTasks);
var teamDescriptors = teamContacts.Select(contact => contact.SubjectDescriptor.ToString());
var teamSet = new HashSet <string>(teamDescriptors);
// Synchronize contacts
var contactsToRemove = teamSet.Except(codeownersSet);
var contactsToAdd = codeownersSet.Except(teamSet);
var teamDescriptor = await devOpsService.GetDescriptorAsync(group.Team.Id);
foreach (var descriptor in contactsToRemove)
{
logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
if (!dryRun)
{
await devOpsService.RemoveMember(teamDescriptor, descriptor);
}
}
foreach (var descriptor in contactsToAdd)
{
logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
if (!dryRun)
{
await devOpsService.AddToTeamAsync(teamDescriptor, descriptor);
}
}
}
}
}
}
private async Task SyncTeamWithCodeOwnerFile(BuildDefinition pipeline, WebApiTeam team, GitHubToAADConverter gitHubToAADConverter, GitHubService gitHubService, bool persistChanges)
{
using (logger.BeginScope("Team Name = {0}", team.Name))
{
if (pipeline.Process.Type != PipelineYamlProcessType)
{
return;
}
// Get contents of CODEOWNERS
logger.LogInformation("Fetching CODEOWNERS file");
Uri repoUrl = pipeline.Repository.Url;
if (repoUrl != null)
{
repoUrl = new Uri(Regex.Replace(repoUrl.ToString(), @"\.git$", String.Empty));
}
else
{
logger.LogError("No repository url returned from pipeline. Repo id: {0}", pipeline.Repository.Id);
return;
}
var codeOwnerEntries = await gitHubService.GetCodeownersFile(repoUrl);
if (codeOwnerEntries == default)
{
logger.LogInformation("CODEOWNERS file not found, skipping sync");
return;
}
var process = pipeline.Process as YamlProcess;
logger.LogInformation("Searching CODEOWNERS for matching path for {0}", process.YamlFilename);
var codeOwnerEntry = CodeOwnersFile.FindOwnersForClosestMatch(codeOwnerEntries, process.YamlFilename);
codeOwnerEntry.FilterOutNonUserAliases();
logger.LogInformation("Matching Contacts Path = {0}, NumContacts = {1}", process.YamlFilename, codeOwnerEntry.Owners.Count);
// Get set of team members in the CODEOWNERS file
var codeownersDescriptors = new List <String>();
foreach (var contact in codeOwnerEntry.Owners)
{
if (!codeOwnerCache.ContainsKey(contact))
{
// TODO: Better to have retry if no success on this call.
var userPrincipal = gitHubToAADConverter.GetUserPrincipalNameFromGithub(contact);
if (!string.IsNullOrEmpty(userPrincipal))
{
codeOwnerCache[contact] = await service.GetDescriptorForPrincipal(userPrincipal);
}
else
{
logger.LogInformation("Cannot find the user principal for github {0}", contact);
codeOwnerCache[contact] = null;
}
}
codeownersDescriptors.Add(codeOwnerCache[contact]);
}
var codeownersSet = new HashSet <string>(codeownersDescriptors);
// Get set of team members in the DevOps teams
var teamMembers = await service.GetMembersAsync(team);
var teamDescriptors = new List <String>();
foreach (var member in teamMembers)
{
if (!teamMemberCache.ContainsKey(member.Identity.Id))
{
var teamMemberDescriptor = (await service.GetUserFromId(new Guid(member.Identity.Id))).SubjectDescriptor.ToString();
teamMemberCache[member.Identity.Id] = teamMemberDescriptor;
}
teamDescriptors.Add(teamMemberCache[member.Identity.Id]);
}
var teamSet = new HashSet <string>(teamDescriptors);
var contactsToRemove = teamSet.Except(codeownersSet);
var contactsToAdd = codeownersSet.Except(teamSet);
foreach (var descriptor in contactsToRemove)
{
if (persistChanges && descriptor != null)
{
var teamDescriptor = await service.GetDescriptorAsync(team.Id);
logger.LogInformation("Delete Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
await service.RemoveMember(teamDescriptor, descriptor);
}
}
foreach (var descriptor in contactsToAdd)
{
if (persistChanges && descriptor != null)
{
var teamDescriptor = await service.GetDescriptorAsync(team.Id);
logger.LogInformation("Add Contact TeamDescriptor = {0}, ContactDescriptor = {1}", teamDescriptor, descriptor);
await service.AddToTeamAsync(teamDescriptor, descriptor);
}
}
}
}
public override string GetPayload()
{
List <CodeOwnerEntry> entries = CodeOwnersFile.ParseFile(_codeownersFile);
StringBuilder configPayload = new StringBuilder();
List <CodeOwnerEntry> entriesToCreate = new List <CodeOwnerEntry>();
// Filter our the list of entries that we want to create.
for (int i = 0; i < entries.Count; i++)
{
// Entries with wildcards are not yet supported
if (entries[i].ContainsWildcard)
{
// log a warning there
if (entries[i].PRLabels.Any())
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' contains a wildcard and a label '[Magenta!{1}]' which is not supported!", entries[i].PathExpression, string.Join(',', entries[i].PRLabels));
}
continue; //TODO: regex expressions are not yet supported
}
if (entries[i].PathExpression.IndexOf(CodeOwnerEntry.MissingFolder, StringComparison.OrdinalIgnoreCase) != -1)
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' is marked with the non-existing path marker.", entries[i].PathExpression);
continue;
}
// Entries with more than one label are not yet supported.
if (entries[i].PRLabels.Count > 1)
{
Colorizer.WriteLine("[Yellow!Warning]: Multiple labels for the same path '[Cyan!{0}]' are not yet supported", entries[i].PathExpression);
continue;
}
if (entries[i].PRLabels.Count == 0)
{
Colorizer.WriteLine("[Yellow!Warning]: The path '[Cyan!{0}]' does not contain a label.", entries[i].PathExpression, string.Join(',', entries[i].PRLabels));
continue;
}
entriesToCreate.Add(entries[i]);
}
Colorizer.WriteLine("Found the following rules:");
// Create the payload.
foreach (var entry in entriesToCreate)
{
// get the payload
string entryPayload = ToConfigString(entry);
Colorizer.WriteLine("[Cyan!{0}] => [Magenta!{1}]", entry.PathExpression, entry.PRLabels.FirstOrDefault());
configPayload.Append(ToConfigString(entry));
configPayload.Append(',');
}
// remove the trailing ','
configPayload.Remove(configPayload.Length - 1, 1);
// Log the set of paths we are creating.
// create the payload from the template
return(s_template
.Replace("###taskId###", GetTaskId())
.Replace("###labelConfig###", configPayload.ToString()));
}