/// <summary>
/// Get a tree item blob from github.
/// </summary>
/// <param name="path">Base path of final git file</param>
/// <param name="treeItem">Tree item to retrieve</param>
/// <param name="owner">Organization</param>
/// <param name="repo">Repository</param>
/// <returns>Git file with tree item contents.</returns>
private async Task <GitFile> GetGitItemImpl(string path, TreeItem treeItem, string owner, string repo)
{
Octokit.Blob blob = await ExponentialRetry.RetryAsync(
async() => await Client.Git.Blob.Get(owner, repo, treeItem.Sha),
ex => _logger.LogError(ex, $"Failed to get blob at sha {treeItem.Sha}"),
ex => ex is ApiException apiex && apiex.StatusCode >= HttpStatusCode.InternalServerError);
ContentEncoding encoding;
switch (blob.Encoding.Value)
{
case EncodingType.Base64:
encoding = ContentEncoding.Base64;
break;
case EncodingType.Utf8:
encoding = ContentEncoding.Utf8;
break;
default:
throw new NotImplementedException($"Unknown github encoding type {blob.Encoding.StringValue}");
}
GitFile newFile = new GitFile(
path + "/" + treeItem.Path,
blob.Content,
encoding,
treeItem.Mode);
return(newFile);
}
public async Task AddDependencyToGlobalJson(
string repo,
string branch,
string parentField,
string dependencyName,
string version)
{
JToken versionProperty = new JProperty(dependencyName, version);
JObject globalJson = await ReadGlobalJsonAsync(repo, branch);
JToken parent = globalJson[parentField];
if (parent != null)
{
parent.Last.AddAfterSelf(versionProperty);
}
else
{
globalJson.Add(new JProperty(parentField, new JObject(versionProperty)));
}
var file = new GitFile(VersionFiles.GlobalJson, globalJson);
await _gitClient.PushFilesAsync(new List <GitFile> {
file
}, repo, branch, $"Add {dependencyName} to " +
$"'{VersionFiles.GlobalJson}'");
_logger.LogInformation(
$"Dependency '{dependencyName}' with version '{version}' successfully added to global.json");
}
public async Task GetCommitMapForPathAsync(string repoUri, string branch, string assetsProducedInCommit, List <GitFile> files, string pullRequestBaseBranch, string path = "eng/common/")
{
if (path.EndsWith("/"))
{
path = path.Substring(0, path.Length - 1);
}
_logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}'");
string ownerAndRepo = GetOwnerAndRepoFromRepoUri(repoUri);
HttpResponseMessage response = await this.ExecuteGitCommand(HttpMethod.Get, $"repos/{ownerAndRepo}/contents/{path}?ref={assetsProducedInCommit}", _logger);
List <GitHubContent> contents = JsonConvert.DeserializeObject <List <GitHubContent> >(await response.Content.ReadAsStringAsync());
foreach (GitHubContent content in contents)
{
if (content.Type == GitHubContentType.File)
{
if (!GitFileManager.DependencyFiles.Contains(content.Path))
{
string fileContent = await GetFileContentsAsync(ownerAndRepo, content.Path);
GitFile gitCommit = new GitFile(content.Path, fileContent);
files.Add(gitCommit);
}
}
else
{
await GetCommitMapForPathAsync(repoUri, branch, assetsProducedInCommit, files, pullRequestBaseBranch, content.Path);
}
}
_logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}' succeeded!");
}
/// <summary>
/// Get a tree item blob from github, using the cache if it exists.
/// </summary>
/// <param name="path">Base path of final git file</param>
/// <param name="treeItem">Tree item to retrieve</param>
/// <param name="owner">Organization</param>
/// <param name="repo">Repository</param>
/// <returns>Git file with tree item contents.</returns>
public async Task <GitFile> GetGitTreeItem(string path, TreeItem treeItem, string owner, string repo)
{
// If we have a cache available here, attempt to get the value in the cache
// before making the request. Generally, we are requesting the same files for each
// arcade update sent to each repository daily per subscription. This is inefficient, as it means we
// request each file N times, where N is the number of subscriptions. The number of files (M),
// is non-trivial, so reducing N*M to M is vast improvement.
// Use a combination of (treeItem.Path, treeItem.Sha as the key) as items with identical contents but
// different paths will have the same SHA. I think it is overkill to hash the repo and owner into
// the key.
if (Cache != null)
{
return(await Cache.GetOrCreateAsync((treeItem.Path, treeItem.Sha), async (entry) =>
{
GitFile file = await GetGitItemImpl(path, treeItem, owner, repo);
// Set the size of the entry. The size is not computed by the caching system
// (it has no way to do so). There are two bytes per each character in a string.
// We do not really need to worry about the size of the GitFile class itself,
// just the variable length elements.
entry.Size = 2 * (file.Content.Length + file.FilePath.Length + file.Mode.Length);
return file;
}));
}
else
{
return(await GetGitItemImpl(path, treeItem, owner, repo));
}
}
public async Task AddDependencyToVersionDetailsAsync(
string repo,
string branch,
DependencyDetail dependency,
DependencyType dependencyType)
{
XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null);
XmlNode newDependency = versionDetails.CreateElement("Dependency");
XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name");
nameAttribute.Value = dependency.Name;
newDependency.Attributes.Append(nameAttribute);
XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version");
versionAttribute.Value = dependency.Version;
newDependency.Attributes.Append(versionAttribute);
XmlNode uri = versionDetails.CreateElement("Uri");
uri.InnerText = dependency.RepoUri;
newDependency.AppendChild(uri);
XmlNode sha = versionDetails.CreateElement("Sha");
sha.InnerText = dependency.Commit;
newDependency.AppendChild(sha);
XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies");
if (dependenciesNode == null)
{
dependenciesNode = versionDetails.CreateElement($"{dependencyType}Dependencies");
versionDetails.DocumentElement.AppendChild(dependenciesNode);
}
dependenciesNode.AppendChild(newDependency);
// TODO: This should not be done here. This should return some kind of generic file container to the caller,
// who will gather up all updates and then call the git client to write the files all at once:
// https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so
// it's okay for now.
var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails);
await _gitClient.PushFilesAsync(new List <GitFile> {
file
}, repo, branch, $"Add {dependency} to " +
$"'{VersionFiles.VersionDetailsXml}'");
_logger.LogInformation(
$"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
$"'{VersionFiles.VersionDetailsXml}'");
}
public async Task AddDependencyToVersionDetailsAsync(
string repo,
string branch,
DependencyDetail dependency)
{
XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null);
XmlNode newDependency = versionDetails.CreateElement(VersionFiles.DependencyElementName);
SetAttribute(versionDetails, newDependency, VersionFiles.NameAttributeName, dependency.Name);
SetAttribute(versionDetails, newDependency, VersionFiles.VersionAttributeName, dependency.Version);
// Only add the pinned attribute if the pinned option is set to true
if (dependency.Pinned)
{
SetAttribute(versionDetails, newDependency, VersionFiles.PinnedAttributeName, "True");
}
// Only add the coherent parent attribute if it is set
if (!string.IsNullOrEmpty(dependency.CoherentParentDependencyName))
{
SetAttribute(versionDetails, newDependency, VersionFiles.CoherentParentAttributeName, dependency.CoherentParentDependencyName);
}
SetElement(versionDetails, newDependency, VersionFiles.UriElementName, dependency.RepoUri);
SetElement(versionDetails, newDependency, VersionFiles.ShaElementName, dependency.Commit);
XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependency.Type}{VersionFiles.DependenciesElementName}");
if (dependenciesNode == null)
{
dependenciesNode = versionDetails.CreateElement($"{dependency.Type}{VersionFiles.DependenciesElementName}");
versionDetails.DocumentElement.AppendChild(dependenciesNode);
}
dependenciesNode.AppendChild(newDependency);
// TODO: This should not be done here. This should return some kind of generic file container to the caller,
// who will gather up all updates and then call the git client to write the files all at once:
// https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so
// it's okay for now.
var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails);
await _gitClient.CommitFilesAsync(new List <GitFile> {
file
}, repo, branch, $"Add {dependency} to " +
$"'{VersionFiles.VersionDetailsXml}'");
_logger.LogInformation(
$"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
$"'{VersionFiles.VersionDetailsXml}'");
}
public async Task AddDependencyToVersionProps(string filePath, DependencyDetail dependency)
{
XmlDocument versionProps = await ReadVersionPropsAsync(filePath, null);
string versionedName = dependency.Name.Replace(".", string.Empty);
versionedName = $"{versionedName}Version";
XmlNode versionNode = versionProps.DocumentElement.SelectNodes($"//PropertyGroup").Item(0);
XmlNode newDependency = versionProps.CreateElement(versionedName);
newDependency.InnerText = dependency.Version;
versionNode.AppendChild(newDependency);
GitFile file = new GitFile(filePath, versionProps);
File.WriteAllText(file.FilePath, file.Content);
_logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.props");
}
public async Task AddDependencyToGlobalJson(string filePath, string parentField, string dependencyName, string version)
{
JToken versionProperty = new JProperty(dependencyName, version);
JObject globalJson = await ReadGlobalJsonAsync(filePath, null);
JToken parent = globalJson[parentField];
if (parent != null)
{
parent.Last.AddAfterSelf(versionProperty);
}
else
{
globalJson.Add(new JProperty(parentField, new JObject(versionProperty)));
}
GitFile file = new GitFile(filePath, globalJson);
File.WriteAllText(file.FilePath, file.Content);
_logger.LogInformation($"Dependency '{dependencyName}' with version '{version}' successfully added to global.json");
}
public async Task AddDependencyToVersionDetails(string filePath, DependencyDetail dependency, DependencyType dependencyType)
{
XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(filePath, null);
XmlNode newDependency = versionDetails.CreateElement("Dependency");
XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name");
nameAttribute.Value = dependency.Name;
newDependency.Attributes.Append(nameAttribute);
XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version");
versionAttribute.Value = dependency.Version;
newDependency.Attributes.Append(versionAttribute);
XmlNode uri = versionDetails.CreateElement("Uri");
uri.InnerText = dependency.RepoUri;
newDependency.AppendChild(uri);
XmlNode sha = versionDetails.CreateElement("Sha");
sha.InnerText = dependency.Commit;
newDependency.AppendChild(sha);
XmlNode dependencies = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies");
dependencies.AppendChild(newDependency);
GitFile file = new GitFile(filePath, versionDetails);
File.WriteAllText(file.FilePath, file.Content);
_logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.Details.xml");
}
/// <summary>
/// Add a dependency to Versions.props. This has the form:
/// <!-- Package names -->
/// <PropertyGroup>
/// <MicrosoftDotNetApiCompatPackage>Microsoft.DotNet.ApiCompat</MicrosoftDotNetApiCompatPackage>
/// </PropertyGroup>
///
/// <!-- Package versions -->
/// <PropertyGroup>
/// <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.18478.5</MicrosoftDotNetApiCompatPackageVersion>
/// </PropertyGroup>
///
/// See https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md for more
/// information.
/// </summary>
/// <param name="repo">Path to Versions.props file</param>
/// <param name="dependency">Dependency information to add.</param>
/// <returns>Async task.</returns>
public async Task AddDependencyToVersionsPropsAsync(string repo, string branch, DependencyDetail dependency)
{
XmlDocument versionProps = await ReadVersionPropsAsync(repo, null);
string documentNamespaceUri = versionProps.DocumentElement.NamespaceURI;
string packageNameElementName = VersionFiles.GetVersionPropsPackageElementName(dependency.Name);
string packageVersionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(dependency.Name);
string packageVersionAlternateElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName(
dependency.Name);
// Select elements by local name, since the Project (DocumentElement) element often has a default
// xmlns set.
XmlNodeList propertyGroupNodes = versionProps.DocumentElement.SelectNodes($"//*[local-name()='PropertyGroup']");
XmlNode newPackageNameElement = versionProps.CreateElement(packageNameElementName, documentNamespaceUri);
newPackageNameElement.InnerText = dependency.Name;
bool addedPackageVersionElement = false;
bool addedPackageNameElement = false;
// There can be more than one property group. Find the appropriate one containing an existing element of
// the same type, and add it to the parent.
foreach (XmlNode propertyGroupNode in propertyGroupNodes)
{
if (propertyGroupNode.HasChildNodes)
{
foreach (XmlNode propertyNode in propertyGroupNode.ChildNodes)
{
if (!addedPackageVersionElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsVersionElementSuffix))
{
XmlNode newPackageVersionElement = versionProps.CreateElement(
packageVersionElementName,
documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
propertyGroupNode.AppendChild(newPackageVersionElement);
addedPackageVersionElement = true;
break;
}
// Test for alternate suffixes. This will eventually be replaced by repo-level configuration:
// https://github.com/dotnet/arcade/issues/1095
else if (!addedPackageVersionElement && propertyNode.Name.EndsWith(
VersionFiles.VersionPropsAlternateVersionElementSuffix))
{
XmlNode newPackageVersionElement = versionProps.CreateElement(
packageVersionAlternateElementName,
documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
propertyGroupNode.AppendChild(newPackageVersionElement);
addedPackageVersionElement = true;
break;
}
else if (!addedPackageNameElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsPackageElementSuffix))
{
propertyGroupNode.AppendChild(newPackageNameElement);
addedPackageNameElement = true;
break;
}
}
if (addedPackageVersionElement && addedPackageNameElement)
{
break;
}
}
}
// Add the property groups if none were present
if (!addedPackageVersionElement)
{
// If the repository doesn't have any package version element, then
// use the non-alternate element name.
XmlNode newPackageVersionElement = versionProps.CreateElement(packageVersionElementName, documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package versions");
versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
versionProps.DocumentElement.AppendChild(propertyGroupElement);
propertyGroupElement.AppendChild(newPackageVersionElement);
}
if (!addedPackageNameElement)
{
XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package names");
versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
versionProps.DocumentElement.AppendChild(propertyGroupElement);
propertyGroupElement.AppendChild(newPackageNameElement);
}
// TODO: This should not be done here. This should return some kind of generic file container to the caller,
// who will gather up all updates and then call the git client to write the files all at once:
// https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so
// it's okay for now.
var file = new GitFile(VersionFiles.VersionProps, versionProps);
await _gitClient.PushFilesAsync(new List <GitFile> {
file
}, repo, branch, $"Add {dependency} to " +
$"'{VersionFiles.VersionProps}'");
_logger.LogInformation(
$"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
$"'{VersionFiles.VersionProps}'");
}
/// <summary>
/// Updates the global.json entries for tools.dotnet and sdk.version if they are older than an incoming version
/// </summary>
/// <param name="incomingDotnetVersion">version to compare against</param>
/// <param name="repoGlobalJson">Global.Json file to update</param>
/// <returns>Updated global.json file if was able to update, or the unchanged global.json if unable to</returns>
private GitFile UpdateDotnetVersionGlobalJson(SemanticVersion incomingDotnetVersion, GitFile repoGlobalJson)
{
string repoGlobalJsonContent = repoGlobalJson.ContentEncoding == ContentEncoding.Base64 ?
_gitClient.GetDecodedContent(repoGlobalJson.Content) :
repoGlobalJson.Content;
try
{
JObject parsedGlobalJson = JObject.Parse(repoGlobalJsonContent);
if (SemanticVersion.TryParse(parsedGlobalJson.SelectToken("tools.dotnet").ToString(), out SemanticVersion repoDotnetVersion))
{
if (repoDotnetVersion.CompareTo(incomingDotnetVersion) < 0)
{
parsedGlobalJson["tools"]["dotnet"] = incomingDotnetVersion.ToNormalizedString();
// Also update and keep sdk.version in sync.
JToken sdkVersion = parsedGlobalJson.SelectToken("sdk.version");
if (sdkVersion != null)
{
parsedGlobalJson["sdk"]["version"] = incomingDotnetVersion.ToNormalizedString();
}
return(new GitFile(VersionFiles.GlobalJson, parsedGlobalJson));
}
return(repoGlobalJson);
}
else
{
_logger.LogError("Could not parse the repo's dotnet version from the global.json. Skipping update to dotnet version sections");
return(repoGlobalJson);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to update Dotnet version for global.json. Skipping update to version sections.");
return(repoGlobalJson);
}
}