/// <summary> /// Get a tree item blob from github. /// </summary> /// <param name="path">Base path of final git file</param> /// <param name="treeItem">Tree item to retrieve</param> /// <param name="owner">Organization</param> /// <param name="repo">Repository</param> /// <returns>Git file with tree item contents.</returns> private async Task <GitFile> GetGitItemImpl(string path, TreeItem treeItem, string owner, string repo) { Octokit.Blob blob = await ExponentialRetry.RetryAsync( async() => await Client.Git.Blob.Get(owner, repo, treeItem.Sha), ex => _logger.LogError(ex, $"Failed to get blob at sha {treeItem.Sha}"), ex => ex is ApiException apiex && apiex.StatusCode >= HttpStatusCode.InternalServerError); ContentEncoding encoding; switch (blob.Encoding.Value) { case EncodingType.Base64: encoding = ContentEncoding.Base64; break; case EncodingType.Utf8: encoding = ContentEncoding.Utf8; break; default: throw new NotImplementedException($"Unknown github encoding type {blob.Encoding.StringValue}"); } GitFile newFile = new GitFile( path + "/" + treeItem.Path, blob.Content, encoding, treeItem.Mode); return(newFile); }
public async Task AddDependencyToGlobalJson( string repo, string branch, string parentField, string dependencyName, string version) { JToken versionProperty = new JProperty(dependencyName, version); JObject globalJson = await ReadGlobalJsonAsync(repo, branch); JToken parent = globalJson[parentField]; if (parent != null) { parent.Last.AddAfterSelf(versionProperty); } else { globalJson.Add(new JProperty(parentField, new JObject(versionProperty))); } var file = new GitFile(VersionFiles.GlobalJson, globalJson); await _gitClient.PushFilesAsync(new List <GitFile> { file }, repo, branch, $"Add {dependencyName} to " + $"'{VersionFiles.GlobalJson}'"); _logger.LogInformation( $"Dependency '{dependencyName}' with version '{version}' successfully added to global.json"); }
public async Task GetCommitMapForPathAsync(string repoUri, string branch, string assetsProducedInCommit, List <GitFile> files, string pullRequestBaseBranch, string path = "eng/common/") { if (path.EndsWith("/")) { path = path.Substring(0, path.Length - 1); } _logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}'"); string ownerAndRepo = GetOwnerAndRepoFromRepoUri(repoUri); HttpResponseMessage response = await this.ExecuteGitCommand(HttpMethod.Get, $"repos/{ownerAndRepo}/contents/{path}?ref={assetsProducedInCommit}", _logger); List <GitHubContent> contents = JsonConvert.DeserializeObject <List <GitHubContent> >(await response.Content.ReadAsStringAsync()); foreach (GitHubContent content in contents) { if (content.Type == GitHubContentType.File) { if (!GitFileManager.DependencyFiles.Contains(content.Path)) { string fileContent = await GetFileContentsAsync(ownerAndRepo, content.Path); GitFile gitCommit = new GitFile(content.Path, fileContent); files.Add(gitCommit); } } else { await GetCommitMapForPathAsync(repoUri, branch, assetsProducedInCommit, files, pullRequestBaseBranch, content.Path); } } _logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}' succeeded!"); }
/// <summary> /// Get a tree item blob from github, using the cache if it exists. /// </summary> /// <param name="path">Base path of final git file</param> /// <param name="treeItem">Tree item to retrieve</param> /// <param name="owner">Organization</param> /// <param name="repo">Repository</param> /// <returns>Git file with tree item contents.</returns> public async Task <GitFile> GetGitTreeItem(string path, TreeItem treeItem, string owner, string repo) { // If we have a cache available here, attempt to get the value in the cache // before making the request. Generally, we are requesting the same files for each // arcade update sent to each repository daily per subscription. This is inefficient, as it means we // request each file N times, where N is the number of subscriptions. The number of files (M), // is non-trivial, so reducing N*M to M is vast improvement. // Use a combination of (treeItem.Path, treeItem.Sha as the key) as items with identical contents but // different paths will have the same SHA. I think it is overkill to hash the repo and owner into // the key. if (Cache != null) { return(await Cache.GetOrCreateAsync((treeItem.Path, treeItem.Sha), async (entry) => { GitFile file = await GetGitItemImpl(path, treeItem, owner, repo); // Set the size of the entry. The size is not computed by the caching system // (it has no way to do so). There are two bytes per each character in a string. // We do not really need to worry about the size of the GitFile class itself, // just the variable length elements. entry.Size = 2 * (file.Content.Length + file.FilePath.Length + file.Mode.Length); return file; })); } else { return(await GetGitItemImpl(path, treeItem, owner, repo)); } }
public async Task AddDependencyToVersionDetailsAsync( string repo, string branch, DependencyDetail dependency, DependencyType dependencyType) { XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null); XmlNode newDependency = versionDetails.CreateElement("Dependency"); XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name"); nameAttribute.Value = dependency.Name; newDependency.Attributes.Append(nameAttribute); XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version"); versionAttribute.Value = dependency.Version; newDependency.Attributes.Append(versionAttribute); XmlNode uri = versionDetails.CreateElement("Uri"); uri.InnerText = dependency.RepoUri; newDependency.AppendChild(uri); XmlNode sha = versionDetails.CreateElement("Sha"); sha.InnerText = dependency.Commit; newDependency.AppendChild(sha); XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies"); if (dependenciesNode == null) { dependenciesNode = versionDetails.CreateElement($"{dependencyType}Dependencies"); versionDetails.DocumentElement.AppendChild(dependenciesNode); } dependenciesNode.AppendChild(newDependency); // TODO: This should not be done here. This should return some kind of generic file container to the caller, // who will gather up all updates and then call the git client to write the files all at once: // https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so // it's okay for now. var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails); await _gitClient.PushFilesAsync(new List <GitFile> { file }, repo, branch, $"Add {dependency} to " + $"'{VersionFiles.VersionDetailsXml}'"); _logger.LogInformation( $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " + $"'{VersionFiles.VersionDetailsXml}'"); }
public async Task AddDependencyToVersionDetailsAsync( string repo, string branch, DependencyDetail dependency) { XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null); XmlNode newDependency = versionDetails.CreateElement(VersionFiles.DependencyElementName); SetAttribute(versionDetails, newDependency, VersionFiles.NameAttributeName, dependency.Name); SetAttribute(versionDetails, newDependency, VersionFiles.VersionAttributeName, dependency.Version); // Only add the pinned attribute if the pinned option is set to true if (dependency.Pinned) { SetAttribute(versionDetails, newDependency, VersionFiles.PinnedAttributeName, "True"); } // Only add the coherent parent attribute if it is set if (!string.IsNullOrEmpty(dependency.CoherentParentDependencyName)) { SetAttribute(versionDetails, newDependency, VersionFiles.CoherentParentAttributeName, dependency.CoherentParentDependencyName); } SetElement(versionDetails, newDependency, VersionFiles.UriElementName, dependency.RepoUri); SetElement(versionDetails, newDependency, VersionFiles.ShaElementName, dependency.Commit); XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependency.Type}{VersionFiles.DependenciesElementName}"); if (dependenciesNode == null) { dependenciesNode = versionDetails.CreateElement($"{dependency.Type}{VersionFiles.DependenciesElementName}"); versionDetails.DocumentElement.AppendChild(dependenciesNode); } dependenciesNode.AppendChild(newDependency); // TODO: This should not be done here. This should return some kind of generic file container to the caller, // who will gather up all updates and then call the git client to write the files all at once: // https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so // it's okay for now. var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails); await _gitClient.CommitFilesAsync(new List <GitFile> { file }, repo, branch, $"Add {dependency} to " + $"'{VersionFiles.VersionDetailsXml}'"); _logger.LogInformation( $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " + $"'{VersionFiles.VersionDetailsXml}'"); }
public async Task AddDependencyToVersionProps(string filePath, DependencyDetail dependency) { XmlDocument versionProps = await ReadVersionPropsAsync(filePath, null); string versionedName = dependency.Name.Replace(".", string.Empty); versionedName = $"{versionedName}Version"; XmlNode versionNode = versionProps.DocumentElement.SelectNodes($"//PropertyGroup").Item(0); XmlNode newDependency = versionProps.CreateElement(versionedName); newDependency.InnerText = dependency.Version; versionNode.AppendChild(newDependency); GitFile file = new GitFile(filePath, versionProps); File.WriteAllText(file.FilePath, file.Content); _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.props"); }
public async Task AddDependencyToGlobalJson(string filePath, string parentField, string dependencyName, string version) { JToken versionProperty = new JProperty(dependencyName, version); JObject globalJson = await ReadGlobalJsonAsync(filePath, null); JToken parent = globalJson[parentField]; if (parent != null) { parent.Last.AddAfterSelf(versionProperty); } else { globalJson.Add(new JProperty(parentField, new JObject(versionProperty))); } GitFile file = new GitFile(filePath, globalJson); File.WriteAllText(file.FilePath, file.Content); _logger.LogInformation($"Dependency '{dependencyName}' with version '{version}' successfully added to global.json"); }
public async Task AddDependencyToVersionDetails(string filePath, DependencyDetail dependency, DependencyType dependencyType) { XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(filePath, null); XmlNode newDependency = versionDetails.CreateElement("Dependency"); XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name"); nameAttribute.Value = dependency.Name; newDependency.Attributes.Append(nameAttribute); XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version"); versionAttribute.Value = dependency.Version; newDependency.Attributes.Append(versionAttribute); XmlNode uri = versionDetails.CreateElement("Uri"); uri.InnerText = dependency.RepoUri; newDependency.AppendChild(uri); XmlNode sha = versionDetails.CreateElement("Sha"); sha.InnerText = dependency.Commit; newDependency.AppendChild(sha); XmlNode dependencies = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies"); dependencies.AppendChild(newDependency); GitFile file = new GitFile(filePath, versionDetails); File.WriteAllText(file.FilePath, file.Content); _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.Details.xml"); }
/// <summary> /// Add a dependency to Versions.props. This has the form: /// <!-- Package names --> /// <PropertyGroup> /// <MicrosoftDotNetApiCompatPackage>Microsoft.DotNet.ApiCompat</MicrosoftDotNetApiCompatPackage> /// </PropertyGroup> /// /// <!-- Package versions --> /// <PropertyGroup> /// <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.18478.5</MicrosoftDotNetApiCompatPackageVersion> /// </PropertyGroup> /// /// See https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md for more /// information. /// </summary> /// <param name="repo">Path to Versions.props file</param> /// <param name="dependency">Dependency information to add.</param> /// <returns>Async task.</returns> public async Task AddDependencyToVersionsPropsAsync(string repo, string branch, DependencyDetail dependency) { XmlDocument versionProps = await ReadVersionPropsAsync(repo, null); string documentNamespaceUri = versionProps.DocumentElement.NamespaceURI; string packageNameElementName = VersionFiles.GetVersionPropsPackageElementName(dependency.Name); string packageVersionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(dependency.Name); string packageVersionAlternateElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName( dependency.Name); // Select elements by local name, since the Project (DocumentElement) element often has a default // xmlns set. XmlNodeList propertyGroupNodes = versionProps.DocumentElement.SelectNodes($"//*[local-name()='PropertyGroup']"); XmlNode newPackageNameElement = versionProps.CreateElement(packageNameElementName, documentNamespaceUri); newPackageNameElement.InnerText = dependency.Name; bool addedPackageVersionElement = false; bool addedPackageNameElement = false; // There can be more than one property group. Find the appropriate one containing an existing element of // the same type, and add it to the parent. foreach (XmlNode propertyGroupNode in propertyGroupNodes) { if (propertyGroupNode.HasChildNodes) { foreach (XmlNode propertyNode in propertyGroupNode.ChildNodes) { if (!addedPackageVersionElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsVersionElementSuffix)) { XmlNode newPackageVersionElement = versionProps.CreateElement( packageVersionElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; propertyGroupNode.AppendChild(newPackageVersionElement); addedPackageVersionElement = true; break; } // Test for alternate suffixes. This will eventually be replaced by repo-level configuration: // https://github.com/dotnet/arcade/issues/1095 else if (!addedPackageVersionElement && propertyNode.Name.EndsWith( VersionFiles.VersionPropsAlternateVersionElementSuffix)) { XmlNode newPackageVersionElement = versionProps.CreateElement( packageVersionAlternateElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; propertyGroupNode.AppendChild(newPackageVersionElement); addedPackageVersionElement = true; break; } else if (!addedPackageNameElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsPackageElementSuffix)) { propertyGroupNode.AppendChild(newPackageNameElement); addedPackageNameElement = true; break; } } if (addedPackageVersionElement && addedPackageNameElement) { break; } } } // Add the property groups if none were present if (!addedPackageVersionElement) { // If the repository doesn't have any package version element, then // use the non-alternate element name. XmlNode newPackageVersionElement = versionProps.CreateElement(packageVersionElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri); XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package versions"); versionProps.DocumentElement.AppendChild(propertyGroupCommentElement); versionProps.DocumentElement.AppendChild(propertyGroupElement); propertyGroupElement.AppendChild(newPackageVersionElement); } if (!addedPackageNameElement) { XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri); XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package names"); versionProps.DocumentElement.AppendChild(propertyGroupCommentElement); versionProps.DocumentElement.AppendChild(propertyGroupElement); propertyGroupElement.AppendChild(newPackageNameElement); } // TODO: This should not be done here. This should return some kind of generic file container to the caller, // who will gather up all updates and then call the git client to write the files all at once: // https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so // it's okay for now. var file = new GitFile(VersionFiles.VersionProps, versionProps); await _gitClient.PushFilesAsync(new List <GitFile> { file }, repo, branch, $"Add {dependency} to " + $"'{VersionFiles.VersionProps}'"); _logger.LogInformation( $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " + $"'{VersionFiles.VersionProps}'"); }
/// <summary> /// Updates the global.json entries for tools.dotnet and sdk.version if they are older than an incoming version /// </summary> /// <param name="incomingDotnetVersion">version to compare against</param> /// <param name="repoGlobalJson">Global.Json file to update</param> /// <returns>Updated global.json file if was able to update, or the unchanged global.json if unable to</returns> private GitFile UpdateDotnetVersionGlobalJson(SemanticVersion incomingDotnetVersion, GitFile repoGlobalJson) { string repoGlobalJsonContent = repoGlobalJson.ContentEncoding == ContentEncoding.Base64 ? _gitClient.GetDecodedContent(repoGlobalJson.Content) : repoGlobalJson.Content; try { JObject parsedGlobalJson = JObject.Parse(repoGlobalJsonContent); if (SemanticVersion.TryParse(parsedGlobalJson.SelectToken("tools.dotnet").ToString(), out SemanticVersion repoDotnetVersion)) { if (repoDotnetVersion.CompareTo(incomingDotnetVersion) < 0) { parsedGlobalJson["tools"]["dotnet"] = incomingDotnetVersion.ToNormalizedString(); // Also update and keep sdk.version in sync. JToken sdkVersion = parsedGlobalJson.SelectToken("sdk.version"); if (sdkVersion != null) { parsedGlobalJson["sdk"]["version"] = incomingDotnetVersion.ToNormalizedString(); } return(new GitFile(VersionFiles.GlobalJson, parsedGlobalJson)); } return(repoGlobalJson); } else { _logger.LogError("Could not parse the repo's dotnet version from the global.json. Skipping update to dotnet version sections"); return(repoGlobalJson); } } catch (Exception ex) { _logger.LogError(ex, "Failed to update Dotnet version for global.json. Skipping update to version sections."); return(repoGlobalJson); } }