Ejemplo n.º 1
0
        /// <summary>
        ///     Get a tree item blob from github.
        /// </summary>
        /// <param name="path">Base path of final git file</param>
        /// <param name="treeItem">Tree item to retrieve</param>
        /// <param name="owner">Organization</param>
        /// <param name="repo">Repository</param>
        /// <returns>Git file with tree item contents.</returns>
        private async Task <GitFile> GetGitItemImpl(string path, TreeItem treeItem, string owner, string repo)
        {
            Octokit.Blob blob = await ExponentialRetry.RetryAsync(
                async() => await Client.Git.Blob.Get(owner, repo, treeItem.Sha),
                ex => _logger.LogError(ex, $"Failed to get blob at sha {treeItem.Sha}"),
                ex => ex is ApiException apiex && apiex.StatusCode >= HttpStatusCode.InternalServerError);

            ContentEncoding encoding;

            switch (blob.Encoding.Value)
            {
            case EncodingType.Base64:
                encoding = ContentEncoding.Base64;
                break;

            case EncodingType.Utf8:
                encoding = ContentEncoding.Utf8;
                break;

            default:
                throw new NotImplementedException($"Unknown github encoding type {blob.Encoding.StringValue}");
            }
            GitFile newFile = new GitFile(
                path + "/" + treeItem.Path,
                blob.Content,
                encoding,
                treeItem.Mode);

            return(newFile);
        }
Ejemplo n.º 2
0
        public async Task AddDependencyToGlobalJson(
            string repo,
            string branch,
            string parentField,
            string dependencyName,
            string version)
        {
            JToken  versionProperty = new JProperty(dependencyName, version);
            JObject globalJson      = await ReadGlobalJsonAsync(repo, branch);

            JToken parent = globalJson[parentField];

            if (parent != null)
            {
                parent.Last.AddAfterSelf(versionProperty);
            }
            else
            {
                globalJson.Add(new JProperty(parentField, new JObject(versionProperty)));
            }

            var file = new GitFile(VersionFiles.GlobalJson, globalJson);
            await _gitClient.PushFilesAsync(new List <GitFile> {
                file
            }, repo, branch, $"Add {dependencyName} to " +
                                            $"'{VersionFiles.GlobalJson}'");

            _logger.LogInformation(
                $"Dependency '{dependencyName}' with version '{version}' successfully added to global.json");
        }
Ejemplo n.º 3
0
        public async Task GetCommitMapForPathAsync(string repoUri, string branch, string assetsProducedInCommit, List <GitFile> files, string pullRequestBaseBranch, string path = "eng/common/")
        {
            if (path.EndsWith("/"))
            {
                path = path.Substring(0, path.Length - 1);
            }
            _logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}'");

            string ownerAndRepo          = GetOwnerAndRepoFromRepoUri(repoUri);
            HttpResponseMessage response = await this.ExecuteGitCommand(HttpMethod.Get, $"repos/{ownerAndRepo}/contents/{path}?ref={assetsProducedInCommit}", _logger);

            List <GitHubContent> contents = JsonConvert.DeserializeObject <List <GitHubContent> >(await response.Content.ReadAsStringAsync());

            foreach (GitHubContent content in contents)
            {
                if (content.Type == GitHubContentType.File)
                {
                    if (!GitFileManager.DependencyFiles.Contains(content.Path))
                    {
                        string fileContent = await GetFileContentsAsync(ownerAndRepo, content.Path);

                        GitFile gitCommit = new GitFile(content.Path, fileContent);
                        files.Add(gitCommit);
                    }
                }
                else
                {
                    await GetCommitMapForPathAsync(repoUri, branch, assetsProducedInCommit, files, pullRequestBaseBranch, content.Path);
                }
            }

            _logger.LogInformation($"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}' succeeded!");
        }
Ejemplo n.º 4
0
        /// <summary>
        ///     Get a tree item blob from github, using the cache if it exists.
        /// </summary>
        /// <param name="path">Base path of final git file</param>
        /// <param name="treeItem">Tree item to retrieve</param>
        /// <param name="owner">Organization</param>
        /// <param name="repo">Repository</param>
        /// <returns>Git file with tree item contents.</returns>
        public async Task <GitFile> GetGitTreeItem(string path, TreeItem treeItem, string owner, string repo)
        {
            // If we have a cache available here, attempt to get the value in the cache
            // before making the request. Generally, we are requesting the same files for each
            // arcade update sent to each repository daily per subscription. This is inefficient, as it means we
            // request each file N times, where N is the number of subscriptions. The number of files (M),
            // is non-trivial, so reducing N*M to M is vast improvement.
            // Use a combination of (treeItem.Path, treeItem.Sha as the key) as items with identical contents but
            // different paths will have the same SHA. I think it is overkill to hash the repo and owner into
            // the key.

            if (Cache != null)
            {
                return(await Cache.GetOrCreateAsync((treeItem.Path, treeItem.Sha), async (entry) =>
                {
                    GitFile file = await GetGitItemImpl(path, treeItem, owner, repo);

                    // Set the size of the entry. The size is not computed by the caching system
                    // (it has no way to do so). There are two bytes per each character in a string.
                    // We do not really need to worry about the size of the GitFile class itself,
                    // just the variable length elements.
                    entry.Size = 2 * (file.Content.Length + file.FilePath.Length + file.Mode.Length);

                    return file;
                }));
            }
            else
            {
                return(await GetGitItemImpl(path, treeItem, owner, repo));
            }
        }
Ejemplo n.º 5
0
        public async Task AddDependencyToVersionDetailsAsync(
            string repo,
            string branch,
            DependencyDetail dependency,
            DependencyType dependencyType)
        {
            XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null);

            XmlNode newDependency = versionDetails.CreateElement("Dependency");

            XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name");

            nameAttribute.Value = dependency.Name;
            newDependency.Attributes.Append(nameAttribute);

            XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version");

            versionAttribute.Value = dependency.Version;
            newDependency.Attributes.Append(versionAttribute);

            XmlNode uri = versionDetails.CreateElement("Uri");

            uri.InnerText = dependency.RepoUri;
            newDependency.AppendChild(uri);

            XmlNode sha = versionDetails.CreateElement("Sha");

            sha.InnerText = dependency.Commit;
            newDependency.AppendChild(sha);

            XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies");

            if (dependenciesNode == null)
            {
                dependenciesNode = versionDetails.CreateElement($"{dependencyType}Dependencies");
                versionDetails.DocumentElement.AppendChild(dependenciesNode);
            }
            dependenciesNode.AppendChild(newDependency);

            // TODO: This should not be done here.  This should return some kind of generic file container to the caller,
            // who will gather up all updates and then call the git client to write the files all at once:
            // https://github.com/dotnet/arcade/issues/1095.  Today this is only called from the Local interface so
            // it's okay for now.
            var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails);
            await _gitClient.PushFilesAsync(new List <GitFile> {
                file
            }, repo, branch, $"Add {dependency} to " +
                                            $"'{VersionFiles.VersionDetailsXml}'");

            _logger.LogInformation(
                $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
                $"'{VersionFiles.VersionDetailsXml}'");
        }
Ejemplo n.º 6
0
        public async Task AddDependencyToVersionDetailsAsync(
            string repo,
            string branch,
            DependencyDetail dependency)
        {
            XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repo, null);

            XmlNode newDependency = versionDetails.CreateElement(VersionFiles.DependencyElementName);

            SetAttribute(versionDetails, newDependency, VersionFiles.NameAttributeName, dependency.Name);
            SetAttribute(versionDetails, newDependency, VersionFiles.VersionAttributeName, dependency.Version);

            // Only add the pinned attribute if the pinned option is set to true
            if (dependency.Pinned)
            {
                SetAttribute(versionDetails, newDependency, VersionFiles.PinnedAttributeName, "True");
            }

            // Only add the coherent parent attribute if it is set
            if (!string.IsNullOrEmpty(dependency.CoherentParentDependencyName))
            {
                SetAttribute(versionDetails, newDependency, VersionFiles.CoherentParentAttributeName, dependency.CoherentParentDependencyName);
            }

            SetElement(versionDetails, newDependency, VersionFiles.UriElementName, dependency.RepoUri);
            SetElement(versionDetails, newDependency, VersionFiles.ShaElementName, dependency.Commit);

            XmlNode dependenciesNode = versionDetails.SelectSingleNode($"//{dependency.Type}{VersionFiles.DependenciesElementName}");

            if (dependenciesNode == null)
            {
                dependenciesNode = versionDetails.CreateElement($"{dependency.Type}{VersionFiles.DependenciesElementName}");
                versionDetails.DocumentElement.AppendChild(dependenciesNode);
            }
            dependenciesNode.AppendChild(newDependency);

            // TODO: This should not be done here.  This should return some kind of generic file container to the caller,
            // who will gather up all updates and then call the git client to write the files all at once:
            // https://github.com/dotnet/arcade/issues/1095.  Today this is only called from the Local interface so
            // it's okay for now.
            var file = new GitFile(VersionFiles.VersionDetailsXml, versionDetails);
            await _gitClient.CommitFilesAsync(new List <GitFile> {
                file
            }, repo, branch, $"Add {dependency} to " +
                                              $"'{VersionFiles.VersionDetailsXml}'");

            _logger.LogInformation(
                $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
                $"'{VersionFiles.VersionDetailsXml}'");
        }
Ejemplo n.º 7
0
        public async Task AddDependencyToVersionProps(string filePath, DependencyDetail dependency)
        {
            XmlDocument versionProps = await ReadVersionPropsAsync(filePath, null);

            string versionedName = dependency.Name.Replace(".", string.Empty);

            versionedName = $"{versionedName}Version";

            XmlNode versionNode = versionProps.DocumentElement.SelectNodes($"//PropertyGroup").Item(0);

            XmlNode newDependency = versionProps.CreateElement(versionedName);

            newDependency.InnerText = dependency.Version;
            versionNode.AppendChild(newDependency);

            GitFile file = new GitFile(filePath, versionProps);

            File.WriteAllText(file.FilePath, file.Content);

            _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.props");
        }
Ejemplo n.º 8
0
        public async Task AddDependencyToGlobalJson(string filePath, string parentField, string dependencyName, string version)
        {
            JToken  versionProperty = new JProperty(dependencyName, version);
            JObject globalJson      = await ReadGlobalJsonAsync(filePath, null);

            JToken parent = globalJson[parentField];

            if (parent != null)
            {
                parent.Last.AddAfterSelf(versionProperty);
            }
            else
            {
                globalJson.Add(new JProperty(parentField, new JObject(versionProperty)));
            }

            GitFile file = new GitFile(filePath, globalJson);

            File.WriteAllText(file.FilePath, file.Content);

            _logger.LogInformation($"Dependency '{dependencyName}' with version '{version}' successfully added to global.json");
        }
Ejemplo n.º 9
0
        public async Task AddDependencyToVersionDetails(string filePath, DependencyDetail dependency, DependencyType dependencyType)
        {
            XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(filePath, null);

            XmlNode newDependency = versionDetails.CreateElement("Dependency");

            XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name");

            nameAttribute.Value = dependency.Name;
            newDependency.Attributes.Append(nameAttribute);

            XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version");

            versionAttribute.Value = dependency.Version;
            newDependency.Attributes.Append(versionAttribute);

            XmlNode uri = versionDetails.CreateElement("Uri");

            uri.InnerText = dependency.RepoUri;
            newDependency.AppendChild(uri);

            XmlNode sha = versionDetails.CreateElement("Sha");

            sha.InnerText = dependency.Commit;
            newDependency.AppendChild(sha);

            XmlNode dependencies = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies");

            dependencies.AppendChild(newDependency);

            GitFile file = new GitFile(filePath, versionDetails);

            File.WriteAllText(file.FilePath, file.Content);

            _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.Details.xml");
        }
Ejemplo n.º 10
0
        /// <summary>
        ///     Add a dependency to Versions.props.  This has the form:
        ///     <!-- Package names -->
        ///     <PropertyGroup>
        ///         <MicrosoftDotNetApiCompatPackage>Microsoft.DotNet.ApiCompat</MicrosoftDotNetApiCompatPackage>
        ///     </PropertyGroup>
        ///
        ///     <!-- Package versions -->
        ///     <PropertyGroup>
        ///         <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.18478.5</MicrosoftDotNetApiCompatPackageVersion>
        ///     </PropertyGroup>
        ///
        ///     See https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md for more
        ///     information.
        /// </summary>
        /// <param name="repo">Path to Versions.props file</param>
        /// <param name="dependency">Dependency information to add.</param>
        /// <returns>Async task.</returns>
        public async Task AddDependencyToVersionsPropsAsync(string repo, string branch, DependencyDetail dependency)
        {
            XmlDocument versionProps = await ReadVersionPropsAsync(repo, null);

            string documentNamespaceUri = versionProps.DocumentElement.NamespaceURI;

            string packageNameElementName             = VersionFiles.GetVersionPropsPackageElementName(dependency.Name);
            string packageVersionElementName          = VersionFiles.GetVersionPropsPackageVersionElementName(dependency.Name);
            string packageVersionAlternateElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName(
                dependency.Name);

            // Select elements by local name, since the Project (DocumentElement) element often has a default
            // xmlns set.
            XmlNodeList propertyGroupNodes = versionProps.DocumentElement.SelectNodes($"//*[local-name()='PropertyGroup']");

            XmlNode newPackageNameElement = versionProps.CreateElement(packageNameElementName, documentNamespaceUri);

            newPackageNameElement.InnerText = dependency.Name;

            bool addedPackageVersionElement = false;
            bool addedPackageNameElement    = false;

            // There can be more than one property group.  Find the appropriate one containing an existing element of
            // the same type, and add it to the parent.
            foreach (XmlNode propertyGroupNode in propertyGroupNodes)
            {
                if (propertyGroupNode.HasChildNodes)
                {
                    foreach (XmlNode propertyNode in propertyGroupNode.ChildNodes)
                    {
                        if (!addedPackageVersionElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsVersionElementSuffix))
                        {
                            XmlNode newPackageVersionElement = versionProps.CreateElement(
                                packageVersionElementName,
                                documentNamespaceUri);
                            newPackageVersionElement.InnerText = dependency.Version;

                            propertyGroupNode.AppendChild(newPackageVersionElement);
                            addedPackageVersionElement = true;
                            break;
                        }
                        // Test for alternate suffixes.  This will eventually be replaced by repo-level configuration:
                        // https://github.com/dotnet/arcade/issues/1095
                        else if (!addedPackageVersionElement && propertyNode.Name.EndsWith(
                                     VersionFiles.VersionPropsAlternateVersionElementSuffix))
                        {
                            XmlNode newPackageVersionElement = versionProps.CreateElement(
                                packageVersionAlternateElementName,
                                documentNamespaceUri);
                            newPackageVersionElement.InnerText = dependency.Version;

                            propertyGroupNode.AppendChild(newPackageVersionElement);
                            addedPackageVersionElement = true;
                            break;
                        }
                        else if (!addedPackageNameElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsPackageElementSuffix))
                        {
                            propertyGroupNode.AppendChild(newPackageNameElement);
                            addedPackageNameElement = true;
                            break;
                        }
                    }

                    if (addedPackageVersionElement && addedPackageNameElement)
                    {
                        break;
                    }
                }
            }

            // Add the property groups if none were present
            if (!addedPackageVersionElement)
            {
                // If the repository doesn't have any package version element, then
                // use the non-alternate element name.
                XmlNode newPackageVersionElement = versionProps.CreateElement(packageVersionElementName, documentNamespaceUri);
                newPackageVersionElement.InnerText = dependency.Version;

                XmlNode propertyGroupElement        = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
                XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package versions");
                versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
                versionProps.DocumentElement.AppendChild(propertyGroupElement);
                propertyGroupElement.AppendChild(newPackageVersionElement);
            }

            if (!addedPackageNameElement)
            {
                XmlNode propertyGroupElement        = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
                XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package names");
                versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
                versionProps.DocumentElement.AppendChild(propertyGroupElement);
                propertyGroupElement.AppendChild(newPackageNameElement);
            }

            // TODO: This should not be done here.  This should return some kind of generic file container to the caller,
            // who will gather up all updates and then call the git client to write the files all at once:
            // https://github.com/dotnet/arcade/issues/1095.  Today this is only called from the Local interface so
            // it's okay for now.
            var file = new GitFile(VersionFiles.VersionProps, versionProps);
            await _gitClient.PushFilesAsync(new List <GitFile> {
                file
            }, repo, branch, $"Add {dependency} to " +
                                            $"'{VersionFiles.VersionProps}'");

            _logger.LogInformation(
                $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
                $"'{VersionFiles.VersionProps}'");
        }
Ejemplo n.º 11
0
        /// <summary>
        /// Updates the global.json entries for tools.dotnet and sdk.version if they are older than an incoming version
        /// </summary>
        /// <param name="incomingDotnetVersion">version to compare against</param>
        /// <param name="repoGlobalJson">Global.Json file to update</param>
        /// <returns>Updated global.json file if was able to update, or the unchanged global.json if unable to</returns>
        private GitFile UpdateDotnetVersionGlobalJson(SemanticVersion incomingDotnetVersion, GitFile repoGlobalJson)
        {
            string repoGlobalJsonContent = repoGlobalJson.ContentEncoding == ContentEncoding.Base64 ?
                                           _gitClient.GetDecodedContent(repoGlobalJson.Content) :
                                           repoGlobalJson.Content;

            try
            {
                JObject parsedGlobalJson = JObject.Parse(repoGlobalJsonContent);
                if (SemanticVersion.TryParse(parsedGlobalJson.SelectToken("tools.dotnet").ToString(), out SemanticVersion repoDotnetVersion))
                {
                    if (repoDotnetVersion.CompareTo(incomingDotnetVersion) < 0)
                    {
                        parsedGlobalJson["tools"]["dotnet"] = incomingDotnetVersion.ToNormalizedString();

                        // Also update and keep sdk.version in sync.
                        JToken sdkVersion = parsedGlobalJson.SelectToken("sdk.version");
                        if (sdkVersion != null)
                        {
                            parsedGlobalJson["sdk"]["version"] = incomingDotnetVersion.ToNormalizedString();
                        }
                        return(new GitFile(VersionFiles.GlobalJson, parsedGlobalJson));
                    }
                    return(repoGlobalJson);
                }
                else
                {
                    _logger.LogError("Could not parse the repo's dotnet version from the global.json. Skipping update to dotnet version sections");
                    return(repoGlobalJson);
                }
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Failed to update Dotnet version for global.json. Skipping update to version sections.");
                return(repoGlobalJson);
            }
        }