private IEnumerable <DependencyDetail> GetDependencyDetails(XmlDocument document, string branch = null) { List <DependencyDetail> dependencyDetails = new List <DependencyDetail>(); if (document != null) { BuildDependencies(document.DocumentElement.SelectNodes("//Dependency")); void BuildDependencies(XmlNodeList dependencies) { if (dependencies.Count > 0) { foreach (XmlNode dependency in dependencies) { if (dependency.NodeType != XmlNodeType.Comment && dependency.NodeType != XmlNodeType.Whitespace) { DependencyDetail dependencyDetail = new DependencyDetail { Branch = branch, Name = dependency.Attributes["Name"].Value, RepoUri = dependency.SelectSingleNode("Uri").InnerText, Commit = dependency.SelectSingleNode("Sha").InnerText, Version = dependency.Attributes["Version"].Value }; dependencyDetails.Add(dependencyDetail); } } } else { _logger.LogWarning("No dependencies defined in file."); } } } else { _logger.LogError($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " + $"Look for exceptions above."); } return(dependencyDetails); }
/// <summary> /// Update well-known version files. /// </summary> /// <param name="versionProps">Versions.props xml document</param> /// <param name="token">Global.json document</param> /// <param name="itemToUpdate">Item that needs an update.</param> /// <remarks> /// TODO: https://github.com/dotnet/arcade/issues/1095 /// </remarks> private void UpdateVersionFiles(XmlDocument versionProps, JToken token, DependencyDetail itemToUpdate) { string versionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(itemToUpdate.Name); string alternateVersionElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName(itemToUpdate.Name); // Select nodes case insensitively, then update the name. XmlNode packageVersionNode = versionProps.DocumentElement.SelectSingleNode( $"//*[translate(local-name(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')=" + $"'{versionElementName.ToLower()}']"); string foundElementName = versionElementName; // Find alternate names if (packageVersionNode == null) { packageVersionNode = versionProps.DocumentElement.SelectSingleNode( $"//*[translate(local-name(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')=" + $"'{alternateVersionElementName.ToLower()}']"); foundElementName = alternateVersionElementName; } if (packageVersionNode != null) { packageVersionNode.InnerText = itemToUpdate.Version; // If the node name was updated, then create a new node with the new name, unlink this node // and create a new one in the same location. if (packageVersionNode.LocalName != foundElementName) { { XmlNode parentNode = packageVersionNode.ParentNode; XmlNode newPackageVersionElement = versionProps.CreateElement( foundElementName, versionProps.DocumentElement.NamespaceURI); newPackageVersionElement.InnerText = itemToUpdate.Version; parentNode.ReplaceChild(newPackageVersionElement, packageVersionNode); } } } // Update the global json too, even if there was an element in the props file, in case // it was listed in both UpdateVersionGlobalJson(itemToUpdate, token); }
private async Task CommitFilesForPullRequestAsync(string repoUri, string branch, string assetsProducedInCommit, IEnumerable <DependencyDetail> itemsToUpdate, string pullRequestBaseBranch = null) { CheckForValidGitClient(); GitFileContentContainer fileContainer = await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch); List <GitFile> filesToCommit = fileContainer.GetFilesToCommitMap(pullRequestBaseBranch); // If there is an arcade asset that we need to update we try to update the script files as well DependencyDetail arcadeItem = itemsToUpdate.Where(i => i.Name.ToLower().Contains("arcade")).FirstOrDefault(); if (arcadeItem != null && repoUri != arcadeItem.RepoUri) { List <GitFile> engCommonsFiles = await GetScriptFilesAsync(arcadeItem.RepoUri, assetsProducedInCommit); filesToCommit.AddRange(engCommonsFiles); } await _gitClient.PushFilesAsync(filesToCommit, repoUri, pullRequestBaseBranch, "Updating version files"); }
public async Task AddDependencyToVersionProps(string filePath, DependencyDetail dependency) { XmlDocument versionProps = await ReadVersionPropsAsync(filePath, null); string versionedName = dependency.Name.Replace(".", string.Empty); versionedName = $"{versionedName}Version"; XmlNode versionNode = versionProps.DocumentElement.SelectNodes($"//PropertyGroup").Item(0); XmlNode newDependency = versionProps.CreateElement(versionedName); newDependency.InnerText = dependency.Version; versionNode.AppendChild(newDependency); GitFile file = new GitFile(filePath, versionProps); File.WriteAllText(file.FilePath, file.Content); _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.props"); }
public async Task CommitUpdatesAsync( string repoUri, string branch, List <DependencyDetail> itemsToUpdate, string message) { CheckForValidGitClient(); GitFileContentContainer fileContainer = await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch); List <GitFile> filesToCommit = fileContainer.GetFilesToCommit(); // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = itemsToUpdate.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null && repoUri != arcadeItem.RepoUri) { // Files in arcade repository List <GitFile> engCommonFiles = await GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToCommit.AddRange(engCommonFiles); // Files in the target repo string latestCommit = await _gitClient.GetLastCommitShaAsync(_gitClient.GetOwnerAndRepoFromRepoUri(repoUri), branch); List <GitFile> targetEngCommonFiles = await GetCommonScriptFilesAsync(repoUri, latestCommit); foreach (GitFile file in targetEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { file.Operation = GitFileOperation.Delete; filesToCommit.Add(file); } } } await _gitClient.PushFilesAsync(filesToCommit, repoUri, branch, message); }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemote remote) { // TODO: This should use known updaters, but today the updaters for global.json can only // add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095 /*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote); * foreach (DependencyDetail dependency in dependencies) * { * if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function)) * { * await (Task)function.DynamicInvoke(_fileManager, _repo, dependency); * } * else * { * defaultUpdates.Add(dependency); * } * }*/ var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); // TODO: This needs to be moved into some consistent handling between local/remote and add/update: // https://github.com/dotnet/arcade/issues/1095 // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null) { List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); } // Push on local does not commit. await _gitClient.PushFilesAsync(filesToUpdate, _repo, null, null); }
public async Task AddDependencyToVersionDetails(string filePath, DependencyDetail dependency, DependencyType dependencyType) { XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(filePath, null); XmlNode newDependency = versionDetails.CreateElement("Dependency"); XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name"); nameAttribute.Value = dependency.Name; newDependency.Attributes.Append(nameAttribute); XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version"); versionAttribute.Value = dependency.Version; newDependency.Attributes.Append(versionAttribute); XmlNode uri = versionDetails.CreateElement("Uri"); uri.InnerText = dependency.RepoUri; newDependency.AppendChild(uri); XmlNode sha = versionDetails.CreateElement("Sha"); sha.InnerText = dependency.Commit; newDependency.AppendChild(sha); XmlNode dependencies = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies"); dependencies.AppendChild(newDependency); GitFile file = new GitFile(filePath, versionDetails); File.WriteAllText(file.FilePath, file.Content); _logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.Details.xml"); }
public void AddChild(DependencyGraphNode newChild, DependencyDetail dependency) { Children.Add(newChild); newChild.Parents.Add(this); }
/// <summary> /// Adds a dependency to the dependency files /// </summary> /// <returns></returns> public async Task AddDependencyAsync(DependencyDetail dependency, DependencyType dependencyType) { // TODO: https://github.com/dotnet/arcade/issues/1095 // This should be getting back a container and writing the files from here. await _fileManager.AddDependencyAsync(dependency, dependencyType, _repo, null); }
private static string GetRepoPath( DependencyDetail dependency, IEnumerable <string> remotesMap, string reposFolder, ILogger logger) { string repoPath = null; if (remotesMap != null) { if (_remotesMapping == null) { _remotesMapping = CreateRemotesMapping(remotesMap); } if (!_remotesMapping.ContainsKey(repoPath)) { throw new DarcException($"A key matching '{dependency.RepoUri}' was not " + $"found in the mapping. Please make sure to include it..."); } repoPath = _remotesMapping[repoPath]; } else { string folder = null; if (!string.IsNullOrEmpty(reposFolder)) { folder = reposFolder; } else { // If a repo folder or a mapping was not set we use the current parent's // parent folder. string gitDir = LocalHelpers.GetGitDir(logger); string parent = Directory.GetParent(gitDir).FullName; folder = Directory.GetParent(parent).FullName; } // There are cases when the sha is not specified in Version.Details.xml // since owners want that Maestro++ fills this in. Without a sha we // cannot walk the graph. We do not fail the process but display/return // a dependency with no sha and for that graph path that would be the end of the walk if (string.IsNullOrEmpty(dependency.Commit)) { return(null); } repoPath = LocalHelpers.GetRepoPathFromFolder(folder, dependency.Commit, logger); if (string.IsNullOrEmpty(repoPath)) { throw new DarcException($"Commit '{dependency.Commit}' was not found in any " + $"folder in '{folder}'. Make sure a folder for '{dependency.RepoUri}' exists " + $"and it has all the latest changes..."); } } return(repoPath); }
/// <summary> /// Add a dependency to Versions.props. This has the form: /// <!-- Package names --> /// <PropertyGroup> /// <MicrosoftDotNetApiCompatPackage>Microsoft.DotNet.ApiCompat</MicrosoftDotNetApiCompatPackage> /// </PropertyGroup> /// /// <!-- Package versions --> /// <PropertyGroup> /// <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.18478.5</MicrosoftDotNetApiCompatPackageVersion> /// </PropertyGroup> /// /// See https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md for more /// information. /// </summary> /// <param name="repo">Path to Versions.props file</param> /// <param name="dependency">Dependency information to add.</param> /// <returns>Async task.</returns> public async Task AddDependencyToVersionsPropsAsync(string repo, string branch, DependencyDetail dependency) { XmlDocument versionProps = await ReadVersionPropsAsync(repo, null); string documentNamespaceUri = versionProps.DocumentElement.NamespaceURI; string packageNameElementName = VersionFiles.GetVersionPropsPackageElementName(dependency.Name); string packageVersionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(dependency.Name); string packageVersionAlternateElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName( dependency.Name); // Select elements by local name, since the Project (DocumentElement) element often has a default // xmlns set. XmlNodeList propertyGroupNodes = versionProps.DocumentElement.SelectNodes($"//*[local-name()='PropertyGroup']"); XmlNode newPackageNameElement = versionProps.CreateElement(packageNameElementName, documentNamespaceUri); newPackageNameElement.InnerText = dependency.Name; bool addedPackageVersionElement = false; bool addedPackageNameElement = false; // There can be more than one property group. Find the appropriate one containing an existing element of // the same type, and add it to the parent. foreach (XmlNode propertyGroupNode in propertyGroupNodes) { if (propertyGroupNode.HasChildNodes) { foreach (XmlNode propertyNode in propertyGroupNode.ChildNodes) { if (!addedPackageVersionElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsVersionElementSuffix)) { XmlNode newPackageVersionElement = versionProps.CreateElement( packageVersionElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; propertyGroupNode.AppendChild(newPackageVersionElement); addedPackageVersionElement = true; break; } // Test for alternate suffixes. This will eventually be replaced by repo-level configuration: // https://github.com/dotnet/arcade/issues/1095 else if (!addedPackageVersionElement && propertyNode.Name.EndsWith( VersionFiles.VersionPropsAlternateVersionElementSuffix)) { XmlNode newPackageVersionElement = versionProps.CreateElement( packageVersionAlternateElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; propertyGroupNode.AppendChild(newPackageVersionElement); addedPackageVersionElement = true; break; } else if (!addedPackageNameElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsPackageElementSuffix)) { propertyGroupNode.AppendChild(newPackageNameElement); addedPackageNameElement = true; break; } } if (addedPackageVersionElement && addedPackageNameElement) { break; } } } // Add the property groups if none were present if (!addedPackageVersionElement) { // If the repository doesn't have any package version element, then // use the non-alternate element name. XmlNode newPackageVersionElement = versionProps.CreateElement(packageVersionElementName, documentNamespaceUri); newPackageVersionElement.InnerText = dependency.Version; XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri); XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package versions"); versionProps.DocumentElement.AppendChild(propertyGroupCommentElement); versionProps.DocumentElement.AppendChild(propertyGroupElement); propertyGroupElement.AppendChild(newPackageVersionElement); } if (!addedPackageNameElement) { XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri); XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package names"); versionProps.DocumentElement.AppendChild(propertyGroupCommentElement); versionProps.DocumentElement.AppendChild(propertyGroupElement); propertyGroupElement.AppendChild(newPackageNameElement); } // TODO: This should not be done here. This should return some kind of generic file container to the caller, // who will gather up all updates and then call the git client to write the files all at once: // https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so // it's okay for now. var file = new GitFile(VersionFiles.VersionProps, versionProps); await _gitClient.PushFilesAsync(new List <GitFile> { file }, repo, branch, $"Add {dependency} to " + $"'{VersionFiles.VersionProps}'"); _logger.LogInformation( $"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " + $"'{VersionFiles.VersionProps}'"); }
public DependencyGraphNode(DependencyDetail dependencyDetail) : this( dependencyDetail, new HashSet <string>()) { }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { // TODO: This should use known updaters, but today the updaters for global.json can only // add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095 /*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote); * foreach (DependencyDetail dependency in dependencies) * { * if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function)) * { * await (Task)function.DynamicInvoke(_fileManager, _repo, dependency); * } * else * { * defaultUpdates.Add(dependency); * } * }*/ var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); // TODO: This needs to be moved into some consistent handling between local/remote and add/update: // https://github.com/dotnet/arcade/issues/1095 // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null) { try { IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger); List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common"); foreach (GitFile file in localEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { file.Operation = GitFileOperation.Delete; filesToUpdate.Add(file); } } } catch (Exception exc) when (exc.Message == "Not Found") { _logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " + "where a packages folder was passed and the commit which generated them is not " + "yet pushed."); } } // Push on local does not commit. await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null); }
public static async Task UpdateGlobalJson(GitFileManager fileManager, string repository, DependencyDetail dependency) { Dictionary <string, string> dependencyMapping = new Dictionary <string, string> { { "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" }, { "dotnet", "tools" } }; if (!dependencyMapping.ContainsKey(dependency.Name)) { throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined."); } string parent = dependencyMapping[dependency.Name]; await fileManager.AddDependencyToGlobalJson(Path.Combine(repository, VersionFilePath.GlobalJson), parent, dependency.Name, dependency.Version); await fileManager.AddDependencyToVersionDetails(Path.Combine(repository, VersionFilePath.VersionDetailsXml), dependency, DependencyType.Toolset); }
public bool Equals(Asset x, DependencyDetail y) { return(x.Name.Equals(y.Name, StringComparison.OrdinalIgnoreCase) && x.Version == y.Version); }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { // Read the current dependency files and grab their locations so that nuget.config can be updated appropriately. // Update the incoming dependencies with locations. IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync(); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger); await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies); await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies); // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); SemanticVersion targetDotNetVersion = null; IRemote arcadeRemote = null; if (arcadeItem != null) { arcadeRemote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger); targetDotNetVersion = await arcadeRemote.GetToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit); } var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies, targetDotNetVersion); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); if (arcadeItem != null) { try { List <GitFile> engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common"); foreach (GitFile file in localEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { // This is a file in the repo's eng/common folder that isn't present in Arcade at the // requested SHA so delete it during the update. // GitFile instances do not have public setters since we insert/retrieve them from an // In-memory cache during remote updates and we don't want anything to modify the cached, // references, so add a copy with a Delete FileOperation. filesToUpdate.Add(new GitFile( file.FilePath, file.Content, file.ContentEncoding, file.Mode, GitFileOperation.Delete)); } } } catch (Exception exc) when (exc.Message == "Not Found") { _logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " + "where a packages folder was passed and the commit which generated them is not " + "yet pushed."); } } // Push on local does not commit. await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null); }
private IEnumerable <DependencyDetail> GetDependencyDetails(XmlDocument document, string branch = null, bool includePinned = true) { List <DependencyDetail> dependencyDetails = new List <DependencyDetail>(); if (document != null) { BuildDependencies(document.DocumentElement.SelectNodes("//Dependency")); void BuildDependencies(XmlNodeList dependencies) { if (dependencies.Count > 0) { foreach (XmlNode dependency in dependencies) { if (dependency.NodeType != XmlNodeType.Comment && dependency.NodeType != XmlNodeType.Whitespace) { DependencyType type; switch (dependency.ParentNode.Name) { case "ProductDependencies": type = DependencyType.Product; break; case "ToolsetDependencies": type = DependencyType.Toolset; break; default: throw new DarcException($"Unknown dependency type '{dependency.ParentNode.Name}'"); } bool isPinned = false; // If the 'Pinned' attribute does not exist or if it is set to false we just not update it if (dependency.Attributes[VersionFiles.PinnedAttributeName] != null) { if (!bool.TryParse(dependency.Attributes[VersionFiles.PinnedAttributeName].Value, out isPinned)) { throw new DarcException($"The '{VersionFiles.PinnedAttributeName}' attribute is set but the value " + $"'{dependency.Attributes[VersionFiles.PinnedAttributeName].Value}' " + $"is not a valid boolean..."); } } DependencyDetail dependencyDetail = new DependencyDetail { Name = dependency.Attributes[VersionFiles.NameAttributeName].Value, RepoUri = dependency.SelectSingleNode(VersionFiles.UriElementName).InnerText, Commit = dependency.SelectSingleNode(VersionFiles.ShaElementName)?.InnerText, Version = dependency.Attributes[VersionFiles.VersionAttributeName].Value, CoherentParentDependencyName = dependency.Attributes[VersionFiles.CoherentParentAttributeName]?.Value, Pinned = isPinned, Type = type }; dependencyDetails.Add(dependencyDetail); } } } } } else { _logger.LogError($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " + $"Look for exceptions above."); } if (includePinned) { return(dependencyDetails); } return(dependencyDetails.Where(d => !d.Pinned)); }
public async Task CommitUpdatesAsync( string repoUri, string branch, List <DependencyDetail> itemsToUpdate, string message) { CheckForValidGitClient(); GitFileContentContainer fileContainer = await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch); List <GitFile> filesToCommit = new List <GitFile>(); // If we are updating the arcade sdk we need to update the eng/common files and the dotnet versions as well DependencyDetail arcadeItem = itemsToUpdate.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null && repoUri != arcadeItem.RepoUri) { // Files in arcade repository. All Arcade items have a GitHub repo URI by default so we need to change the // URI from we are getting the eng/common files. If in an AzDO context we change the URI to that of // dotnet-arcade in dnceng string arcadeRepoUri = arcadeItem.RepoUri; if (Uri.TryCreate(repoUri, UriKind.Absolute, out Uri parsedUri)) { if (parsedUri.Host == "dev.azure.com" || parsedUri.Host.EndsWith("visualstudio.com")) { arcadeRepoUri = "https://dev.azure.com/dnceng/internal/_git/dotnet-arcade"; } } SemanticVersion arcadeDotnetVersion = await GetToolsDotnetVersionAsync(arcadeRepoUri, arcadeItem.Commit); if (arcadeDotnetVersion != null) { fileContainer.GlobalJson = UpdateDotnetVersionGlobalJson(arcadeDotnetVersion, fileContainer.GlobalJson); } List <GitFile> engCommonFiles = await GetCommonScriptFilesAsync(arcadeRepoUri, arcadeItem.Commit); filesToCommit.AddRange(engCommonFiles); // Files in the target repo string latestCommit = await _gitClient.GetLastCommitShaAsync(repoUri, branch); List <GitFile> targetEngCommonFiles = await GetCommonScriptFilesAsync(repoUri, latestCommit); foreach (GitFile file in targetEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { file.Operation = GitFileOperation.Delete; filesToCommit.Add(file); } } } filesToCommit.AddRange(fileContainer.GetFilesToCommit()); await _gitClient.CommitFilesAsync(filesToCommit, repoUri, branch, message); }
/// <summary> /// Determine what dependencies need to be updated given an input set of assets. /// </summary> /// <param name="sourceCommit">Commit the assets come from.</param> /// <param name="assets">Assets as inputs for the update.</param> /// <param name="dependencies">Current set of the dependencies.</param> /// <param name="sourceRepoUri">Repository the assets came from.</param> /// <returns>Map of existing dependency->updated dependency</returns> public Task <List <DependencyUpdate> > GetRequiredNonCoherencyUpdatesAsync( string sourceRepoUri, string sourceCommit, IEnumerable <AssetData> assets, IEnumerable <DependencyDetail> dependencies) { Dictionary <DependencyDetail, DependencyDetail> toUpdate = new Dictionary <DependencyDetail, DependencyDetail>(); // Walk the assets, finding the dependencies that don't have coherency markers. // those must be updated in a second pass. foreach (AssetData asset in assets) { DependencyDetail matchingDependencyByName = dependencies.FirstOrDefault(d => d.Name.Equals(asset.Name, StringComparison.OrdinalIgnoreCase) && string.IsNullOrEmpty(d.CoherentParentDependencyName)); if (matchingDependencyByName == null) { continue; } // If the dependency is pinned, don't touch it. if (matchingDependencyByName.Pinned) { continue; } // Build might contain multiple assets of the same name if (toUpdate.ContainsKey(matchingDependencyByName)) { continue; } // Check if an update is actually needed. // Case-sensitive compare as case-correction is desired. if (matchingDependencyByName.Name == asset.Name && matchingDependencyByName.Version == asset.Version && matchingDependencyByName.Commit == sourceCommit && matchingDependencyByName.RepoUri == sourceRepoUri) { continue; } DependencyDetail newDependency = new DependencyDetail(matchingDependencyByName) { Commit = sourceCommit, RepoUri = sourceRepoUri, Version = asset.Version, Name = asset.Name }; toUpdate.Add(matchingDependencyByName, newDependency); } List <DependencyUpdate> dependencyUpdates = toUpdate.Select(kv => new DependencyUpdate { From = kv.Key, To = kv.Value }).ToList(); return(Task.FromResult(dependencyUpdates)); }
/// <summary> /// Get updates required by coherency constraints. /// </summary> /// <param name="dependencies">Current set of dependencies.</param> /// <param name="remoteFactory">Remote factory for remote queries.</param> /// <returns>Dependencies with updates.</returns> public async Task <List <DependencyUpdate> > GetRequiredCoherencyUpdatesAsync( IEnumerable <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { List <DependencyUpdate> toUpdate = new List <DependencyUpdate>(); IEnumerable <DependencyDetail> leavesOfCoherencyTrees = CalculateLeavesOfCoherencyTrees(dependencies); if (!leavesOfCoherencyTrees.Any()) { // Nothing to do. return(toUpdate); } DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = true, LookupBuilds = true, NodeDiff = NodeDiff.None }; // Now make a walk over coherent dependencies. Note that coherent dependencies could make // a chain (A->B->C). In all cases we need to walk to the head of the chain, keeping track // of all elements in the chain. Also note that we are walking all dependencies here, not // just those that match the incoming AssetData and aligning all of these based on the coherency data. Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyDetail> visited = new HashSet <DependencyDetail>(); foreach (DependencyDetail dependency in leavesOfCoherencyTrees) { // If the dependency was already updated, then skip it (could have been part of a larger // dependency chain) if (visited.Contains(dependency)) { continue; } // Walk to head of dependency tree, keeping track of elements along the way. // If we hit a pinned dependency in the walk, that means we can't move // the dependency and therefore it is effectively the "head" of the subtree. // We will still visit all the elements in the chain eventually in this algorithm: // Consider A->B(pinned)->C(pinned)->D. List <DependencyDetail> updateList = new List <DependencyDetail>(); DependencyDetail currentDependency = dependency; while (!string.IsNullOrEmpty(currentDependency.CoherentParentDependencyName) && !currentDependency.Pinned) { updateList.Add(currentDependency); DependencyDetail parentCoherentDependency = dependencies.FirstOrDefault(d => d.Name.Equals(currentDependency.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase)); currentDependency = parentCoherentDependency ?? throw new DarcException($"Dependency {currentDependency.Name} has non-existent parent " + $"dependency {currentDependency.CoherentParentDependencyName}"); } DependencyGraphNode rootNode = null; // Build the graph to find the assets if we don't have the root in the cache. // The graph build is automatically broken when // all the desired assets are found (breadth first search). This means the cache may or // may not contain a complete graph for a given node. So, we first search the cache for the desired assets, // then if not found (or if there was no cache), we then build the graph from that node. bool nodeFromCache = nodeCache.TryGetValue($"{currentDependency.RepoUri}@{currentDependency.Commit}", out rootNode); if (!nodeFromCache) { _logger.LogInformation($"Node not found in cache, starting graph build at " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, updateList, nodeCache); } List <DependencyDetail> leftToFind = new List <DependencyDetail>(updateList); // Now do the lookup to find the element in the tree for each item in the update list foreach (DependencyDetail dependencyInUpdateChain in updateList) { (Asset coherentAsset, Build buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); // If we originally got the root node from the cache the graph may be incomplete. // Rebuild to attempt to find all the assets we have left to find. If we still can't find, or if // the root node did not come the cache, then we're in an invalid state. if (coherentAsset == null && nodeFromCache) { _logger.LogInformation($"Asset {dependencyInUpdateChain.Name} was not found in cached graph, rebuilding from " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, leftToFind, nodeCache); // And attempt to find again. (coherentAsset, buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); } if (coherentAsset == null) { // This is an invalid state. We can't satisfy the // constraints so they should either be removed or pinned. throw new DarcException($"Unable to update {dependencyInUpdateChain.Name} to have coherency with " + $"parent {dependencyInUpdateChain.CoherentParentDependencyName}. No matching asset found in tree. " + $"Either remove the coherency attribute or mark as pinned."); } else { leftToFind.Remove(dependencyInUpdateChain); } string buildRepoUri = buildForAsset.GitHubRepository ?? buildForAsset.AzureDevOpsRepository; if (dependencyInUpdateChain.Name == coherentAsset.Name && dependencyInUpdateChain.Version == coherentAsset.Version && dependencyInUpdateChain.Commit == buildForAsset.Commit && dependencyInUpdateChain.RepoUri == buildRepoUri) { continue; } DependencyDetail updatedDependency = new DependencyDetail(dependencyInUpdateChain) { Name = coherentAsset.Name, Version = coherentAsset.Version, RepoUri = buildRepoUri, Commit = buildForAsset.Commit }; toUpdate.Add(new DependencyUpdate { From = dependencyInUpdateChain, To = updatedDependency }); visited.Add(dependencyInUpdateChain); } } return(toUpdate); }