private static async Task DoLatestInChannelGraphNodeDiffAsync( IRemoteFactory remoteFactory, ILogger logger, Dictionary <string, DependencyGraphNode> nodeCache, Dictionary <string, DependencyGraphNode> visitedRepoUriNodes) { logger.LogInformation("Running latest in channel node diff."); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); // Walk each node in the graph and diff against the latest build in the channel // that was also applied to the node. Dictionary <string, string> latestCommitCache = new Dictionary <string, string>(); foreach (DependencyGraphNode node in nodeCache.Values) { // Start with an unknown diff. node.DiffFrom = GitDiff.UnknownDiff(); if (node.ContributingBuilds.Any()) { // Choose latest build of node that has a channel. Build newestBuildWithChannel = node.ContributingBuilds.OrderByDescending(b => b.DateProduced).FirstOrDefault( b => b.Channels != null && b.Channels.Any()); // If no build was found (e.g. build was flowed without a channel or channel was removed from // a build, then no diff from latest. if (newestBuildWithChannel != null) { int channelId = newestBuildWithChannel.Channels.First().Id; // Just choose the first channel. This algorithm is mostly just heuristic. string latestCommitKey = $"{node.Repository}@{channelId}"; string latestCommit = null; if (!latestCommitCache.TryGetValue(latestCommitKey, out latestCommit)) { // Look up latest build in the channel var latestBuild = await barOnlyRemote.GetLatestBuildAsync(node.Repository, channelId); // Could be null, if the only build was removed from the channel if (latestBuild != null) { latestCommit = latestBuild.Commit; } // Add to cache latestCommitCache.Add(latestCommitKey, latestCommit); } // Perform diff if there is a latest commit. if (!string.IsNullOrEmpty(latestCommit)) { IRemote repoRemote = await remoteFactory.GetRemoteAsync(node.Repository, logger); // This will return a no-diff if latestCommit == node.Commit node.DiffFrom = await repoRemote.GitDiffAsync(node.Repository, latestCommit, node.Commit); } } } } }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { // Read the current dependency files and grab their locations so that nuget.config can be updated appropriately. // Update the incoming dependencies with locations. IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync(); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger); await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies); await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies); var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); // TODO: This needs to be moved into some consistent handling between local/remote and add/update: // https://github.com/dotnet/arcade/issues/1095 // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null) { try { IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger); List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common"); foreach (GitFile file in localEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { file.Operation = GitFileOperation.Delete; filesToUpdate.Add(file); } } } catch (Exception exc) when (exc.Message == "Not Found") { _logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " + "where a packages folder was passed and the commit which generated them is not " + "yet pushed."); } } // Push on local does not commit. await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null); }
/// <summary> /// Diff each node in the graph against the latest build in /// the graph. /// </summary> /// <param name="remoteFactory"></param> /// <param name="logger"></param> /// <param name="nodeCache"></param> /// <param name="visitedRepoUriNodes"></param> /// <returns></returns> private static async Task DoLatestInGraphNodeDiffAsync( IRemoteFactory remoteFactory, ILogger logger, Dictionary <string, DependencyGraphNode> nodeCache, Dictionary <string, DependencyGraphNode> visitedRepoUriNodes) { logger.LogInformation("Running latest in graph node diff."); // Find the build of each repo in the graph, then // get the diff info from the latest foreach (string repo in visitedRepoUriNodes.Keys) { // Get all nodes with this value List <DependencyGraphNode> nodes = nodeCache.Values.Where(n => n.Repository == repo).ToList(); // If only one, determine latest if (nodes.Count > 1) { // Find latest DependencyGraphNode newestNode = null; Build newestBuild = null; foreach (DependencyGraphNode node in nodes) { if (newestNode == null) { newestNode = node; if (newestNode.ContributingBuilds.Any()) { newestBuild = newestNode.ContributingBuilds.OrderByDescending(b => b.DateProduced).First(); } } else if (node.ContributingBuilds.Any(b => b.DateProduced > newestBuild?.DateProduced)) { newestNode = node; newestBuild = newestNode.ContributingBuilds.OrderByDescending(b => b.DateProduced).First(); } } // Compare all other nodes to the latest foreach (DependencyGraphNode node in nodes) { IRemote repoRemote = await remoteFactory.GetRemoteAsync(node.Repository, logger); // If node == newestNode, returns no diff. node.DiffFrom = await repoRemote.GitDiffAsync(node.Repository, newestNode.Commit, node.Commit); } } else { DependencyGraphNode singleNode = nodes.Single(); singleNode.DiffFrom = GitDiff.NoDiff(singleNode.Commit); } } }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { // TODO: This should use known updaters, but today the updaters for global.json can only // add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095 /*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote); * foreach (DependencyDetail dependency in dependencies) * { * if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function)) * { * await (Task)function.DynamicInvoke(_fileManager, _repo, dependency); * } * else * { * defaultUpdates.Add(dependency); * } * }*/ var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); // TODO: This needs to be moved into some consistent handling between local/remote and add/update: // https://github.com/dotnet/arcade/issues/1095 // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); if (arcadeItem != null) { try { IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger); List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common"); foreach (GitFile file in localEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { file.Operation = GitFileOperation.Delete; filesToUpdate.Add(file); } } } catch (Exception exc) when (exc.Message == "Not Found") { _logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " + "where a packages folder was passed and the commit which generated them is not " + "yet pushed."); } } // Push on local does not commit. await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null); }
/// <summary> /// Updates existing dependencies in the dependency files /// </summary> /// <param name="dependencies">Dependencies that need updates.</param> /// <param name="remote">Remote instance for gathering eng/common script updates.</param> /// <returns></returns> public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { // Read the current dependency files and grab their locations so that nuget.config can be updated appropriately. // Update the incoming dependencies with locations. IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync(); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger); await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies); await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies); // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.FirstOrDefault( i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase)); SemanticVersion targetDotNetVersion = null; IRemote arcadeRemote = null; if (arcadeItem != null) { arcadeRemote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger); targetDotNetVersion = await arcadeRemote.GetToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit); } var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies, targetDotNetVersion); List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit(); if (arcadeItem != null) { try { List <GitFile> engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); filesToUpdate.AddRange(engCommonFiles); List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common"); foreach (GitFile file in localEngCommonFiles) { if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any()) { // This is a file in the repo's eng/common folder that isn't present in Arcade at the // requested SHA so delete it during the update. // GitFile instances do not have public setters since we insert/retrieve them from an // In-memory cache during remote updates and we don't want anything to modify the cached, // references, so add a copy with a Delete FileOperation. filesToUpdate.Add(new GitFile( file.FilePath, file.Content, file.ContentEncoding, file.Mode, GitFileOperation.Delete)); } } } catch (Exception exc) when (exc.Message == "Not Found") { _logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " + "where a packages folder was passed and the commit which generated them is not " + "yet pushed."); } } // Push on local does not commit. await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null); }
private static async Task <IEnumerable <DependencyDetail> > GetDependenciesAsync( IRemoteFactory remoteFactory, bool remote, ILogger logger, string repoUri, string commit, bool includeToolset, IEnumerable <string> remotesMap, string reposFolder, string testPath = null) { try { IEnumerable <DependencyDetail> dependencies = null; if (!string.IsNullOrEmpty(testPath)) { testPath = Path.Combine( testPath, repoUri, commit); if (Directory.Exists(testPath)) { Local local = new Local(logger, testPath); dependencies = await local.GetDependenciesAsync(); } } else if (remote) { IRemote remoteClient = await remoteFactory.GetRemoteAsync(repoUri, logger); dependencies = await remoteClient.GetDependenciesAsync( repoUri, commit); } else { string repoPath = GetRepoPath(repoUri, commit, remotesMap, reposFolder, logger); if (!string.IsNullOrEmpty(repoPath)) { Local local = new Local(logger); string fileContents = LocalHelpers.GitShow( repoPath, commit, VersionFiles.VersionDetailsXml, logger); dependencies = local.GetDependenciesFromFileContents(fileContents); } } if (!includeToolset) { dependencies = dependencies.Where(dependency => dependency.Type != DependencyType.Toolset); } return(dependencies); } catch (DependencyFileNotFoundException) { // This is not an error. Dependencies can be specified with explicit shas that // may not have eng/Version.Details.xml at that point. logger.LogWarning($"{repoUri}@{commit} does not have an eng/Version.Details.xml."); return(null); } catch (Exception exc) { logger.LogError(exc, $"Something failed while trying the fetch the " + $"dependencies of repo '{repoUri}' at sha " + $"'{commit}'"); throw; } }