/// <summary> /// Creates a new dependency graph /// </summary> /// <param name="remoteFactory">Remote for factory for obtaining remotes to</param> /// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param> /// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param> /// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param> /// <param name="includeToolset">If true, toolset dependencies are included.</param> /// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param> /// <param name="remote">If true, remote graph build is used.</param> /// <param name="logger">Logger</param> /// <param name="reposFolder">Path to repos</param> /// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param> /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param> /// <returns>New dependency graph</returns> private static async Task <DependencyGraph> BuildDependencyGraphImplAsync( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, bool remote, ILogger logger, string reposFolder, IEnumerable <string> remotesMap, string testPath) { ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit, options, remote, logger, reposFolder, remotesMap, testPath); if (rootDependencies != null) { logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " + $"({repoUri}@{commit})"); foreach (DependencyDetail dependency in rootDependencies) { logger.LogInformation($" {dependency.Name}@{dependency.Version}"); } } else { logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})"); } AssetComparer assetEqualityComparer = new AssetComparer(); HashSet <Build> allContributingBuilds = null; HashSet <DependencyDetail> dependenciesMissingBuilds = null; HashSet <Build> rootNodeBuilds = null; Dictionary <DependencyDetail, Build> dependencyCache = new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer()); List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >(); EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type; HashSet <string> breakOn = null; if (breakOnType != EarlyBreakOnType.None) { breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase); } if (options.LookupBuilds) { allContributingBuilds = new HashSet <Build>(new BuildComparer()); dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer()); rootNodeBuilds = new HashSet <Build>(new BuildComparer()); // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit); // Filter by those actually producing the root dependencies, if they were supplied if (rootDependencies != null) { potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b => b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d)))); } // It's entirely possible that the root has no builds (e.g. change just checked in). // Don't record those. Instead, users of the graph should just look at the // root node's contributing builds and determine whether it's empty or not. foreach (Build build in potentialRootNodeBuilds) { allContributingBuilds.Add(build); rootNodeBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } // Create the root node and add the repo to the visited bit vector. DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds); rootGraphNode.VisitedNodes.Add(repoUri); // Nodes to visit is a queue, so that the evaluation order // of the graph is breadth first. Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>(); nodesToVisit.Enqueue(rootGraphNode); HashSet <DependencyDetail> uniqueDependencyDetails; if (rootGraphNode.Dependencies != null) { uniqueDependencyDetails = new HashSet <DependencyDetail>( rootGraphNode.Dependencies, new DependencyDetailComparer()); // Remove the dependencies details from the // break on if break on type is Dependencies if (breakOnType == EarlyBreakOnType.Dependencies) { rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name)); } } else { uniqueDependencyDetails = new HashSet <DependencyDetail>( new DependencyDetailComparer()); } // If we already found the assets/dependencies we wanted, clear the // visit list and we'll drop through. if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); } // Cache of nodes we've visited. If we reach a repo/commit combo already in the cache, // we can just add these nodes as a child. The cache key is '{repoUri}@{commit}' Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode); // Cache of incoherent nodes, looked up by repo URI. Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>(); // Cache of incoherent dependencies, looked up by name Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>(); HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>(); while (nodesToVisit.Count > 0) { DependencyGraphNode node = nodesToVisit.Dequeue(); logger.LogInformation($"Visiting {node.Repository}@{node.Commit}"); IEnumerable <DependencyDetail> dependencies; // In case of the root node which is initially put on the stack, // we already have the set of dependencies to start at (this may have been // filtered by the caller). So no need to get the dependencies again. if (node.Dependencies != null) { dependencies = node.Dependencies; } else { logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}"); dependencies = await GetDependenciesAsync( remoteFactory, remote, logger, options.GitExecutable, node.Repository, node.Commit, options.IncludeToolset, remotesMap, reposFolder, testPath); // Set the dependencies on the current node. node.Dependencies = dependencies; } if (dependencies != null) { foreach (DependencyDetail dependency in dependencies) { // If this dependency is missing information, then skip it. if (string.IsNullOrEmpty(dependency.RepoUri) || string.IsNullOrEmpty(dependency.Commit)) { logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " + $"{node.Repository}@{node.Commit} " + $"is missing repository uri or commit information, skipping"); continue; } // If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph // and should break. if (node.VisitedNodes.Contains(dependency.RepoUri)) { logger.LogInformation($"Node {node.Repository}@{node.Commit} " + $"introduces a cycle to {dependency.RepoUri}, skipping"); if (options.ComputeCyclePaths) { var newCycles = ComputeCyclePaths(node, dependency.RepoUri); cycles.AddRange(newCycles); } continue; } // Add the individual dependency to the set of unique dependencies seen // in the whole graph. uniqueDependencyDetails.Add(dependency); if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency)) { incoherentDependencies.Add(existingDependency); incoherentDependencies.Add(dependency); } else { incoherentDependenciesCache.Add(dependency.Name, dependency); } HashSet <Build> nodeContributingBuilds = null; if (options.LookupBuilds) { nodeContributingBuilds = new HashSet <Build>(new BuildComparer()); // Look up dependency in cache first if (dependencyCache.TryGetValue(dependency, out Build existingBuild)) { nodeContributingBuilds.Add(existingBuild); allContributingBuilds.Add(existingBuild); } else { // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit); // Filter by those actually producing the dependency. Most of the time this won't // actually result in a different set of contributing builds, but should avoid any subtle bugs where // there might be overlap between repos, or cases where there were multiple builds at the same sha. potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b => b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency))); if (!potentiallyContributingBuilds.Any()) { // Couldn't find a build that produced the dependency. dependenciesMissingBuilds.Add(dependency); } else { foreach (Build build in potentiallyContributingBuilds) { allContributingBuilds.Add(build); nodeContributingBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } } } // We may have visited this node before. If so, add it as a child and avoid additional walks. // Update the list of contributing builds. if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode)) { if (options.LookupBuilds) { // Add the contributing builds. It's possible that // different dependencies on a single node (repo/sha) were produced // from multiple builds foreach (Build build in nodeContributingBuilds) { existingNode.ContributingBuilds.Add(build); } } logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child"); node.AddChild(existingNode, dependency); continue; } // Otherwise, create a new node for this dependency. DependencyGraphNode newNode = new DependencyGraphNode( dependency.RepoUri, dependency.Commit, null, node.VisitedNodes, nodeContributingBuilds); // Cache the dependency and add it to the visitation stack. nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode); nodesToVisit.Enqueue(newNode); newNode.VisitedNodes.Add(dependency.RepoUri); node.AddChild(newNode, dependency); // Calculate incoherencies. If we've not yet visited the repo uri, add the // new node based on its repo uri. Otherwise, add both the new node and the visited // node to the incoherent nodes. if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode)) { incoherentNodes.Add(visitedNode); incoherentNodes.Add(newNode); } else { visitedRepoUriNodes.Add(newNode.Repository, newNode); } // If breaking on dependencies, then decide whether we need to break // here. if (breakOnType == EarlyBreakOnType.Dependencies) { breakOn.Remove(dependency.Name); } if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); break; } } } } switch (options.NodeDiff) { case NodeDiff.None: // Nothing break; case NodeDiff.LatestInGraph: await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; case NodeDiff.LatestInChannel: await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; } return(new DependencyGraph(rootGraphNode, uniqueDependencyDetails, incoherentDependencies, nodeCache.Values, incoherentNodes, allContributingBuilds, dependenciesMissingBuilds, cycles)); }
/// <summary> /// Given an asset name, find the asset in the dependency tree. /// Returns the asset with the shortest path to the root node. /// </summary> /// <param name="assetName">Name of asset.</param> /// <param name="currentNode">Dependency graph node to find the asset in.</param> /// <returns>(Asset, Build, depth), or (null, null, maxint) if not found.</returns> private (Asset asset, Build build, int buildDepth) FindAssetInBuildTree(string assetName, DependencyGraphNode currentNode, int currentDepth) { foreach (Build build in currentNode.ContributingBuilds) { Asset matchingAsset = build.Assets.FirstOrDefault(a => a.Name.Equals(assetName, StringComparison.OrdinalIgnoreCase)); if (matchingAsset != null) { return(matchingAsset, build, currentDepth); } } // Walk child nodes Asset shallowestAsset = null; Build shallowestBuild = null; int shallowestBuildDepth = int.MaxValue; foreach (DependencyGraphNode childNode in currentNode.Children) { (Asset asset, Build build, int buildDepth) = FindAssetInBuildTree(assetName, childNode, currentDepth + 1); if (asset != null) { if (buildDepth < shallowestBuildDepth) { shallowestAsset = asset; shallowestBuild = build; shallowestBuildDepth = buildDepth; } } } return(shallowestAsset, shallowestBuild, shallowestBuildDepth); }
/// <summary> /// Given an asset name, find the asset in the dependency tree. /// Returns the asset with the shortest path to the root node. /// </summary> /// <param name="assetName">Name of asset.</param> /// <param name="currentNode">Dependency graph node to find the asset in.</param> /// <returns>(Asset, Build), or (null, null) if not found.</returns> private (Asset asset, Build build) FindAssetInBuildTree(string assetName, DependencyGraphNode currentNode) { (Asset asset, Build build, int depth) = FindAssetInBuildTree(assetName, currentNode, 0); return(asset, build); }
/// <summary> /// Get updates required by coherency constraints. /// </summary> /// <param name="dependencies">Current set of dependencies.</param> /// <param name="remoteFactory">Remote factory for remote queries.</param> /// <returns>Dependencies with updates.</returns> public async Task <List <DependencyUpdate> > GetRequiredCoherencyUpdatesAsync( IEnumerable <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { List <DependencyUpdate> toUpdate = new List <DependencyUpdate>(); IEnumerable <DependencyDetail> leavesOfCoherencyTrees = CalculateLeavesOfCoherencyTrees(dependencies); if (!leavesOfCoherencyTrees.Any()) { // Nothing to do. return(toUpdate); } DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = true, LookupBuilds = true, NodeDiff = NodeDiff.None }; // Now make a walk over coherent dependencies. Note that coherent dependencies could make // a chain (A->B->C). In all cases we need to walk to the head of the chain, keeping track // of all elements in the chain. Also note that we are walking all dependencies here, not // just those that match the incoming AssetData and aligning all of these based on the coherency data. Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyDetail> visited = new HashSet <DependencyDetail>(); foreach (DependencyDetail dependency in leavesOfCoherencyTrees) { // If the dependency was already updated, then skip it (could have been part of a larger // dependency chain) if (visited.Contains(dependency)) { continue; } // Walk to head of dependency tree, keeping track of elements along the way. // If we hit a pinned dependency in the walk, that means we can't move // the dependency and therefore it is effectively the "head" of the subtree. // We will still visit all the elements in the chain eventually in this algorithm: // Consider A->B(pinned)->C(pinned)->D. List <DependencyDetail> updateList = new List <DependencyDetail>(); DependencyDetail currentDependency = dependency; while (!string.IsNullOrEmpty(currentDependency.CoherentParentDependencyName) && !currentDependency.Pinned) { updateList.Add(currentDependency); DependencyDetail parentCoherentDependency = dependencies.FirstOrDefault(d => d.Name.Equals(currentDependency.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase)); currentDependency = parentCoherentDependency ?? throw new DarcException($"Dependency {currentDependency.Name} has non-existent parent " + $"dependency {currentDependency.CoherentParentDependencyName}"); } DependencyGraphNode rootNode = null; // Build the graph to find the assets if we don't have the root in the cache. // The graph build is automatically broken when // all the desired assets are found (breadth first search). This means the cache may or // may not contain a complete graph for a given node. So, we first search the cache for the desired assets, // then if not found (or if there was no cache), we then build the graph from that node. bool nodeFromCache = nodeCache.TryGetValue($"{currentDependency.RepoUri}@{currentDependency.Commit}", out rootNode); if (!nodeFromCache) { _logger.LogInformation($"Node not found in cache, starting graph build at " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, updateList, nodeCache); } List <DependencyDetail> leftToFind = new List <DependencyDetail>(updateList); // Now do the lookup to find the element in the tree for each item in the update list foreach (DependencyDetail dependencyInUpdateChain in updateList) { (Asset coherentAsset, Build buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); // If we originally got the root node from the cache the graph may be incomplete. // Rebuild to attempt to find all the assets we have left to find. If we still can't find, or if // the root node did not come the cache, then we're in an invalid state. if (coherentAsset == null && nodeFromCache) { _logger.LogInformation($"Asset {dependencyInUpdateChain.Name} was not found in cached graph, rebuilding from " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, leftToFind, nodeCache); // And attempt to find again. (coherentAsset, buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); } if (coherentAsset == null) { // This is an invalid state. We can't satisfy the // constraints so they should either be removed or pinned. throw new DarcException($"Unable to update {dependencyInUpdateChain.Name} to have coherency with " + $"parent {dependencyInUpdateChain.CoherentParentDependencyName}. No matching asset found in tree. " + $"Either remove the coherency attribute or mark as pinned."); } else { leftToFind.Remove(dependencyInUpdateChain); } string buildRepoUri = buildForAsset.GitHubRepository ?? buildForAsset.AzureDevOpsRepository; if (dependencyInUpdateChain.Name == coherentAsset.Name && dependencyInUpdateChain.Version == coherentAsset.Version && dependencyInUpdateChain.Commit == buildForAsset.Commit && dependencyInUpdateChain.RepoUri == buildRepoUri) { continue; } DependencyDetail updatedDependency = new DependencyDetail(dependencyInUpdateChain) { Name = coherentAsset.Name, Version = coherentAsset.Version, RepoUri = buildRepoUri, Commit = buildForAsset.Commit }; toUpdate.Add(new DependencyUpdate { From = dependencyInUpdateChain, To = updatedDependency }); visited.Add(dependencyInUpdateChain); } } return(toUpdate); }