private async Task <DependencyGraphNode> BuildGraphAtDependency( IRemoteFactory remoteFactory, DependencyDetail rootDependency, List <DependencyDetail> updateList, Dictionary <string, DependencyGraphNode> nodeCache) { DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = true, LookupBuilds = true, NodeDiff = NodeDiff.None, EarlyBuildBreak = new EarlyBreakOn { Type = EarlyBreakOnType.Assets, BreakOn = new List <string>(updateList.Select(d => d.Name)) } }; DependencyGraph dependencyGraph = await DependencyGraph.BuildRemoteDependencyGraphAsync( remoteFactory, null, rootDependency.RepoUri, rootDependency.Commit, dependencyGraphBuildOptions, _logger); // Cache all nodes in this built graph. foreach (DependencyGraphNode node in dependencyGraph.Nodes) { if (!nodeCache.ContainsKey($"{node.Repository}@{node.Commit}")) { nodeCache.Add($"{node.Repository}@{node.Commit}", node); } } return(dependencyGraph.Root); }
/// <summary> /// Builds a dependency graph given a root repo and commit using remotes. /// </summary> /// <param name="remoteFactory">Factory that can create remotes based on repo uris</param> /// <param name="repoUri">Root repository URI</param> /// <param name="commit">Root commit</param> /// <param name="options">Graph build options.</param> /// <param name="logger">Logger</param> /// <returns>New dependency graph.</returns> public static async Task <DependencyGraph> BuildRemoteDependencyGraphAsync( IRemoteFactory remoteFactory, string repoUri, string commit, DependencyGraphBuildOptions options, ILogger logger) { return(await BuildDependencyGraphImplAsync( remoteFactory, null, /* no initial root dependencies */ repoUri, commit, options, true, logger, null, null, null)); }
/// <summary> /// Validate that the graph build options are correct. /// </summary> /// <param name="remoteFactory"></param> /// <param name="rootDependencies"></param> /// <param name="repoUri"></param> /// <param name="commit"></param> /// <param name="options"></param> /// <param name="remote"></param> /// <param name="logger"></param> /// <param name="reposFolder"></param> /// <param name="remotesMap"></param> /// <param name="testPath"></param> private static void ValidateBuildOptions( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, bool remote, ILogger logger, string reposFolder, IEnumerable <string> remotesMap, string testPath) { // Fail fast if darcSettings is null in a remote scenario if (remote && remoteFactory == null) { throw new DarcException("Remote graph build requires a remote factory."); } if (rootDependencies != null && !rootDependencies.Any()) { throw new DarcException("Root dependencies were not supplied."); } if (!remote) { if (options.LookupBuilds) { throw new DarcException("Build lookup only available in remote build mode."); } if (options.NodeDiff != NodeDiff.None) { throw new DarcException($"Node diff type '{options.NodeDiff}' only available in remote build mode."); } } else { if (options.NodeDiff != NodeDiff.None && !options.LookupBuilds) { throw new DarcException("Node diff requires build lookup."); } } }
/// <summary> /// Builds a dependency graph given a root repo and commit. /// </summary> /// <param name="remoteFactory">Factory that can create remotes based on repo uris</param> /// <param name="rootDependencies">Root set of dependencies</param> /// <param name="repoUri">Root repository URI</param> /// <param name="commit">Root commit</param> /// <param name="options">Graph build options.</param> /// <param name="logger">Logger</param> /// <returns>New dependency graph.</returns> public static async Task <DependencyGraph> BuildRemoteDependencyGraphAsync( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, ILogger logger) { return(await BuildDependencyGraphImplAsync( remoteFactory, rootDependencies, repoUri, commit, options, true, logger, null, null, null)); }
/// <summary> /// Builds a dependency graph using only local resources /// </summary> /// <param name="remoteFactory">Factory that can create remotes based on repo uris</param> /// <param name="rootDependencies">Root set of dependencies</param> /// <param name="rootRepoFolder">Root repository folder</param> /// <param name="rootRepoCommit">Root commit</param> /// <param name="options">Graph build options</param> /// <param name="logger">Logger</param> /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param> /// <param name="remotesMap">Map of remote uris to local paths</param> /// <param name="reposFolder">Folder containing local repositories.</param> /// <returns>New dependency graph.</returns> public static async Task <DependencyGraph> BuildLocalDependencyGraphAsync( IEnumerable <DependencyDetail> rootDependencies, DependencyGraphBuildOptions options, ILogger logger, string rootRepoFolder, string rootRepoCommit, string reposFolder, IEnumerable <string> remotesMap, string testPath = null) { return(await BuildDependencyGraphImplAsync( null, rootDependencies, rootRepoFolder, rootRepoCommit, options, false, logger, reposFolder, remotesMap, testPath)); }
/// <summary> /// Creates a new dependency graph /// </summary> /// <param name="remoteFactory">Remote for factory for obtaining remotes to</param> /// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param> /// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param> /// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param> /// <param name="includeToolset">If true, toolset dependencies are included.</param> /// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param> /// <param name="remote">If true, remote graph build is used.</param> /// <param name="logger">Logger</param> /// <param name="reposFolder">Path to repos</param> /// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param> /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param> /// <returns>New dependency graph</returns> private static async Task <DependencyGraph> BuildDependencyGraphImplAsync( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, bool remote, ILogger logger, string reposFolder, IEnumerable <string> remotesMap, string testPath) { ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit, options, remote, logger, reposFolder, remotesMap, testPath); if (rootDependencies != null) { logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " + $"({repoUri}@{commit})"); foreach (DependencyDetail dependency in rootDependencies) { logger.LogInformation($" {dependency.Name}@{dependency.Version}"); } } else { logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})"); } AssetComparer assetEqualityComparer = new AssetComparer(); HashSet <Build> allContributingBuilds = null; HashSet <DependencyDetail> dependenciesMissingBuilds = null; HashSet <Build> rootNodeBuilds = null; Dictionary <DependencyDetail, Build> dependencyCache = new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer()); List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >(); EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type; HashSet <string> breakOn = null; if (breakOnType != EarlyBreakOnType.None) { breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase); } if (options.LookupBuilds) { allContributingBuilds = new HashSet <Build>(new BuildComparer()); dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer()); rootNodeBuilds = new HashSet <Build>(new BuildComparer()); // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit); // Filter by those actually producing the root dependencies, if they were supplied if (rootDependencies != null) { potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b => b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d)))); } // It's entirely possible that the root has no builds (e.g. change just checked in). // Don't record those. Instead, users of the graph should just look at the // root node's contributing builds and determine whether it's empty or not. foreach (Build build in potentialRootNodeBuilds) { allContributingBuilds.Add(build); rootNodeBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } // Create the root node and add the repo to the visited bit vector. DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds); rootGraphNode.VisitedNodes.Add(repoUri); // Nodes to visit is a queue, so that the evaluation order // of the graph is breadth first. Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>(); nodesToVisit.Enqueue(rootGraphNode); HashSet <DependencyDetail> uniqueDependencyDetails; if (rootGraphNode.Dependencies != null) { uniqueDependencyDetails = new HashSet <DependencyDetail>( rootGraphNode.Dependencies, new DependencyDetailComparer()); // Remove the dependencies details from the // break on if break on type is Dependencies if (breakOnType == EarlyBreakOnType.Dependencies) { rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name)); } } else { uniqueDependencyDetails = new HashSet <DependencyDetail>( new DependencyDetailComparer()); } // If we already found the assets/dependencies we wanted, clear the // visit list and we'll drop through. if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); } // Cache of nodes we've visited. If we reach a repo/commit combo already in the cache, // we can just add these nodes as a child. The cache key is '{repoUri}@{commit}' Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode); // Cache of incoherent nodes, looked up by repo URI. Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>(); // Cache of incoherent dependencies, looked up by name Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>(); HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>(); while (nodesToVisit.Count > 0) { DependencyGraphNode node = nodesToVisit.Dequeue(); logger.LogInformation($"Visiting {node.Repository}@{node.Commit}"); IEnumerable <DependencyDetail> dependencies; // In case of the root node which is initially put on the stack, // we already have the set of dependencies to start at (this may have been // filtered by the caller). So no need to get the dependencies again. if (node.Dependencies != null) { dependencies = node.Dependencies; } else { logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}"); dependencies = await GetDependenciesAsync( remoteFactory, remote, logger, node.Repository, node.Commit, options.IncludeToolset, remotesMap, reposFolder, testPath); // Set the dependencies on the current node. node.Dependencies = dependencies; } if (dependencies != null) { foreach (DependencyDetail dependency in dependencies) { // If this dependency is missing information, then skip it. if (string.IsNullOrEmpty(dependency.RepoUri) || string.IsNullOrEmpty(dependency.Commit)) { logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " + $"{node.Repository}@{node.Commit} " + $"is missing repository uri or commit information, skipping"); continue; } // If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph // and should break. if (node.VisitedNodes.Contains(dependency.RepoUri)) { logger.LogInformation($"Node {node.Repository}@{node.Commit} " + $"introduces a cycle to {dependency.RepoUri}, skipping"); if (options.ComputeCyclePaths) { var newCycles = ComputeCyclePaths(node, dependency.RepoUri); cycles.AddRange(newCycles); } continue; } // Add the individual dependency to the set of unique dependencies seen // in the whole graph. uniqueDependencyDetails.Add(dependency); if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency)) { incoherentDependencies.Add(existingDependency); incoherentDependencies.Add(dependency); } else { incoherentDependenciesCache.Add(dependency.Name, dependency); } HashSet <Build> nodeContributingBuilds = null; if (options.LookupBuilds) { nodeContributingBuilds = new HashSet <Build>(new BuildComparer()); // Look up dependency in cache first if (dependencyCache.TryGetValue(dependency, out Build existingBuild)) { nodeContributingBuilds.Add(existingBuild); allContributingBuilds.Add(existingBuild); } else { // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit); // Filter by those actually producing the dependency. Most of the time this won't // actually result in a different set of contributing builds, but should avoid any subtle bugs where // there might be overlap between repos, or cases where there were multiple builds at the same sha. potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b => b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency))); if (!potentiallyContributingBuilds.Any()) { // Couldn't find a build that produced the dependency. dependenciesMissingBuilds.Add(dependency); } else { foreach (Build build in potentiallyContributingBuilds) { allContributingBuilds.Add(build); nodeContributingBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } } } // We may have visited this node before. If so, add it as a child and avoid additional walks. // Update the list of contributing builds. if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode)) { if (options.LookupBuilds) { // Add the contributing builds. It's possible that // different dependencies on a single node (repo/sha) were produced // from multiple builds foreach (Build build in nodeContributingBuilds) { existingNode.ContributingBuilds.Add(build); } } logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child"); node.AddChild(existingNode, dependency); continue; } // Otherwise, create a new node for this dependency. DependencyGraphNode newNode = new DependencyGraphNode( dependency.RepoUri, dependency.Commit, null, node.VisitedNodes, nodeContributingBuilds); // Cache the dependency and add it to the visitation stack. nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode); nodesToVisit.Enqueue(newNode); newNode.VisitedNodes.Add(dependency.RepoUri); node.AddChild(newNode, dependency); // Calculate incoherencies. If we've not yet visited the repo uri, add the // new node based on its repo uri. Otherwise, add both the new node and the visited // node to the incoherent nodes. if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode)) { incoherentNodes.Add(visitedNode); incoherentNodes.Add(newNode); } else { visitedRepoUriNodes.Add(newNode.Repository, newNode); } // If breaking on dependencies, then decide whether we need to break // here. if (breakOnType == EarlyBreakOnType.Dependencies) { breakOn.Remove(dependency.Name); } if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); break; } } } } switch (options.NodeDiff) { case NodeDiff.None: // Nothing break; case NodeDiff.LatestInGraph: await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; case NodeDiff.LatestInChannel: await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; } return(new DependencyGraph(rootGraphNode, uniqueDependencyDetails, incoherentDependencies, nodeCache.Values, incoherentNodes, allContributingBuilds, dependenciesMissingBuilds, cycles)); }
/// <summary> /// Creates a new dependency graph /// </summary> /// <param name="remoteFactory">Remote for factory for obtaining remotes to</param> /// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param> /// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param> /// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param> /// <param name="includeToolset">If true, toolset dependencies are included.</param> /// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param> /// <param name="remote">If true, remote graph build is used.</param> /// <param name="logger">Logger</param> /// <param name="reposFolder">Path to repos</param> /// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param> /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param> /// <returns>New dependency graph</returns> private static async Task <DependencyGraph> BuildDependencyGraphImplAsync( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, bool remote, ILogger logger, string reposFolder, IEnumerable <string> remotesMap, string testPath) { ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit, options, remote, logger, reposFolder, remotesMap, testPath); if (rootDependencies != null) { logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " + $"({repoUri}@{commit})"); foreach (DependencyDetail dependency in rootDependencies) { logger.LogInformation($" {dependency.Name}@{dependency.Version}"); } } else { logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})"); } IRemote barOnlyRemote = null; if (remote) { // Look up the dependency and get the creating build. barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); } List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >(); Dictionary <string, Task <IEnumerable <Build> > > buildLookupTasks = null; if (options.LookupBuilds) { buildLookupTasks = new Dictionary <string, Task <IEnumerable <Build> > >(); // Look up the dependency and get the creating build. buildLookupTasks.Add($"{repoUri}@{commit}", barOnlyRemote.GetBuildsAsync(repoUri, commit)); } // Create the root node and add the repo to the visited bit vector. List <Build> allContributingBuilds = null; DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, null); rootGraphNode.VisitedNodes.Add(repoUri); // Nodes to visit is a queue, so that the evaluation order // of the graph is breadth first. Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>(); nodesToVisit.Enqueue(rootGraphNode); HashSet <DependencyDetail> uniqueDependencyDetails; if (rootGraphNode.Dependencies != null) { uniqueDependencyDetails = new HashSet <DependencyDetail>( rootGraphNode.Dependencies, new DependencyDetailComparer()); } else { uniqueDependencyDetails = new HashSet <DependencyDetail>( new DependencyDetailComparer()); } // Cache of nodes we've visited. If we reach a repo/commit combo already in the cache, // we can just add these nodes as a child. The cache key is '{repoUri}@{commit}' Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode); // Cache of incoherent nodes, looked up by repo URI. Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>(); // Cache of incoherent dependencies, looked up by name Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>(); HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>(); while (nodesToVisit.Count > 0) { DependencyGraphNode node = nodesToVisit.Dequeue(); logger.LogInformation($"Visiting {node.Repository}@{node.Commit}"); IEnumerable <DependencyDetail> dependencies; // In case of the root node which is initially put on the stack, // we already have the set of dependencies to start at (this may have been // filtered by the caller). So no need to get the dependencies again. if (node.Dependencies != null) { dependencies = node.Dependencies; } else { logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}"); dependencies = await GetDependenciesAsync( remoteFactory, remote, logger, options.GitExecutable, node.Repository, node.Commit, options.IncludeToolset, remotesMap, reposFolder, testPath); // Set the dependencies on the current node. node.Dependencies = dependencies; } if (dependencies != null) { foreach (DependencyDetail dependency in dependencies) { // If this dependency is missing information, then skip it. if (string.IsNullOrEmpty(dependency.RepoUri) || string.IsNullOrEmpty(dependency.Commit)) { logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " + $"{node.Repository}@{node.Commit} " + $"is missing repository uri or commit information, skipping"); continue; } if (options.LookupBuilds) { if (!buildLookupTasks.ContainsKey($"{dependency.RepoUri}@{dependency.Commit}")) { buildLookupTasks.Add($"{dependency.RepoUri}@{dependency.Commit}", barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit)); } } // If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph // and should break. if (node.VisitedNodes.Contains(dependency.RepoUri)) { logger.LogInformation($"Node {node.Repository}@{node.Commit} " + $"introduces a cycle to {dependency.RepoUri}, skipping"); if (options.ComputeCyclePaths) { var newCycles = ComputeCyclePaths(node, dependency.RepoUri); cycles.AddRange(newCycles); } continue; } // Add the individual dependency to the set of unique dependencies seen // in the whole graph. uniqueDependencyDetails.Add(dependency); if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency)) { incoherentDependencies.Add(existingDependency); incoherentDependencies.Add(dependency); } else { incoherentDependenciesCache.Add(dependency.Name, dependency); } // We may have visited this node before. If so, add it as a child and avoid additional walks. // Update the list of contributing builds. if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode)) { logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child"); node.AddChild(existingNode, dependency); continue; } // Otherwise, create a new node for this dependency. DependencyGraphNode newNode = new DependencyGraphNode( dependency.RepoUri, dependency.Commit, null, node.VisitedNodes, null); // Cache the dependency and add it to the visitation stack. nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode); nodesToVisit.Enqueue(newNode); newNode.VisitedNodes.Add(dependency.RepoUri); node.AddChild(newNode, dependency); // Calculate incoherencies. If we've not yet visited the repo uri, add the // new node based on its repo uri. Otherwise, add both the new node and the visited // node to the incoherent nodes. if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode)) { incoherentNodes.Add(visitedNode); incoherentNodes.Add(newNode); } else { visitedRepoUriNodes.Add(newNode.Repository, newNode); } } } } if (options.LookupBuilds) { allContributingBuilds = await ComputeContributingBuildsAsync(buildLookupTasks, nodeCache.Values, logger); } switch (options.NodeDiff) { case NodeDiff.None: // Nothing break; case NodeDiff.LatestInGraph: await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; case NodeDiff.LatestInChannel: await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; } return(new DependencyGraph(rootGraphNode, uniqueDependencyDetails, incoherentDependencies, nodeCache.Values, incoherentNodes, allContributingBuilds, cycles)); }
/// <summary> /// Get updates required by coherency constraints. /// </summary> /// <param name="dependencies">Current set of dependencies.</param> /// <param name="remoteFactory">Remote factory for remote queries.</param> /// <returns>Dependencies with updates.</returns> public async Task <List <DependencyUpdate> > GetRequiredCoherencyUpdatesAsync( IEnumerable <DependencyDetail> dependencies, IRemoteFactory remoteFactory) { List <DependencyUpdate> toUpdate = new List <DependencyUpdate>(); IEnumerable <DependencyDetail> leavesOfCoherencyTrees = CalculateLeavesOfCoherencyTrees(dependencies); if (!leavesOfCoherencyTrees.Any()) { // Nothing to do. return(toUpdate); } DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = true, LookupBuilds = true, NodeDiff = NodeDiff.None }; // Now make a walk over coherent dependencies. Note that coherent dependencies could make // a chain (A->B->C). In all cases we need to walk to the head of the chain, keeping track // of all elements in the chain. Also note that we are walking all dependencies here, not // just those that match the incoming AssetData and aligning all of these based on the coherency data. Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyDetail> visited = new HashSet <DependencyDetail>(); foreach (DependencyDetail dependency in leavesOfCoherencyTrees) { // If the dependency was already updated, then skip it (could have been part of a larger // dependency chain) if (visited.Contains(dependency)) { continue; } // Walk to head of dependency tree, keeping track of elements along the way. // If we hit a pinned dependency in the walk, that means we can't move // the dependency and therefore it is effectively the "head" of the subtree. // We will still visit all the elements in the chain eventually in this algorithm: // Consider A->B(pinned)->C(pinned)->D. List <DependencyDetail> updateList = new List <DependencyDetail>(); DependencyDetail currentDependency = dependency; while (!string.IsNullOrEmpty(currentDependency.CoherentParentDependencyName) && !currentDependency.Pinned) { updateList.Add(currentDependency); DependencyDetail parentCoherentDependency = dependencies.FirstOrDefault(d => d.Name.Equals(currentDependency.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase)); currentDependency = parentCoherentDependency ?? throw new DarcException($"Dependency {currentDependency.Name} has non-existent parent " + $"dependency {currentDependency.CoherentParentDependencyName}"); } DependencyGraphNode rootNode = null; // Build the graph to find the assets if we don't have the root in the cache. // The graph build is automatically broken when // all the desired assets are found (breadth first search). This means the cache may or // may not contain a complete graph for a given node. So, we first search the cache for the desired assets, // then if not found (or if there was no cache), we then build the graph from that node. bool nodeFromCache = nodeCache.TryGetValue($"{currentDependency.RepoUri}@{currentDependency.Commit}", out rootNode); if (!nodeFromCache) { _logger.LogInformation($"Node not found in cache, starting graph build at " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, updateList, nodeCache); } List <DependencyDetail> leftToFind = new List <DependencyDetail>(updateList); // Now do the lookup to find the element in the tree for each item in the update list foreach (DependencyDetail dependencyInUpdateChain in updateList) { (Asset coherentAsset, Build buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); // If we originally got the root node from the cache the graph may be incomplete. // Rebuild to attempt to find all the assets we have left to find. If we still can't find, or if // the root node did not come the cache, then we're in an invalid state. if (coherentAsset == null && nodeFromCache) { _logger.LogInformation($"Asset {dependencyInUpdateChain.Name} was not found in cached graph, rebuilding from " + $"{currentDependency.RepoUri}@{currentDependency.Commit}"); rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, leftToFind, nodeCache); // And attempt to find again. (coherentAsset, buildForAsset) = FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode); } if (coherentAsset == null) { // This is an invalid state. We can't satisfy the // constraints so they should either be removed or pinned. throw new DarcException($"Unable to update {dependencyInUpdateChain.Name} to have coherency with " + $"parent {dependencyInUpdateChain.CoherentParentDependencyName}. No matching asset found in tree. " + $"Either remove the coherency attribute or mark as pinned."); } else { leftToFind.Remove(dependencyInUpdateChain); } string buildRepoUri = buildForAsset.GitHubRepository ?? buildForAsset.AzureDevOpsRepository; if (dependencyInUpdateChain.Name == coherentAsset.Name && dependencyInUpdateChain.Version == coherentAsset.Version && dependencyInUpdateChain.Commit == buildForAsset.Commit && dependencyInUpdateChain.RepoUri == buildRepoUri) { continue; } DependencyDetail updatedDependency = new DependencyDetail(dependencyInUpdateChain) { Name = coherentAsset.Name, Version = coherentAsset.Version, RepoUri = buildRepoUri, Commit = buildForAsset.Commit }; toUpdate.Add(new DependencyUpdate { From = dependencyInUpdateChain, To = updatedDependency }); visited.Add(dependencyInUpdateChain); } } return(toUpdate); }