/// <summary> /// Determines whether a build contributes to a given node by looking at the parents' /// input dependencies. If there are no parents, then we assume that the build could contribute. This /// would happen for the root node. /// </summary> /// <param name="node">Node</param> /// <param name="potentialContributingBuild">Potentially contributing build</param> /// <returns>True if the build contributes, false otherwise.</returns> private static bool BuildContributesToNode(DependencyGraphNode node, Build potentialContributingBuild) { if (!node.Parents.Any()) { return(true); } AssetComparer assetEqualityComparer = new AssetComparer(); foreach (DependencyGraphNode parentNode in node.Parents) { foreach (var dependency in parentNode.Dependencies) { if (dependency.Commit == node.Commit && dependency.RepoUri == node.Repository && potentialContributingBuild.Assets.Any(a => assetEqualityComparer.Equals(a, dependency))) { return(true); } } } return(false); }
/// <summary> /// Creates a new dependency graph /// </summary> /// <param name="remoteFactory">Remote for factory for obtaining remotes to</param> /// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param> /// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param> /// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param> /// <param name="includeToolset">If true, toolset dependencies are included.</param> /// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param> /// <param name="remote">If true, remote graph build is used.</param> /// <param name="logger">Logger</param> /// <param name="reposFolder">Path to repos</param> /// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param> /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param> /// <returns>New dependency graph</returns> private static async Task <DependencyGraph> BuildDependencyGraphImplAsync( IRemoteFactory remoteFactory, IEnumerable <DependencyDetail> rootDependencies, string repoUri, string commit, DependencyGraphBuildOptions options, bool remote, ILogger logger, string reposFolder, IEnumerable <string> remotesMap, string testPath) { ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit, options, remote, logger, reposFolder, remotesMap, testPath); if (rootDependencies != null) { logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " + $"({repoUri}@{commit})"); foreach (DependencyDetail dependency in rootDependencies) { logger.LogInformation($" {dependency.Name}@{dependency.Version}"); } } else { logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})"); } AssetComparer assetEqualityComparer = new AssetComparer(); HashSet <Build> allContributingBuilds = null; HashSet <DependencyDetail> dependenciesMissingBuilds = null; HashSet <Build> rootNodeBuilds = null; Dictionary <DependencyDetail, Build> dependencyCache = new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer()); List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >(); EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type; HashSet <string> breakOn = null; if (breakOnType != EarlyBreakOnType.None) { breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase); } if (options.LookupBuilds) { allContributingBuilds = new HashSet <Build>(new BuildComparer()); dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer()); rootNodeBuilds = new HashSet <Build>(new BuildComparer()); // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit); // Filter by those actually producing the root dependencies, if they were supplied if (rootDependencies != null) { potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b => b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d)))); } // It's entirely possible that the root has no builds (e.g. change just checked in). // Don't record those. Instead, users of the graph should just look at the // root node's contributing builds and determine whether it's empty or not. foreach (Build build in potentialRootNodeBuilds) { allContributingBuilds.Add(build); rootNodeBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } // Create the root node and add the repo to the visited bit vector. DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds); rootGraphNode.VisitedNodes.Add(repoUri); // Nodes to visit is a queue, so that the evaluation order // of the graph is breadth first. Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>(); nodesToVisit.Enqueue(rootGraphNode); HashSet <DependencyDetail> uniqueDependencyDetails; if (rootGraphNode.Dependencies != null) { uniqueDependencyDetails = new HashSet <DependencyDetail>( rootGraphNode.Dependencies, new DependencyDetailComparer()); // Remove the dependencies details from the // break on if break on type is Dependencies if (breakOnType == EarlyBreakOnType.Dependencies) { rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name)); } } else { uniqueDependencyDetails = new HashSet <DependencyDetail>( new DependencyDetailComparer()); } // If we already found the assets/dependencies we wanted, clear the // visit list and we'll drop through. if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); } // Cache of nodes we've visited. If we reach a repo/commit combo already in the cache, // we can just add these nodes as a child. The cache key is '{repoUri}@{commit}' Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>(); nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode); // Cache of incoherent nodes, looked up by repo URI. Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>(); HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>(); // Cache of incoherent dependencies, looked up by name Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>(); HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>(); while (nodesToVisit.Count > 0) { DependencyGraphNode node = nodesToVisit.Dequeue(); logger.LogInformation($"Visiting {node.Repository}@{node.Commit}"); IEnumerable <DependencyDetail> dependencies; // In case of the root node which is initially put on the stack, // we already have the set of dependencies to start at (this may have been // filtered by the caller). So no need to get the dependencies again. if (node.Dependencies != null) { dependencies = node.Dependencies; } else { logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}"); dependencies = await GetDependenciesAsync( remoteFactory, remote, logger, node.Repository, node.Commit, options.IncludeToolset, remotesMap, reposFolder, testPath); // Set the dependencies on the current node. node.Dependencies = dependencies; } if (dependencies != null) { foreach (DependencyDetail dependency in dependencies) { // If this dependency is missing information, then skip it. if (string.IsNullOrEmpty(dependency.RepoUri) || string.IsNullOrEmpty(dependency.Commit)) { logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " + $"{node.Repository}@{node.Commit} " + $"is missing repository uri or commit information, skipping"); continue; } // If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph // and should break. if (node.VisitedNodes.Contains(dependency.RepoUri)) { logger.LogInformation($"Node {node.Repository}@{node.Commit} " + $"introduces a cycle to {dependency.RepoUri}, skipping"); if (options.ComputeCyclePaths) { var newCycles = ComputeCyclePaths(node, dependency.RepoUri); cycles.AddRange(newCycles); } continue; } // Add the individual dependency to the set of unique dependencies seen // in the whole graph. uniqueDependencyDetails.Add(dependency); if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency)) { incoherentDependencies.Add(existingDependency); incoherentDependencies.Add(dependency); } else { incoherentDependenciesCache.Add(dependency.Name, dependency); } HashSet <Build> nodeContributingBuilds = null; if (options.LookupBuilds) { nodeContributingBuilds = new HashSet <Build>(new BuildComparer()); // Look up dependency in cache first if (dependencyCache.TryGetValue(dependency, out Build existingBuild)) { nodeContributingBuilds.Add(existingBuild); allContributingBuilds.Add(existingBuild); } else { // Look up the dependency and get the creating build. IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit); // Filter by those actually producing the dependency. Most of the time this won't // actually result in a different set of contributing builds, but should avoid any subtle bugs where // there might be overlap between repos, or cases where there were multiple builds at the same sha. potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b => b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency))); if (!potentiallyContributingBuilds.Any()) { // Couldn't find a build that produced the dependency. dependenciesMissingBuilds.Add(dependency); } else { foreach (Build build in potentiallyContributingBuilds) { allContributingBuilds.Add(build); nodeContributingBuilds.Add(build); AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn); } } } } // We may have visited this node before. If so, add it as a child and avoid additional walks. // Update the list of contributing builds. if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode)) { if (options.LookupBuilds) { // Add the contributing builds. It's possible that // different dependencies on a single node (repo/sha) were produced // from multiple builds foreach (Build build in nodeContributingBuilds) { existingNode.ContributingBuilds.Add(build); } } logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child"); node.AddChild(existingNode, dependency); continue; } // Otherwise, create a new node for this dependency. DependencyGraphNode newNode = new DependencyGraphNode( dependency.RepoUri, dependency.Commit, null, node.VisitedNodes, nodeContributingBuilds); // Cache the dependency and add it to the visitation stack. nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode); nodesToVisit.Enqueue(newNode); newNode.VisitedNodes.Add(dependency.RepoUri); node.AddChild(newNode, dependency); // Calculate incoherencies. If we've not yet visited the repo uri, add the // new node based on its repo uri. Otherwise, add both the new node and the visited // node to the incoherent nodes. if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode)) { incoherentNodes.Add(visitedNode); incoherentNodes.Add(newNode); } else { visitedRepoUriNodes.Add(newNode.Repository, newNode); } // If breaking on dependencies, then decide whether we need to break // here. if (breakOnType == EarlyBreakOnType.Dependencies) { breakOn.Remove(dependency.Name); } if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0) { logger.LogInformation($"Stopping graph build after finding all assets/dependencies."); nodesToVisit.Clear(); break; } } } } switch (options.NodeDiff) { case NodeDiff.None: // Nothing break; case NodeDiff.LatestInGraph: await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; case NodeDiff.LatestInChannel: await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes); break; } return(new DependencyGraph(rootGraphNode, uniqueDependencyDetails, incoherentDependencies, nodeCache.Values, incoherentNodes, allContributingBuilds, dependenciesMissingBuilds, cycles)); }