Exemple #1
0
        /// <summary>
        /// Determines whether a build contributes to a given node by looking at the parents'
        /// input dependencies. If there are no parents, then we assume that the build could contribute. This
        /// would happen for the root node.
        /// </summary>
        /// <param name="node">Node</param>
        /// <param name="potentialContributingBuild">Potentially contributing build</param>
        /// <returns>True if the build contributes, false otherwise.</returns>
        private static bool BuildContributesToNode(DependencyGraphNode node, Build potentialContributingBuild)
        {
            if (!node.Parents.Any())
            {
                return(true);
            }

            AssetComparer assetEqualityComparer = new AssetComparer();

            foreach (DependencyGraphNode parentNode in node.Parents)
            {
                foreach (var dependency in parentNode.Dependencies)
                {
                    if (dependency.Commit == node.Commit &&
                        dependency.RepoUri == node.Repository &&
                        potentialContributingBuild.Assets.Any(a => assetEqualityComparer.Equals(a, dependency)))
                    {
                        return(true);
                    }
                }
            }
            return(false);
        }
        /// <summary>
        ///     Creates a new dependency graph
        /// </summary>
        /// <param name="remoteFactory">Remote for factory for obtaining remotes to</param>
        /// <param name="rootDependencies">Root set of dependencies.  If null, then repoUri and commit should be set</param>
        /// <param name="repoUri">Root repository uri.  Must be valid if no root dependencies are passed.</param>
        /// <param name="commit">Root commit.  Must be valid if no root dependencies were passed.</param>
        /// <param name="includeToolset">If true, toolset dependencies are included.</param>
        /// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param>
        /// <param name="remote">If true, remote graph build is used.</param>
        /// <param name="logger">Logger</param>
        /// <param name="reposFolder">Path to repos</param>
        /// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param>
        /// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param>
        /// <returns>New dependency graph</returns>
        private static async Task <DependencyGraph> BuildDependencyGraphImplAsync(
            IRemoteFactory remoteFactory,
            IEnumerable <DependencyDetail> rootDependencies,
            string repoUri,
            string commit,
            DependencyGraphBuildOptions options,
            bool remote,
            ILogger logger,
            string reposFolder,
            IEnumerable <string> remotesMap,
            string testPath)
        {
            ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit,
                                 options, remote, logger, reposFolder, remotesMap, testPath);

            if (rootDependencies != null)
            {
                logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " +
                                      $"({repoUri}@{commit})");
                foreach (DependencyDetail dependency in rootDependencies)
                {
                    logger.LogInformation($"  {dependency.Name}@{dependency.Version}");
                }
            }
            else
            {
                logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})");
            }

            AssetComparer                        assetEqualityComparer     = new AssetComparer();
            HashSet <Build>                      allContributingBuilds     = null;
            HashSet <DependencyDetail>           dependenciesMissingBuilds = null;
            HashSet <Build>                      rootNodeBuilds            = null;
            Dictionary <DependencyDetail, Build> dependencyCache           =
                new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer());
            List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >();

            EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type;
            HashSet <string> breakOn     = null;

            if (breakOnType != EarlyBreakOnType.None)
            {
                breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase);
            }

            if (options.LookupBuilds)
            {
                allContributingBuilds     = new HashSet <Build>(new BuildComparer());
                dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer());
                rootNodeBuilds            = new HashSet <Build>(new BuildComparer());

                // Look up the dependency and get the creating build.
                IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);

                IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit);

                // Filter by those actually producing the root dependencies, if they were supplied
                if (rootDependencies != null)
                {
                    potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b =>
                                                                            b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d))));
                }
                // It's entirely possible that the root has no builds (e.g. change just checked in).
                // Don't record those. Instead, users of the graph should just look at the
                // root node's contributing builds and determine whether it's empty or not.
                foreach (Build build in potentialRootNodeBuilds)
                {
                    allContributingBuilds.Add(build);
                    rootNodeBuilds.Add(build);
                    AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
                }
            }

            // Create the root node and add the repo to the visited bit vector.
            DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds);

            rootGraphNode.VisitedNodes.Add(repoUri);
            // Nodes to visit is a queue, so that the evaluation order
            // of the graph is breadth first.
            Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>();

            nodesToVisit.Enqueue(rootGraphNode);
            HashSet <DependencyDetail> uniqueDependencyDetails;

            if (rootGraphNode.Dependencies != null)
            {
                uniqueDependencyDetails = new HashSet <DependencyDetail>(
                    rootGraphNode.Dependencies,
                    new DependencyDetailComparer());
                // Remove the dependencies details from the
                // break on if break on type is Dependencies
                if (breakOnType == EarlyBreakOnType.Dependencies)
                {
                    rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name));
                }
            }
            else
            {
                uniqueDependencyDetails = new HashSet <DependencyDetail>(
                    new DependencyDetailComparer());
            }

            // If we already found the assets/dependencies we wanted, clear the
            // visit list and we'll drop through.
            if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
            {
                logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
                nodesToVisit.Clear();
            }

            // Cache of nodes we've visited. If we reach a repo/commit combo already in the cache,
            // we can just add these nodes as a child. The cache key is '{repoUri}@{commit}'
            Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();

            nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode);

            // Cache of incoherent nodes, looked up by repo URI.
            Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>();
            HashSet <DependencyGraphNode>            incoherentNodes     = new HashSet <DependencyGraphNode>();
            // Cache of incoherent dependencies, looked up by name
            Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>();
            HashSet <DependencyDetail>            incoherentDependencies      = new HashSet <DependencyDetail>();

            while (nodesToVisit.Count > 0)
            {
                DependencyGraphNode node = nodesToVisit.Dequeue();

                logger.LogInformation($"Visiting {node.Repository}@{node.Commit}");

                IEnumerable <DependencyDetail> dependencies;
                // In case of the root node which is initially put on the stack,
                // we already have the set of dependencies to start at (this may have been
                // filtered by the caller). So no need to get the dependencies again.
                if (node.Dependencies != null)
                {
                    dependencies = node.Dependencies;
                }
                else
                {
                    logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}");

                    dependencies = await GetDependenciesAsync(
                        remoteFactory,
                        remote,
                        logger,
                        node.Repository,
                        node.Commit,
                        options.IncludeToolset,
                        remotesMap,
                        reposFolder,
                        testPath);

                    // Set the dependencies on the current node.
                    node.Dependencies = dependencies;
                }

                if (dependencies != null)
                {
                    foreach (DependencyDetail dependency in dependencies)
                    {
                        // If this dependency is missing information, then skip it.
                        if (string.IsNullOrEmpty(dependency.RepoUri) ||
                            string.IsNullOrEmpty(dependency.Commit))
                        {
                            logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " +
                                                  $"{node.Repository}@{node.Commit} " +
                                                  $"is missing repository uri or commit information, skipping");
                            continue;
                        }

                        // If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph
                        // and should break.
                        if (node.VisitedNodes.Contains(dependency.RepoUri))
                        {
                            logger.LogInformation($"Node {node.Repository}@{node.Commit} " +
                                                  $"introduces a cycle to {dependency.RepoUri}, skipping");

                            if (options.ComputeCyclePaths)
                            {
                                var newCycles = ComputeCyclePaths(node, dependency.RepoUri);
                                cycles.AddRange(newCycles);
                            }
                            continue;
                        }

                        // Add the individual dependency to the set of unique dependencies seen
                        // in the whole graph.
                        uniqueDependencyDetails.Add(dependency);
                        if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency))
                        {
                            incoherentDependencies.Add(existingDependency);
                            incoherentDependencies.Add(dependency);
                        }
                        else
                        {
                            incoherentDependenciesCache.Add(dependency.Name, dependency);
                        }

                        HashSet <Build> nodeContributingBuilds = null;
                        if (options.LookupBuilds)
                        {
                            nodeContributingBuilds = new HashSet <Build>(new BuildComparer());
                            // Look up dependency in cache first
                            if (dependencyCache.TryGetValue(dependency, out Build existingBuild))
                            {
                                nodeContributingBuilds.Add(existingBuild);
                                allContributingBuilds.Add(existingBuild);
                            }
                            else
                            {
                                // Look up the dependency and get the creating build.
                                IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);

                                IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit);

                                // Filter by those actually producing the dependency. Most of the time this won't
                                // actually result in a different set of contributing builds, but should avoid any subtle bugs where
                                // there might be overlap between repos, or cases where there were multiple builds at the same sha.
                                potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b =>
                                                                                                    b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency)));
                                if (!potentiallyContributingBuilds.Any())
                                {
                                    // Couldn't find a build that produced the dependency.
                                    dependenciesMissingBuilds.Add(dependency);
                                }
                                else
                                {
                                    foreach (Build build in potentiallyContributingBuilds)
                                    {
                                        allContributingBuilds.Add(build);
                                        nodeContributingBuilds.Add(build);
                                        AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
                                    }
                                }
                            }
                        }

                        // We may have visited this node before.  If so, add it as a child and avoid additional walks.
                        // Update the list of contributing builds.
                        if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode))
                        {
                            if (options.LookupBuilds)
                            {
                                // Add the contributing builds. It's possible that
                                // different dependencies on a single node (repo/sha) were produced
                                // from multiple builds
                                foreach (Build build in nodeContributingBuilds)
                                {
                                    existingNode.ContributingBuilds.Add(build);
                                }
                            }
                            logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child");
                            node.AddChild(existingNode, dependency);
                            continue;
                        }

                        // Otherwise, create a new node for this dependency.
                        DependencyGraphNode newNode = new DependencyGraphNode(
                            dependency.RepoUri,
                            dependency.Commit,
                            null,
                            node.VisitedNodes,
                            nodeContributingBuilds);

                        // Cache the dependency and add it to the visitation stack.
                        nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode);
                        nodesToVisit.Enqueue(newNode);
                        newNode.VisitedNodes.Add(dependency.RepoUri);
                        node.AddChild(newNode, dependency);

                        // Calculate incoherencies. If we've not yet visited the repo uri, add the
                        // new node based on its repo uri. Otherwise, add both the new node and the visited
                        // node to the incoherent nodes.
                        if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode))
                        {
                            incoherentNodes.Add(visitedNode);
                            incoherentNodes.Add(newNode);
                        }
                        else
                        {
                            visitedRepoUriNodes.Add(newNode.Repository, newNode);
                        }

                        // If breaking on dependencies, then decide whether we need to break
                        // here.
                        if (breakOnType == EarlyBreakOnType.Dependencies)
                        {
                            breakOn.Remove(dependency.Name);
                        }

                        if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
                        {
                            logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
                            nodesToVisit.Clear();
                            break;
                        }
                    }
                }
            }

            switch (options.NodeDiff)
            {
            case NodeDiff.None:
                // Nothing
                break;

            case NodeDiff.LatestInGraph:
                await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);

                break;

            case NodeDiff.LatestInChannel:
                await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);

                break;
            }

            return(new DependencyGraph(rootGraphNode,
                                       uniqueDependencyDetails,
                                       incoherentDependencies,
                                       nodeCache.Values,
                                       incoherentNodes,
                                       allContributingBuilds,
                                       dependenciesMissingBuilds,
                                       cycles));
        }