private static async Task CloneRemoteRepoAndDependencies(RemoteFactory remoteFactory, string reposFolder, string repoUri, string commit, bool includeToolset, ILogger logger) { IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger); IEnumerable <DependencyDetail> rootDependencies = await rootRepoRemote.GetDependenciesAsync(repoUri, commit); rootDependencies = FilterToolsetDependencies(rootDependencies, includeToolset); if (!rootDependencies.Any()) { string repoPath = GetRepoDirectory(reposFolder, repoUri, commit); if (Directory.Exists(repoPath)) { logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping"); } else { logger.LogInformation($"Remote repo {repoUri}@{commit} has no dependencies. Cloning shallowly into {repoPath}"); IRemote repoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger); repoRemote.Clone(repoUri, commit, repoPath); } return; } DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = includeToolset, LookupBuilds = true, NodeDiff = NodeDiff.None }; logger.LogDebug($"Building depdendency graph for {repoUri}@{commit} with {rootDependencies.Count()} dependencies"); DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( remoteFactory, rootDependencies, repoUri, commit, graphBuildOptions, logger); foreach (DependencyGraphNode repo in graph.Nodes) { string repoPath = GetRepoDirectory(reposFolder, repo.Repository, repo.Commit); if (Directory.Exists(repoPath)) { logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping"); } else { logger.LogInformation($"Cloning {repo.Repository}@{repo.Commit} into {repoPath}"); IRemote repoRemote = await remoteFactory.GetRemoteAsync(repo.Repository, logger); repoRemote.Clone(repo.Repository, repo.Commit, repoPath); } } }
public async Task ProcessAsync(JToken argumentToken) { // This method is called asynchronously whenever a new build is inserted in BAR. // It's goal is to compute the incoherent dependencies that the build have and // persist the list of them in BAR. int buildId = argumentToken.Value <int>(); DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = false, LookupBuilds = false, NodeDiff = NodeDiff.None }; try { Data.Models.Build build = await _context.Builds.FindAsync(buildId); DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( _remoteFactory, build.GitHubRepository ?? build.AzureDevOpsRepository, build.Commit, graphBuildOptions, _logger); var incoherencies = new List <Data.Models.BuildIncoherence>(); foreach (var incoherence in graph.IncoherentDependencies) { incoherencies.Add(new Data.Models.BuildIncoherence { Name = incoherence.Name, Version = incoherence.Version, Repository = incoherence.RepoUri, Commit = incoherence.Commit }); } _context.Entry(build).Reload(); build.Incoherencies = incoherencies; _context.Builds.Update(build); await _context.SaveChangesAsync(); } catch (Exception e) { _logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {buildId}"); } }
/// <summary> /// This method is called asynchronously whenever a new build is inserted in BAR. /// It's goal is to compute the incoherent dependencies that the build have and /// persist the list of them in BAR. /// </summary> /// <param name="buildId">Build id for which the incoherencies should be computed.</param> private async Task SetBuildIncoherencyInfoAsync(int buildId) { DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = false, LookupBuilds = false, NodeDiff = NodeDiff.None }; try { using (IServiceScope scope = ServiceScopeFactory.CreateScope()) { BuildAssetRegistryContext context = scope.ServiceProvider.GetRequiredService <BuildAssetRegistryContext>(); Data.Models.Build build = await context.Builds.FindAsync(buildId); DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( RemoteFactory, build.GitHubRepository ?? build.AzureDevOpsRepository, build.Commit, graphBuildOptions, Logger); var incoherencies = new List <Data.Models.BuildIncoherence>(); foreach (var incoherence in graph.IncoherentDependencies) { build.Incoherencies.Add(new Data.Models.BuildIncoherence { Name = incoherence.Name, Version = incoherence.Version, Repository = incoherence.RepoUri, Commit = incoherence.Commit }); } context.Entry <Data.Models.Build>(build).Reload(); build.Incoherencies = incoherencies; context.Builds.Update(build); await context.SaveChangesAsync(); } } catch (Exception e) { Logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {buildId}"); } }
public override async Task EvaluateAsync() { // Build the repository graph starting at the repo + branch and then look for cycles. // Build without build lookups, toolsets, etc. to minimize time. Toolset dependencies also break // product dependency cycles, so that eliminates some analysis DependencyGraphBuildOptions options = new DependencyGraphBuildOptions { EarlyBuildBreak = EarlyBreakOn.NoEarlyBreak, IncludeToolset = false, LookupBuilds = false, NodeDiff = NodeDiff.None, ComputeCyclePaths = true }; // Evaluate and find out what the latest is on the branch var remote = await RemoteFactory.GetRemoteAsync(Repository, Logger); var commit = await remote.GetLatestCommitAsync(Repository, Branch); if (commit == null) { // If there were no commits, then there can be no cycles. This would be typical of newly // created branches. Result = HealthResult.Passed; Cycles = new List <List <string> >(); return; } DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(RemoteFactory, Repository, commit, options, Logger); // Check to see whether there are any cycles. // Boil down the cycles into just the repositories involved Cycles = graph.Cycles.Select(cycle => cycle.Select(graphNode => graphNode.Repository)); if (Cycles.Any()) { Result = HealthResult.Failed; } else { Result = HealthResult.Passed; } }
public override async Task <int> ExecuteAsync() { try { IEnumerable <DependencyDetail> rootDependencies = null; DependencyGraph graph; RemoteFactory remoteFactory = new RemoteFactory(_options); if (!_options.Local) { NodeDiff diffOption = NodeDiff.None; // Check node diff options switch (_options.DeltaFrom.ToLowerInvariant()) { case "none": break; case "newest-in-channel": diffOption = NodeDiff.LatestInChannel; break; case "newest-in-graph": diffOption = NodeDiff.LatestInGraph; break; default: Console.WriteLine("Unknown --delta-from option, please see help."); return(Constants.ErrorCode); } // If the repo uri and version are set, then call the graph // build operation based on those. Both should be set in this case. // If they are not set, then gather the initial set based on the local repository, // and then call the graph build with that root set. if (!string.IsNullOrEmpty(_options.RepoUri)) { if (string.IsNullOrEmpty(_options.Version)) { Console.WriteLine("If --repo is set, --version should be supplied"); return(Constants.ErrorCode); } Console.WriteLine($"Getting root dependencies from {_options.RepoUri}@{_options.Version}..."); // Grab root dependency set. The graph build can do this, but // if an original asset name is passed, then this will do the initial filtering. IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(_options.RepoUri, Logger); rootDependencies = await rootRepoRemote.GetDependenciesAsync( _options.RepoUri, _options.Version, _options.AssetName); } else { if (!string.IsNullOrEmpty(_options.Version)) { Console.WriteLine("If --version is supplied, then --repo is required"); return(Constants.ErrorCode); } Console.WriteLine($"Getting root dependencies from local repository..."); // Grab root dependency set from local repo Local local = new Local(Logger); rootDependencies = await local.GetDependenciesAsync( _options.AssetName); } Console.WriteLine($"Building repository dependency graph..."); rootDependencies = FilterToolsetDependencies(rootDependencies); if (!rootDependencies.Any()) { Console.WriteLine($"No root dependencies found, exiting."); return(Constants.ErrorCode); } DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = _options.IncludeToolset, LookupBuilds = diffOption != NodeDiff.None || !_options.SkipBuildLookup, NodeDiff = diffOption }; // Build graph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( remoteFactory, rootDependencies, _options.RepoUri ?? LocalHelpers.GetRootDir(Logger), _options.Version ?? LocalHelpers.GetGitCommit(Logger), graphBuildOptions, Logger); } else { Console.WriteLine($"Getting root dependencies from local repository..."); Local local = new Local(Logger); rootDependencies = await local.GetDependenciesAsync( _options.AssetName); rootDependencies = FilterToolsetDependencies(rootDependencies); if (!rootDependencies.Any()) { Console.WriteLine($"No root dependencies found, exiting."); return(Constants.ErrorCode); } Console.WriteLine($"Building repository dependency graph from local information..."); DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions() { IncludeToolset = _options.IncludeToolset, LookupBuilds = false, NodeDiff = NodeDiff.None }; // Build graph using only local resources graph = await DependencyGraph.BuildLocalDependencyGraphAsync( rootDependencies, graphBuildOptions, Logger, LocalHelpers.GetRootDir(Logger), LocalHelpers.GetGitCommit(Logger), _options.ReposFolder, _options.RemotesMap); } if (_options.Flat) { await LogFlatDependencyGraph(graph); } else { await LogDependencyGraph(graph); } if (!string.IsNullOrEmpty(_options.GraphVizOutputFile)) { await LogGraphViz(graph); } return(Constants.SuccessCode); } catch (Exception exc) { Logger.LogError(exc, "Something failed while getting the dependency graph."); return(Constants.ErrorCode); } }
/// <summary> /// Build the list of builds that will need their assets downloaded. /// </summary> /// <returns>List of builds to download</returns> /// <remarks> /// This can be pretty simple if a full build download is /// not desired (just determine the root build and return it) or it could /// be a matter of determining all builds that contributed to all dependencies. /// </remarks> private async Task <InputBuilds> GatherBuildsToDownloadAsync() { Console.WriteLine("Determining what builds to download..."); // Gather the root build Build rootBuild = await GetRootBuildAsync(); if (rootBuild == null) { return(new InputBuilds { Successful = false }); } Console.WriteLine($"Root build - Build number {rootBuild.AzureDevOpsBuildNumber} of {rootBuild.AzureDevOpsRepository} @ {rootBuild.Commit}"); // If transitive (full tree) was not selected, we're done if (!_options.Transitive) { return(new InputBuilds() { Successful = true, Builds = new List <Build>() { rootBuild } }); } HashSet <Build> builds = new HashSet <Build>(new BuildComparer()); builds.Add(rootBuild); IRemoteFactory remoteFactory = new RemoteFactory(_options); // Grab dependencies IRemote rootBuildRemote = await remoteFactory.GetRemoteAsync(rootBuild.AzureDevOpsRepository, Logger); Console.WriteLine($"Getting dependencies of root build..."); // Flatten for convencience and remove dependencies of types that we don't want if need be. if (!_options.IncludeToolset) { Console.WriteLine("Filtering toolset dependencies from the graph..."); } DependencyGraphBuildOptions buildOptions = new DependencyGraphBuildOptions() { IncludeToolset = _options.IncludeToolset, LookupBuilds = true, NodeDiff = NodeDiff.None }; Console.WriteLine("Building graph of all dependencies under root build..."); DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( remoteFactory, rootBuild.GitHubRepository ?? rootBuild.AzureDevOpsRepository, rootBuild.Commit, buildOptions, Logger); Dictionary <DependencyDetail, Build> dependencyCache = new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer()); // Cache root build's assets foreach (Asset buildAsset in rootBuild.Assets) { dependencyCache.Add( new DependencyDetail { Name = buildAsset.Name, Version = buildAsset.Version, Commit = rootBuild.Commit, }, rootBuild); } Console.WriteLine($"There are {graph.UniqueDependencies.Count()} unique dependencies in the graph."); Console.WriteLine("Full set of builds in graph:"); foreach (var build in graph.ContributingBuilds) { Console.WriteLine($" Build - {build.AzureDevOpsBuildNumber} of {build.GitHubRepository ?? build.AzureDevOpsRepository} @ {build.Commit}"); builds.Add(build); } if (graph.DependenciesMissingBuilds.Any()) { Console.WriteLine("Dependencies missing builds:"); foreach (DependencyDetail dependency in graph.DependenciesMissingBuilds) { Console.WriteLine($" {dependency.Name}@{dependency.Version} @ ({dependency.RepoUri}@{dependency.Commit})"); } if (!_options.ContinueOnError) { return(new InputBuilds() { Successful = false, Builds = builds }); } } return(new InputBuilds() { Successful = true, Builds = builds.ToList() }); }