private static async Task CloneRemoteRepoAndDependencies(RemoteFactory remoteFactory, string reposFolder, string repoUri, string commit, bool includeToolset, ILogger logger)
{
IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger);
IEnumerable <DependencyDetail> rootDependencies = await rootRepoRemote.GetDependenciesAsync(repoUri, commit);
rootDependencies = FilterToolsetDependencies(rootDependencies, includeToolset);
if (!rootDependencies.Any())
{
string repoPath = GetRepoDirectory(reposFolder, repoUri, commit);
if (Directory.Exists(repoPath))
{
logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping");
}
else
{
logger.LogInformation($"Remote repo {repoUri}@{commit} has no dependencies. Cloning shallowly into {repoPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger);
repoRemote.Clone(repoUri, commit, repoPath);
}
return;
}
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = includeToolset,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
logger.LogDebug($"Building depdendency graph for {repoUri}@{commit} with {rootDependencies.Count()} dependencies");
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootDependencies,
repoUri,
commit,
graphBuildOptions,
logger);
foreach (DependencyGraphNode repo in graph.Nodes)
{
string repoPath = GetRepoDirectory(reposFolder, repo.Repository, repo.Commit);
if (Directory.Exists(repoPath))
{
logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping");
}
else
{
logger.LogInformation($"Cloning {repo.Repository}@{repo.Commit} into {repoPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repo.Repository, logger);
repoRemote.Clone(repo.Repository, repo.Commit, repoPath);
}
}
}
public async Task ProcessAsync(JToken argumentToken)
{
// This method is called asynchronously whenever a new build is inserted in BAR.
// It's goal is to compute the incoherent dependencies that the build have and
// persist the list of them in BAR.
int buildId = argumentToken.Value <int>();
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = false,
LookupBuilds = false,
NodeDiff = NodeDiff.None
};
try
{
Data.Models.Build build = await _context.Builds.FindAsync(buildId);
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
_remoteFactory,
build.GitHubRepository ?? build.AzureDevOpsRepository,
build.Commit,
graphBuildOptions,
_logger);
var incoherencies = new List <Data.Models.BuildIncoherence>();
foreach (var incoherence in graph.IncoherentDependencies)
{
incoherencies.Add(new Data.Models.BuildIncoherence
{
Name = incoherence.Name,
Version = incoherence.Version,
Repository = incoherence.RepoUri,
Commit = incoherence.Commit
});
}
_context.Entry(build).Reload();
build.Incoherencies = incoherencies;
_context.Builds.Update(build);
await _context.SaveChangesAsync();
}
catch (Exception e)
{
_logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {buildId}");
}
}
/// <summary>
/// This method is called asynchronously whenever a new build is inserted in BAR.
/// It's goal is to compute the incoherent dependencies that the build have and
/// persist the list of them in BAR.
/// </summary>
/// <param name="buildId">Build id for which the incoherencies should be computed.</param>
private async Task SetBuildIncoherencyInfoAsync(int buildId)
{
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = false,
LookupBuilds = false,
NodeDiff = NodeDiff.None
};
try
{
using (IServiceScope scope = ServiceScopeFactory.CreateScope())
{
BuildAssetRegistryContext context = scope.ServiceProvider.GetRequiredService <BuildAssetRegistryContext>();
Data.Models.Build build = await context.Builds.FindAsync(buildId);
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
RemoteFactory,
build.GitHubRepository ?? build.AzureDevOpsRepository,
build.Commit,
graphBuildOptions,
Logger);
var incoherencies = new List <Data.Models.BuildIncoherence>();
foreach (var incoherence in graph.IncoherentDependencies)
{
build.Incoherencies.Add(new Data.Models.BuildIncoherence
{
Name = incoherence.Name,
Version = incoherence.Version,
Repository = incoherence.RepoUri,
Commit = incoherence.Commit
});
}
context.Entry <Data.Models.Build>(build).Reload();
build.Incoherencies = incoherencies;
context.Builds.Update(build);
await context.SaveChangesAsync();
}
}
catch (Exception e)
{
Logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {buildId}");
}
}
public override async Task EvaluateAsync()
{
// Build the repository graph starting at the repo + branch and then look for cycles.
// Build without build lookups, toolsets, etc. to minimize time. Toolset dependencies also break
// product dependency cycles, so that eliminates some analysis
DependencyGraphBuildOptions options = new DependencyGraphBuildOptions
{
EarlyBuildBreak = EarlyBreakOn.NoEarlyBreak,
IncludeToolset = false,
LookupBuilds = false,
NodeDiff = NodeDiff.None,
ComputeCyclePaths = true
};
// Evaluate and find out what the latest is on the branch
var remote = await RemoteFactory.GetRemoteAsync(Repository, Logger);
var commit = await remote.GetLatestCommitAsync(Repository, Branch);
if (commit == null)
{
// If there were no commits, then there can be no cycles. This would be typical of newly
// created branches.
Result = HealthResult.Passed;
Cycles = new List <List <string> >();
return;
}
DependencyGraph graph =
await DependencyGraph.BuildRemoteDependencyGraphAsync(RemoteFactory, Repository, commit, options, Logger);
// Check to see whether there are any cycles.
// Boil down the cycles into just the repositories involved
Cycles = graph.Cycles.Select(cycle => cycle.Select(graphNode => graphNode.Repository));
if (Cycles.Any())
{
Result = HealthResult.Failed;
}
else
{
Result = HealthResult.Passed;
}
}
public override async Task <int> ExecuteAsync()
{
try
{
IEnumerable <DependencyDetail> rootDependencies = null;
DependencyGraph graph;
RemoteFactory remoteFactory = new RemoteFactory(_options);
if (!_options.Local)
{
NodeDiff diffOption = NodeDiff.None;
// Check node diff options
switch (_options.DeltaFrom.ToLowerInvariant())
{
case "none":
break;
case "newest-in-channel":
diffOption = NodeDiff.LatestInChannel;
break;
case "newest-in-graph":
diffOption = NodeDiff.LatestInGraph;
break;
default:
Console.WriteLine("Unknown --delta-from option, please see help.");
return(Constants.ErrorCode);
}
// If the repo uri and version are set, then call the graph
// build operation based on those. Both should be set in this case.
// If they are not set, then gather the initial set based on the local repository,
// and then call the graph build with that root set.
if (!string.IsNullOrEmpty(_options.RepoUri))
{
if (string.IsNullOrEmpty(_options.Version))
{
Console.WriteLine("If --repo is set, --version should be supplied");
return(Constants.ErrorCode);
}
Console.WriteLine($"Getting root dependencies from {_options.RepoUri}@{_options.Version}...");
// Grab root dependency set. The graph build can do this, but
// if an original asset name is passed, then this will do the initial filtering.
IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(_options.RepoUri, Logger);
rootDependencies = await rootRepoRemote.GetDependenciesAsync(
_options.RepoUri,
_options.Version,
_options.AssetName);
}
else
{
if (!string.IsNullOrEmpty(_options.Version))
{
Console.WriteLine("If --version is supplied, then --repo is required");
return(Constants.ErrorCode);
}
Console.WriteLine($"Getting root dependencies from local repository...");
// Grab root dependency set from local repo
Local local = new Local(Logger);
rootDependencies = await local.GetDependenciesAsync(
_options.AssetName);
}
Console.WriteLine($"Building repository dependency graph...");
rootDependencies = FilterToolsetDependencies(rootDependencies);
if (!rootDependencies.Any())
{
Console.WriteLine($"No root dependencies found, exiting.");
return(Constants.ErrorCode);
}
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = diffOption != NodeDiff.None || !_options.SkipBuildLookup,
NodeDiff = diffOption
};
// Build graph
graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootDependencies,
_options.RepoUri ?? LocalHelpers.GetRootDir(Logger),
_options.Version ?? LocalHelpers.GetGitCommit(Logger),
graphBuildOptions,
Logger);
}
else
{
Console.WriteLine($"Getting root dependencies from local repository...");
Local local = new Local(Logger);
rootDependencies = await local.GetDependenciesAsync(
_options.AssetName);
rootDependencies = FilterToolsetDependencies(rootDependencies);
if (!rootDependencies.Any())
{
Console.WriteLine($"No root dependencies found, exiting.");
return(Constants.ErrorCode);
}
Console.WriteLine($"Building repository dependency graph from local information...");
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = false,
NodeDiff = NodeDiff.None
};
// Build graph using only local resources
graph = await DependencyGraph.BuildLocalDependencyGraphAsync(
rootDependencies,
graphBuildOptions,
Logger,
LocalHelpers.GetRootDir(Logger),
LocalHelpers.GetGitCommit(Logger),
_options.ReposFolder,
_options.RemotesMap);
}
if (_options.Flat)
{
await LogFlatDependencyGraph(graph);
}
else
{
await LogDependencyGraph(graph);
}
if (!string.IsNullOrEmpty(_options.GraphVizOutputFile))
{
await LogGraphViz(graph);
}
return(Constants.SuccessCode);
}
catch (Exception exc)
{
Logger.LogError(exc, "Something failed while getting the dependency graph.");
return(Constants.ErrorCode);
}
}
/// <summary>
/// Build the list of builds that will need their assets downloaded.
/// </summary>
/// <returns>List of builds to download</returns>
/// <remarks>
/// This can be pretty simple if a full build download is
/// not desired (just determine the root build and return it) or it could
/// be a matter of determining all builds that contributed to all dependencies.
/// </remarks>
private async Task <InputBuilds> GatherBuildsToDownloadAsync()
{
Console.WriteLine("Determining what builds to download...");
// Gather the root build
Build rootBuild = await GetRootBuildAsync();
if (rootBuild == null)
{
return(new InputBuilds {
Successful = false
});
}
Console.WriteLine($"Root build - Build number {rootBuild.AzureDevOpsBuildNumber} of {rootBuild.AzureDevOpsRepository} @ {rootBuild.Commit}");
// If transitive (full tree) was not selected, we're done
if (!_options.Transitive)
{
return(new InputBuilds()
{
Successful = true,
Builds = new List <Build>()
{
rootBuild
}
});
}
HashSet <Build> builds = new HashSet <Build>(new BuildComparer());
builds.Add(rootBuild);
IRemoteFactory remoteFactory = new RemoteFactory(_options);
// Grab dependencies
IRemote rootBuildRemote = await remoteFactory.GetRemoteAsync(rootBuild.AzureDevOpsRepository, Logger);
Console.WriteLine($"Getting dependencies of root build...");
// Flatten for convencience and remove dependencies of types that we don't want if need be.
if (!_options.IncludeToolset)
{
Console.WriteLine("Filtering toolset dependencies from the graph...");
}
DependencyGraphBuildOptions buildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
Console.WriteLine("Building graph of all dependencies under root build...");
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootBuild.GitHubRepository ?? rootBuild.AzureDevOpsRepository,
rootBuild.Commit,
buildOptions,
Logger);
Dictionary <DependencyDetail, Build> dependencyCache =
new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer());
// Cache root build's assets
foreach (Asset buildAsset in rootBuild.Assets)
{
dependencyCache.Add(
new DependencyDetail
{
Name = buildAsset.Name,
Version = buildAsset.Version,
Commit = rootBuild.Commit,
},
rootBuild);
}
Console.WriteLine($"There are {graph.UniqueDependencies.Count()} unique dependencies in the graph.");
Console.WriteLine("Full set of builds in graph:");
foreach (var build in graph.ContributingBuilds)
{
Console.WriteLine($" Build - {build.AzureDevOpsBuildNumber} of {build.GitHubRepository ?? build.AzureDevOpsRepository} @ {build.Commit}");
builds.Add(build);
}
if (graph.DependenciesMissingBuilds.Any())
{
Console.WriteLine("Dependencies missing builds:");
foreach (DependencyDetail dependency in graph.DependenciesMissingBuilds)
{
Console.WriteLine($" {dependency.Name}@{dependency.Version} @ ({dependency.RepoUri}@{dependency.Commit})");
}
if (!_options.ContinueOnError)
{
return(new InputBuilds()
{
Successful = false,
Builds = builds
});
}
}
return(new InputBuilds()
{
Successful = true,
Builds = builds.ToList()
});
}
