private async Task <DependencyGraphNode> BuildGraphAtDependency(
IRemoteFactory remoteFactory,
DependencyDetail rootDependency,
List <DependencyDetail> updateList,
Dictionary <string, DependencyGraphNode> nodeCache)
{
DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = true,
LookupBuilds = true,
NodeDiff = NodeDiff.None,
EarlyBuildBreak = new EarlyBreakOn
{
Type = EarlyBreakOnType.Assets,
BreakOn = new List <string>(updateList.Select(d => d.Name))
}
};
DependencyGraph dependencyGraph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory, null, rootDependency.RepoUri, rootDependency.Commit, dependencyGraphBuildOptions, _logger);
// Cache all nodes in this built graph.
foreach (DependencyGraphNode node in dependencyGraph.Nodes)
{
if (!nodeCache.ContainsKey($"{node.Repository}@{node.Commit}"))
{
nodeCache.Add($"{node.Repository}@{node.Commit}", node);
}
}
return(dependencyGraph.Root);
}
/// <summary>
/// Builds a dependency graph given a root repo and commit using remotes.
/// </summary>
/// <param name="remoteFactory">Factory that can create remotes based on repo uris</param>
/// <param name="repoUri">Root repository URI</param>
/// <param name="commit">Root commit</param>
/// <param name="options">Graph build options.</param>
/// <param name="logger">Logger</param>
/// <returns>New dependency graph.</returns>
public static async Task <DependencyGraph> BuildRemoteDependencyGraphAsync(
IRemoteFactory remoteFactory,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
ILogger logger)
{
return(await BuildDependencyGraphImplAsync(
remoteFactory,
null, /* no initial root dependencies */
repoUri,
commit,
options,
true,
logger,
null,
null,
null));
}
/// <summary>
/// Validate that the graph build options are correct.
/// </summary>
/// <param name="remoteFactory"></param>
/// <param name="rootDependencies"></param>
/// <param name="repoUri"></param>
/// <param name="commit"></param>
/// <param name="options"></param>
/// <param name="remote"></param>
/// <param name="logger"></param>
/// <param name="reposFolder"></param>
/// <param name="remotesMap"></param>
/// <param name="testPath"></param>
private static void ValidateBuildOptions(
IRemoteFactory remoteFactory,
IEnumerable <DependencyDetail> rootDependencies,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
bool remote,
ILogger logger,
string reposFolder,
IEnumerable <string> remotesMap,
string testPath)
{
// Fail fast if darcSettings is null in a remote scenario
if (remote && remoteFactory == null)
{
throw new DarcException("Remote graph build requires a remote factory.");
}
if (rootDependencies != null && !rootDependencies.Any())
{
throw new DarcException("Root dependencies were not supplied.");
}
if (!remote)
{
if (options.LookupBuilds)
{
throw new DarcException("Build lookup only available in remote build mode.");
}
if (options.NodeDiff != NodeDiff.None)
{
throw new DarcException($"Node diff type '{options.NodeDiff}' only available in remote build mode.");
}
}
else
{
if (options.NodeDiff != NodeDiff.None && !options.LookupBuilds)
{
throw new DarcException("Node diff requires build lookup.");
}
}
}
/// <summary>
/// Builds a dependency graph given a root repo and commit.
/// </summary>
/// <param name="remoteFactory">Factory that can create remotes based on repo uris</param>
/// <param name="rootDependencies">Root set of dependencies</param>
/// <param name="repoUri">Root repository URI</param>
/// <param name="commit">Root commit</param>
/// <param name="options">Graph build options.</param>
/// <param name="logger">Logger</param>
/// <returns>New dependency graph.</returns>
public static async Task <DependencyGraph> BuildRemoteDependencyGraphAsync(
IRemoteFactory remoteFactory,
IEnumerable <DependencyDetail> rootDependencies,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
ILogger logger)
{
return(await BuildDependencyGraphImplAsync(
remoteFactory,
rootDependencies,
repoUri,
commit,
options,
true,
logger,
null,
null,
null));
}
/// <summary>
/// Builds a dependency graph using only local resources
/// </summary>
/// <param name="remoteFactory">Factory that can create remotes based on repo uris</param>
/// <param name="rootDependencies">Root set of dependencies</param>
/// <param name="rootRepoFolder">Root repository folder</param>
/// <param name="rootRepoCommit">Root commit</param>
/// <param name="options">Graph build options</param>
/// <param name="logger">Logger</param>
/// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param>
/// <param name="remotesMap">Map of remote uris to local paths</param>
/// <param name="reposFolder">Folder containing local repositories.</param>
/// <returns>New dependency graph.</returns>
public static async Task <DependencyGraph> BuildLocalDependencyGraphAsync(
IEnumerable <DependencyDetail> rootDependencies,
DependencyGraphBuildOptions options,
ILogger logger,
string rootRepoFolder,
string rootRepoCommit,
string reposFolder,
IEnumerable <string> remotesMap,
string testPath = null)
{
return(await BuildDependencyGraphImplAsync(
null,
rootDependencies,
rootRepoFolder,
rootRepoCommit,
options,
false,
logger,
reposFolder,
remotesMap,
testPath));
}
/// <summary>
/// Creates a new dependency graph
/// </summary>
/// <param name="remoteFactory">Remote for factory for obtaining remotes to</param>
/// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param>
/// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param>
/// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param>
/// <param name="includeToolset">If true, toolset dependencies are included.</param>
/// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param>
/// <param name="remote">If true, remote graph build is used.</param>
/// <param name="logger">Logger</param>
/// <param name="reposFolder">Path to repos</param>
/// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param>
/// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param>
/// <returns>New dependency graph</returns>
private static async Task <DependencyGraph> BuildDependencyGraphImplAsync(
IRemoteFactory remoteFactory,
IEnumerable <DependencyDetail> rootDependencies,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
bool remote,
ILogger logger,
string reposFolder,
IEnumerable <string> remotesMap,
string testPath)
{
ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit,
options, remote, logger, reposFolder, remotesMap, testPath);
if (rootDependencies != null)
{
logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " +
$"({repoUri}@{commit})");
foreach (DependencyDetail dependency in rootDependencies)
{
logger.LogInformation($" {dependency.Name}@{dependency.Version}");
}
}
else
{
logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})");
}
AssetComparer assetEqualityComparer = new AssetComparer();
HashSet <Build> allContributingBuilds = null;
HashSet <DependencyDetail> dependenciesMissingBuilds = null;
HashSet <Build> rootNodeBuilds = null;
Dictionary <DependencyDetail, Build> dependencyCache =
new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer());
List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >();
EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type;
HashSet <string> breakOn = null;
if (breakOnType != EarlyBreakOnType.None)
{
breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase);
}
if (options.LookupBuilds)
{
allContributingBuilds = new HashSet <Build>(new BuildComparer());
dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer());
rootNodeBuilds = new HashSet <Build>(new BuildComparer());
// Look up the dependency and get the creating build.
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit);
// Filter by those actually producing the root dependencies, if they were supplied
if (rootDependencies != null)
{
potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b =>
b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d))));
}
// It's entirely possible that the root has no builds (e.g. change just checked in).
// Don't record those. Instead, users of the graph should just look at the
// root node's contributing builds and determine whether it's empty or not.
foreach (Build build in potentialRootNodeBuilds)
{
allContributingBuilds.Add(build);
rootNodeBuilds.Add(build);
AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
}
}
// Create the root node and add the repo to the visited bit vector.
DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds);
rootGraphNode.VisitedNodes.Add(repoUri);
// Nodes to visit is a queue, so that the evaluation order
// of the graph is breadth first.
Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>();
nodesToVisit.Enqueue(rootGraphNode);
HashSet <DependencyDetail> uniqueDependencyDetails;
if (rootGraphNode.Dependencies != null)
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
rootGraphNode.Dependencies,
new DependencyDetailComparer());
// Remove the dependencies details from the
// break on if break on type is Dependencies
if (breakOnType == EarlyBreakOnType.Dependencies)
{
rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name));
}
}
else
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
new DependencyDetailComparer());
}
// If we already found the assets/dependencies we wanted, clear the
// visit list and we'll drop through.
if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
{
logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
nodesToVisit.Clear();
}
// Cache of nodes we've visited. If we reach a repo/commit combo already in the cache,
// we can just add these nodes as a child. The cache key is '{repoUri}@{commit}'
Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();
nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode);
// Cache of incoherent nodes, looked up by repo URI.
Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>();
HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>();
// Cache of incoherent dependencies, looked up by name
Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>();
HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>();
while (nodesToVisit.Count > 0)
{
DependencyGraphNode node = nodesToVisit.Dequeue();
logger.LogInformation($"Visiting {node.Repository}@{node.Commit}");
IEnumerable <DependencyDetail> dependencies;
// In case of the root node which is initially put on the stack,
// we already have the set of dependencies to start at (this may have been
// filtered by the caller). So no need to get the dependencies again.
if (node.Dependencies != null)
{
dependencies = node.Dependencies;
}
else
{
logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}");
dependencies = await GetDependenciesAsync(
remoteFactory,
remote,
logger,
node.Repository,
node.Commit,
options.IncludeToolset,
remotesMap,
reposFolder,
testPath);
// Set the dependencies on the current node.
node.Dependencies = dependencies;
}
if (dependencies != null)
{
foreach (DependencyDetail dependency in dependencies)
{
// If this dependency is missing information, then skip it.
if (string.IsNullOrEmpty(dependency.RepoUri) ||
string.IsNullOrEmpty(dependency.Commit))
{
logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " +
$"{node.Repository}@{node.Commit} " +
$"is missing repository uri or commit information, skipping");
continue;
}
// If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph
// and should break.
if (node.VisitedNodes.Contains(dependency.RepoUri))
{
logger.LogInformation($"Node {node.Repository}@{node.Commit} " +
$"introduces a cycle to {dependency.RepoUri}, skipping");
if (options.ComputeCyclePaths)
{
var newCycles = ComputeCyclePaths(node, dependency.RepoUri);
cycles.AddRange(newCycles);
}
continue;
}
// Add the individual dependency to the set of unique dependencies seen
// in the whole graph.
uniqueDependencyDetails.Add(dependency);
if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency))
{
incoherentDependencies.Add(existingDependency);
incoherentDependencies.Add(dependency);
}
else
{
incoherentDependenciesCache.Add(dependency.Name, dependency);
}
HashSet <Build> nodeContributingBuilds = null;
if (options.LookupBuilds)
{
nodeContributingBuilds = new HashSet <Build>(new BuildComparer());
// Look up dependency in cache first
if (dependencyCache.TryGetValue(dependency, out Build existingBuild))
{
nodeContributingBuilds.Add(existingBuild);
allContributingBuilds.Add(existingBuild);
}
else
{
// Look up the dependency and get the creating build.
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit);
// Filter by those actually producing the dependency. Most of the time this won't
// actually result in a different set of contributing builds, but should avoid any subtle bugs where
// there might be overlap between repos, or cases where there were multiple builds at the same sha.
potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b =>
b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency)));
if (!potentiallyContributingBuilds.Any())
{
// Couldn't find a build that produced the dependency.
dependenciesMissingBuilds.Add(dependency);
}
else
{
foreach (Build build in potentiallyContributingBuilds)
{
allContributingBuilds.Add(build);
nodeContributingBuilds.Add(build);
AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
}
}
}
}
// We may have visited this node before. If so, add it as a child and avoid additional walks.
// Update the list of contributing builds.
if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode))
{
if (options.LookupBuilds)
{
// Add the contributing builds. It's possible that
// different dependencies on a single node (repo/sha) were produced
// from multiple builds
foreach (Build build in nodeContributingBuilds)
{
existingNode.ContributingBuilds.Add(build);
}
}
logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child");
node.AddChild(existingNode, dependency);
continue;
}
// Otherwise, create a new node for this dependency.
DependencyGraphNode newNode = new DependencyGraphNode(
dependency.RepoUri,
dependency.Commit,
null,
node.VisitedNodes,
nodeContributingBuilds);
// Cache the dependency and add it to the visitation stack.
nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode);
nodesToVisit.Enqueue(newNode);
newNode.VisitedNodes.Add(dependency.RepoUri);
node.AddChild(newNode, dependency);
// Calculate incoherencies. If we've not yet visited the repo uri, add the
// new node based on its repo uri. Otherwise, add both the new node and the visited
// node to the incoherent nodes.
if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode))
{
incoherentNodes.Add(visitedNode);
incoherentNodes.Add(newNode);
}
else
{
visitedRepoUriNodes.Add(newNode.Repository, newNode);
}
// If breaking on dependencies, then decide whether we need to break
// here.
if (breakOnType == EarlyBreakOnType.Dependencies)
{
breakOn.Remove(dependency.Name);
}
if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
{
logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
nodesToVisit.Clear();
break;
}
}
}
}
switch (options.NodeDiff)
{
case NodeDiff.None:
// Nothing
break;
case NodeDiff.LatestInGraph:
await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
case NodeDiff.LatestInChannel:
await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
}
return(new DependencyGraph(rootGraphNode,
uniqueDependencyDetails,
incoherentDependencies,
nodeCache.Values,
incoherentNodes,
allContributingBuilds,
dependenciesMissingBuilds,
cycles));
}
/// <summary>
/// Creates a new dependency graph
/// </summary>
/// <param name="remoteFactory">Remote for factory for obtaining remotes to</param>
/// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param>
/// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param>
/// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param>
/// <param name="includeToolset">If true, toolset dependencies are included.</param>
/// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param>
/// <param name="remote">If true, remote graph build is used.</param>
/// <param name="logger">Logger</param>
/// <param name="reposFolder">Path to repos</param>
/// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param>
/// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param>
/// <returns>New dependency graph</returns>
private static async Task <DependencyGraph> BuildDependencyGraphImplAsync(
IRemoteFactory remoteFactory,
IEnumerable <DependencyDetail> rootDependencies,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
bool remote,
ILogger logger,
string reposFolder,
IEnumerable <string> remotesMap,
string testPath)
{
ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit,
options, remote, logger, reposFolder, remotesMap, testPath);
if (rootDependencies != null)
{
logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " +
$"({repoUri}@{commit})");
foreach (DependencyDetail dependency in rootDependencies)
{
logger.LogInformation($" {dependency.Name}@{dependency.Version}");
}
}
else
{
logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})");
}
IRemote barOnlyRemote = null;
if (remote)
{
// Look up the dependency and get the creating build.
barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
}
List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >();
Dictionary <string, Task <IEnumerable <Build> > > buildLookupTasks = null;
if (options.LookupBuilds)
{
buildLookupTasks = new Dictionary <string, Task <IEnumerable <Build> > >();
// Look up the dependency and get the creating build.
buildLookupTasks.Add($"{repoUri}@{commit}", barOnlyRemote.GetBuildsAsync(repoUri, commit));
}
// Create the root node and add the repo to the visited bit vector.
List <Build> allContributingBuilds = null;
DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, null);
rootGraphNode.VisitedNodes.Add(repoUri);
// Nodes to visit is a queue, so that the evaluation order
// of the graph is breadth first.
Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>();
nodesToVisit.Enqueue(rootGraphNode);
HashSet <DependencyDetail> uniqueDependencyDetails;
if (rootGraphNode.Dependencies != null)
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
rootGraphNode.Dependencies,
new DependencyDetailComparer());
}
else
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
new DependencyDetailComparer());
}
// Cache of nodes we've visited. If we reach a repo/commit combo already in the cache,
// we can just add these nodes as a child. The cache key is '{repoUri}@{commit}'
Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();
nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode);
// Cache of incoherent nodes, looked up by repo URI.
Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>();
HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>();
// Cache of incoherent dependencies, looked up by name
Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>();
HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>();
while (nodesToVisit.Count > 0)
{
DependencyGraphNode node = nodesToVisit.Dequeue();
logger.LogInformation($"Visiting {node.Repository}@{node.Commit}");
IEnumerable <DependencyDetail> dependencies;
// In case of the root node which is initially put on the stack,
// we already have the set of dependencies to start at (this may have been
// filtered by the caller). So no need to get the dependencies again.
if (node.Dependencies != null)
{
dependencies = node.Dependencies;
}
else
{
logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}");
dependencies = await GetDependenciesAsync(
remoteFactory,
remote,
logger,
options.GitExecutable,
node.Repository,
node.Commit,
options.IncludeToolset,
remotesMap,
reposFolder,
testPath);
// Set the dependencies on the current node.
node.Dependencies = dependencies;
}
if (dependencies != null)
{
foreach (DependencyDetail dependency in dependencies)
{
// If this dependency is missing information, then skip it.
if (string.IsNullOrEmpty(dependency.RepoUri) ||
string.IsNullOrEmpty(dependency.Commit))
{
logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " +
$"{node.Repository}@{node.Commit} " +
$"is missing repository uri or commit information, skipping");
continue;
}
if (options.LookupBuilds)
{
if (!buildLookupTasks.ContainsKey($"{dependency.RepoUri}@{dependency.Commit}"))
{
buildLookupTasks.Add($"{dependency.RepoUri}@{dependency.Commit}", barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit));
}
}
// If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph
// and should break.
if (node.VisitedNodes.Contains(dependency.RepoUri))
{
logger.LogInformation($"Node {node.Repository}@{node.Commit} " +
$"introduces a cycle to {dependency.RepoUri}, skipping");
if (options.ComputeCyclePaths)
{
var newCycles = ComputeCyclePaths(node, dependency.RepoUri);
cycles.AddRange(newCycles);
}
continue;
}
// Add the individual dependency to the set of unique dependencies seen
// in the whole graph.
uniqueDependencyDetails.Add(dependency);
if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency))
{
incoherentDependencies.Add(existingDependency);
incoherentDependencies.Add(dependency);
}
else
{
incoherentDependenciesCache.Add(dependency.Name, dependency);
}
// We may have visited this node before. If so, add it as a child and avoid additional walks.
// Update the list of contributing builds.
if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode))
{
logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child");
node.AddChild(existingNode, dependency);
continue;
}
// Otherwise, create a new node for this dependency.
DependencyGraphNode newNode = new DependencyGraphNode(
dependency.RepoUri,
dependency.Commit,
null,
node.VisitedNodes,
null);
// Cache the dependency and add it to the visitation stack.
nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode);
nodesToVisit.Enqueue(newNode);
newNode.VisitedNodes.Add(dependency.RepoUri);
node.AddChild(newNode, dependency);
// Calculate incoherencies. If we've not yet visited the repo uri, add the
// new node based on its repo uri. Otherwise, add both the new node and the visited
// node to the incoherent nodes.
if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode))
{
incoherentNodes.Add(visitedNode);
incoherentNodes.Add(newNode);
}
else
{
visitedRepoUriNodes.Add(newNode.Repository, newNode);
}
}
}
}
if (options.LookupBuilds)
{
allContributingBuilds = await ComputeContributingBuildsAsync(buildLookupTasks,
nodeCache.Values,
logger);
}
switch (options.NodeDiff)
{
case NodeDiff.None:
// Nothing
break;
case NodeDiff.LatestInGraph:
await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
case NodeDiff.LatestInChannel:
await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
}
return(new DependencyGraph(rootGraphNode,
uniqueDependencyDetails,
incoherentDependencies,
nodeCache.Values,
incoherentNodes,
allContributingBuilds,
cycles));
}
/// <summary>
/// Get updates required by coherency constraints.
/// </summary>
/// <param name="dependencies">Current set of dependencies.</param>
/// <param name="remoteFactory">Remote factory for remote queries.</param>
/// <returns>Dependencies with updates.</returns>
public async Task <List <DependencyUpdate> > GetRequiredCoherencyUpdatesAsync(
IEnumerable <DependencyDetail> dependencies,
IRemoteFactory remoteFactory)
{
List <DependencyUpdate> toUpdate = new List <DependencyUpdate>();
IEnumerable <DependencyDetail> leavesOfCoherencyTrees =
CalculateLeavesOfCoherencyTrees(dependencies);
if (!leavesOfCoherencyTrees.Any())
{
// Nothing to do.
return(toUpdate);
}
DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = true,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
// Now make a walk over coherent dependencies. Note that coherent dependencies could make
// a chain (A->B->C). In all cases we need to walk to the head of the chain, keeping track
// of all elements in the chain. Also note that we are walking all dependencies here, not
// just those that match the incoming AssetData and aligning all of these based on the coherency data.
Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();
HashSet <DependencyDetail> visited = new HashSet <DependencyDetail>();
foreach (DependencyDetail dependency in leavesOfCoherencyTrees)
{
// If the dependency was already updated, then skip it (could have been part of a larger
// dependency chain)
if (visited.Contains(dependency))
{
continue;
}
// Walk to head of dependency tree, keeping track of elements along the way.
// If we hit a pinned dependency in the walk, that means we can't move
// the dependency and therefore it is effectively the "head" of the subtree.
// We will still visit all the elements in the chain eventually in this algorithm:
// Consider A->B(pinned)->C(pinned)->D.
List <DependencyDetail> updateList = new List <DependencyDetail>();
DependencyDetail currentDependency = dependency;
while (!string.IsNullOrEmpty(currentDependency.CoherentParentDependencyName) && !currentDependency.Pinned)
{
updateList.Add(currentDependency);
DependencyDetail parentCoherentDependency = dependencies.FirstOrDefault(d =>
d.Name.Equals(currentDependency.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase));
currentDependency = parentCoherentDependency ?? throw new DarcException($"Dependency {currentDependency.Name} has non-existent parent " +
$"dependency {currentDependency.CoherentParentDependencyName}");
}
DependencyGraphNode rootNode = null;
// Build the graph to find the assets if we don't have the root in the cache.
// The graph build is automatically broken when
// all the desired assets are found (breadth first search). This means the cache may or
// may not contain a complete graph for a given node. So, we first search the cache for the desired assets,
// then if not found (or if there was no cache), we then build the graph from that node.
bool nodeFromCache = nodeCache.TryGetValue($"{currentDependency.RepoUri}@{currentDependency.Commit}", out rootNode);
if (!nodeFromCache)
{
_logger.LogInformation($"Node not found in cache, starting graph build at " +
$"{currentDependency.RepoUri}@{currentDependency.Commit}");
rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, updateList, nodeCache);
}
List <DependencyDetail> leftToFind = new List <DependencyDetail>(updateList);
// Now do the lookup to find the element in the tree for each item in the update list
foreach (DependencyDetail dependencyInUpdateChain in updateList)
{
(Asset coherentAsset, Build buildForAsset) =
FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode);
// If we originally got the root node from the cache the graph may be incomplete.
// Rebuild to attempt to find all the assets we have left to find. If we still can't find, or if
// the root node did not come the cache, then we're in an invalid state.
if (coherentAsset == null && nodeFromCache)
{
_logger.LogInformation($"Asset {dependencyInUpdateChain.Name} was not found in cached graph, rebuilding from " +
$"{currentDependency.RepoUri}@{currentDependency.Commit}");
rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, leftToFind, nodeCache);
// And attempt to find again.
(coherentAsset, buildForAsset) =
FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode);
}
if (coherentAsset == null)
{
// This is an invalid state. We can't satisfy the
// constraints so they should either be removed or pinned.
throw new DarcException($"Unable to update {dependencyInUpdateChain.Name} to have coherency with " +
$"parent {dependencyInUpdateChain.CoherentParentDependencyName}. No matching asset found in tree. " +
$"Either remove the coherency attribute or mark as pinned.");
}
else
{
leftToFind.Remove(dependencyInUpdateChain);
}
string buildRepoUri = buildForAsset.GitHubRepository ?? buildForAsset.AzureDevOpsRepository;
if (dependencyInUpdateChain.Name == coherentAsset.Name &&
dependencyInUpdateChain.Version == coherentAsset.Version &&
dependencyInUpdateChain.Commit == buildForAsset.Commit &&
dependencyInUpdateChain.RepoUri == buildRepoUri)
{
continue;
}
DependencyDetail updatedDependency = new DependencyDetail(dependencyInUpdateChain)
{
Name = coherentAsset.Name,
Version = coherentAsset.Version,
RepoUri = buildRepoUri,
Commit = buildForAsset.Commit
};
toUpdate.Add(new DependencyUpdate
{
From = dependencyInUpdateChain,
To = updatedDependency
});
visited.Add(dependencyInUpdateChain);
}
}
return(toUpdate);
}
