/// <summary>
/// Add the assets from each build to the cache.
/// Also evaluate whether we see any of the assets that we are supposed to break
/// on, and remove them from the break on set if so.
/// </summary>
/// <param name="build">Build producing assets</param>
/// <param name="dependencyCache">Dependency cache</param>
/// <param name="earlyBreakOnType">Early break on type</param>
/// <param name="breakOn">Hash set of assets. Any assets in the <paramref name="build"/>
/// that exist in <paramref name="breakOn"/> will be removed from
/// <paramref name="breakOn"/> if <paramref name="earlyBreakOnType"/> is "Assets"</param>
private static void AddAssetsToBuildCache(
Build build,
Dictionary <DependencyDetail, Build> dependencyCache,
EarlyBreakOnType earlyBreakOnType,
HashSet <string> breakOn)
{
foreach (Asset buildAsset in build.Assets)
{
DependencyDetail newDependency =
new DependencyDetail()
{
Name = buildAsset.Name, Version = buildAsset.Version, Commit = build.Commit
};
// Possible that the same asset could be listed multiple times in a build, so avoid accidentally adding
// things multiple times
if (!dependencyCache.ContainsKey(newDependency))
{
dependencyCache.Add(newDependency, build);
}
if (earlyBreakOnType == EarlyBreakOnType.Assets)
{
breakOn.Remove(buildAsset.Name);
}
}
}
/// <summary>
/// Creates a new dependency graph
/// </summary>
/// <param name="remoteFactory">Remote for factory for obtaining remotes to</param>
/// <param name="rootDependencies">Root set of dependencies. If null, then repoUri and commit should be set</param>
/// <param name="repoUri">Root repository uri. Must be valid if no root dependencies are passed.</param>
/// <param name="commit">Root commit. Must be valid if no root dependencies were passed.</param>
/// <param name="includeToolset">If true, toolset dependencies are included.</param>
/// <param name="lookupBuilds">If true, the builds contributing to each node are looked up. Must be a remote build.</param>
/// <param name="remote">If true, remote graph build is used.</param>
/// <param name="logger">Logger</param>
/// <param name="reposFolder">Path to repos</param>
/// <param name="remotesMap">Map of remotes (e.g. https://github.com/dotnet/corefx) to folders</param>
/// <param name="testPath">If running unit tests, commits will be looked up as folders under this path</param>
/// <returns>New dependency graph</returns>
private static async Task <DependencyGraph> BuildDependencyGraphImplAsync(
IRemoteFactory remoteFactory,
IEnumerable <DependencyDetail> rootDependencies,
string repoUri,
string commit,
DependencyGraphBuildOptions options,
bool remote,
ILogger logger,
string reposFolder,
IEnumerable <string> remotesMap,
string testPath)
{
ValidateBuildOptions(remoteFactory, rootDependencies, repoUri, commit,
options, remote, logger, reposFolder, remotesMap, testPath);
if (rootDependencies != null)
{
logger.LogInformation($"Starting build of graph from {rootDependencies.Count()} root dependencies " +
$"({repoUri}@{commit})");
foreach (DependencyDetail dependency in rootDependencies)
{
logger.LogInformation($" {dependency.Name}@{dependency.Version}");
}
}
else
{
logger.LogInformation($"Starting build of graph from ({repoUri}@{commit})");
}
AssetComparer assetEqualityComparer = new AssetComparer();
HashSet <Build> allContributingBuilds = null;
HashSet <DependencyDetail> dependenciesMissingBuilds = null;
HashSet <Build> rootNodeBuilds = null;
Dictionary <DependencyDetail, Build> dependencyCache =
new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer());
List <LinkedList <DependencyGraphNode> > cycles = new List <LinkedList <DependencyGraphNode> >();
EarlyBreakOnType breakOnType = options.EarlyBuildBreak.Type;
HashSet <string> breakOn = null;
if (breakOnType != EarlyBreakOnType.None)
{
breakOn = new HashSet <string>(options.EarlyBuildBreak.BreakOn, StringComparer.OrdinalIgnoreCase);
}
if (options.LookupBuilds)
{
allContributingBuilds = new HashSet <Build>(new BuildComparer());
dependenciesMissingBuilds = new HashSet <DependencyDetail>(new DependencyDetailComparer());
rootNodeBuilds = new HashSet <Build>(new BuildComparer());
// Look up the dependency and get the creating build.
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
IEnumerable <Build> potentialRootNodeBuilds = await barOnlyRemote.GetBuildsAsync(repoUri, commit);
// Filter by those actually producing the root dependencies, if they were supplied
if (rootDependencies != null)
{
potentialRootNodeBuilds = potentialRootNodeBuilds.Where(b =>
b.Assets.Any(a => rootDependencies.Any(d => assetEqualityComparer.Equals(a, d))));
}
// It's entirely possible that the root has no builds (e.g. change just checked in).
// Don't record those. Instead, users of the graph should just look at the
// root node's contributing builds and determine whether it's empty or not.
foreach (Build build in potentialRootNodeBuilds)
{
allContributingBuilds.Add(build);
rootNodeBuilds.Add(build);
AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
}
}
// Create the root node and add the repo to the visited bit vector.
DependencyGraphNode rootGraphNode = new DependencyGraphNode(repoUri, commit, rootDependencies, rootNodeBuilds);
rootGraphNode.VisitedNodes.Add(repoUri);
// Nodes to visit is a queue, so that the evaluation order
// of the graph is breadth first.
Queue <DependencyGraphNode> nodesToVisit = new Queue <DependencyGraphNode>();
nodesToVisit.Enqueue(rootGraphNode);
HashSet <DependencyDetail> uniqueDependencyDetails;
if (rootGraphNode.Dependencies != null)
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
rootGraphNode.Dependencies,
new DependencyDetailComparer());
// Remove the dependencies details from the
// break on if break on type is Dependencies
if (breakOnType == EarlyBreakOnType.Dependencies)
{
rootGraphNode.Dependencies.Select(d => breakOn.Remove(d.Name));
}
}
else
{
uniqueDependencyDetails = new HashSet <DependencyDetail>(
new DependencyDetailComparer());
}
// If we already found the assets/dependencies we wanted, clear the
// visit list and we'll drop through.
if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
{
logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
nodesToVisit.Clear();
}
// Cache of nodes we've visited. If we reach a repo/commit combo already in the cache,
// we can just add these nodes as a child. The cache key is '{repoUri}@{commit}'
Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();
nodeCache.Add($"{rootGraphNode.Repository}@{rootGraphNode.Commit}", rootGraphNode);
// Cache of incoherent nodes, looked up by repo URI.
Dictionary <string, DependencyGraphNode> visitedRepoUriNodes = new Dictionary <string, DependencyGraphNode>();
HashSet <DependencyGraphNode> incoherentNodes = new HashSet <DependencyGraphNode>();
// Cache of incoherent dependencies, looked up by name
Dictionary <string, DependencyDetail> incoherentDependenciesCache = new Dictionary <string, DependencyDetail>();
HashSet <DependencyDetail> incoherentDependencies = new HashSet <DependencyDetail>();
while (nodesToVisit.Count > 0)
{
DependencyGraphNode node = nodesToVisit.Dequeue();
logger.LogInformation($"Visiting {node.Repository}@{node.Commit}");
IEnumerable <DependencyDetail> dependencies;
// In case of the root node which is initially put on the stack,
// we already have the set of dependencies to start at (this may have been
// filtered by the caller). So no need to get the dependencies again.
if (node.Dependencies != null)
{
dependencies = node.Dependencies;
}
else
{
logger.LogInformation($"Getting dependencies at {node.Repository}@{node.Commit}");
dependencies = await GetDependenciesAsync(
remoteFactory,
remote,
logger,
node.Repository,
node.Commit,
options.IncludeToolset,
remotesMap,
reposFolder,
testPath);
// Set the dependencies on the current node.
node.Dependencies = dependencies;
}
if (dependencies != null)
{
foreach (DependencyDetail dependency in dependencies)
{
// If this dependency is missing information, then skip it.
if (string.IsNullOrEmpty(dependency.RepoUri) ||
string.IsNullOrEmpty(dependency.Commit))
{
logger.LogInformation($"Dependency {dependency.Name}@{dependency.Version} in " +
$"{node.Repository}@{node.Commit} " +
$"is missing repository uri or commit information, skipping");
continue;
}
// If the dependency's repo uri has been traversed, we've reached a cycle in this subgraph
// and should break.
if (node.VisitedNodes.Contains(dependency.RepoUri))
{
logger.LogInformation($"Node {node.Repository}@{node.Commit} " +
$"introduces a cycle to {dependency.RepoUri}, skipping");
if (options.ComputeCyclePaths)
{
var newCycles = ComputeCyclePaths(node, dependency.RepoUri);
cycles.AddRange(newCycles);
}
continue;
}
// Add the individual dependency to the set of unique dependencies seen
// in the whole graph.
uniqueDependencyDetails.Add(dependency);
if (incoherentDependenciesCache.TryGetValue(dependency.Name, out DependencyDetail existingDependency))
{
incoherentDependencies.Add(existingDependency);
incoherentDependencies.Add(dependency);
}
else
{
incoherentDependenciesCache.Add(dependency.Name, dependency);
}
HashSet <Build> nodeContributingBuilds = null;
if (options.LookupBuilds)
{
nodeContributingBuilds = new HashSet <Build>(new BuildComparer());
// Look up dependency in cache first
if (dependencyCache.TryGetValue(dependency, out Build existingBuild))
{
nodeContributingBuilds.Add(existingBuild);
allContributingBuilds.Add(existingBuild);
}
else
{
// Look up the dependency and get the creating build.
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
IEnumerable <Build> potentiallyContributingBuilds = await barOnlyRemote.GetBuildsAsync(dependency.RepoUri, dependency.Commit);
// Filter by those actually producing the dependency. Most of the time this won't
// actually result in a different set of contributing builds, but should avoid any subtle bugs where
// there might be overlap between repos, or cases where there were multiple builds at the same sha.
potentiallyContributingBuilds = potentiallyContributingBuilds.Where(b =>
b.Assets.Any(a => assetEqualityComparer.Equals(a, dependency)));
if (!potentiallyContributingBuilds.Any())
{
// Couldn't find a build that produced the dependency.
dependenciesMissingBuilds.Add(dependency);
}
else
{
foreach (Build build in potentiallyContributingBuilds)
{
allContributingBuilds.Add(build);
nodeContributingBuilds.Add(build);
AddAssetsToBuildCache(build, dependencyCache, breakOnType, breakOn);
}
}
}
}
// We may have visited this node before. If so, add it as a child and avoid additional walks.
// Update the list of contributing builds.
if (nodeCache.TryGetValue($"{dependency.RepoUri}@{dependency.Commit}", out DependencyGraphNode existingNode))
{
if (options.LookupBuilds)
{
// Add the contributing builds. It's possible that
// different dependencies on a single node (repo/sha) were produced
// from multiple builds
foreach (Build build in nodeContributingBuilds)
{
existingNode.ContributingBuilds.Add(build);
}
}
logger.LogInformation($"Node {dependency.RepoUri}@{dependency.Commit} has already been created, adding as child");
node.AddChild(existingNode, dependency);
continue;
}
// Otherwise, create a new node for this dependency.
DependencyGraphNode newNode = new DependencyGraphNode(
dependency.RepoUri,
dependency.Commit,
null,
node.VisitedNodes,
nodeContributingBuilds);
// Cache the dependency and add it to the visitation stack.
nodeCache.Add($"{dependency.RepoUri}@{dependency.Commit}", newNode);
nodesToVisit.Enqueue(newNode);
newNode.VisitedNodes.Add(dependency.RepoUri);
node.AddChild(newNode, dependency);
// Calculate incoherencies. If we've not yet visited the repo uri, add the
// new node based on its repo uri. Otherwise, add both the new node and the visited
// node to the incoherent nodes.
if (visitedRepoUriNodes.TryGetValue(dependency.RepoUri, out DependencyGraphNode visitedNode))
{
incoherentNodes.Add(visitedNode);
incoherentNodes.Add(newNode);
}
else
{
visitedRepoUriNodes.Add(newNode.Repository, newNode);
}
// If breaking on dependencies, then decide whether we need to break
// here.
if (breakOnType == EarlyBreakOnType.Dependencies)
{
breakOn.Remove(dependency.Name);
}
if (breakOnType != EarlyBreakOnType.None && breakOn.Count == 0)
{
logger.LogInformation($"Stopping graph build after finding all assets/dependencies.");
nodesToVisit.Clear();
break;
}
}
}
}
switch (options.NodeDiff)
{
case NodeDiff.None:
// Nothing
break;
case NodeDiff.LatestInGraph:
await DoLatestInGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
case NodeDiff.LatestInChannel:
await DoLatestInChannelGraphNodeDiffAsync(remoteFactory, logger, nodeCache, visitedRepoUriNodes);
break;
}
return(new DependencyGraph(rootGraphNode,
uniqueDependencyDetails,
incoherentDependencies,
nodeCache.Values,
incoherentNodes,
allContributingBuilds,
dependenciesMissingBuilds,
cycles));
}
