private static async Task CloneRemoteRepoAndDependencies(RemoteFactory remoteFactory, string reposFolder, string repoUri, string commit, bool includeToolset, ILogger logger)
{
IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger);
IEnumerable <DependencyDetail> rootDependencies = await rootRepoRemote.GetDependenciesAsync(repoUri, commit);
rootDependencies = FilterToolsetDependencies(rootDependencies, includeToolset);
if (!rootDependencies.Any())
{
string repoPath = GetRepoDirectory(reposFolder, repoUri, commit);
if (Directory.Exists(repoPath))
{
logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping");
}
else
{
logger.LogInformation($"Remote repo {repoUri}@{commit} has no dependencies. Cloning shallowly into {repoPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repoUri, logger);
repoRemote.Clone(repoUri, commit, repoPath);
}
return;
}
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = includeToolset,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
logger.LogDebug($"Building depdendency graph for {repoUri}@{commit} with {rootDependencies.Count()} dependencies");
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootDependencies,
repoUri,
commit,
graphBuildOptions,
logger);
foreach (DependencyGraphNode repo in graph.Nodes)
{
string repoPath = GetRepoDirectory(reposFolder, repo.Repository, repo.Commit);
if (Directory.Exists(repoPath))
{
logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping");
}
else
{
logger.LogInformation($"Cloning {repo.Repository}@{repo.Commit} into {repoPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repo.Repository, logger);
repoRemote.Clone(repo.Repository, repo.Commit, repoPath);
}
}
}
/// <summary>
/// Evaluate the metric.
/// </summary>
/// <param name="remoteFactory">Remote factory</param>
/// <returns>True if the metric passed, false otherwise</returns>
public override async Task EvaluateAsync()
{
IRemote remote = await RemoteFactory.GetRemoteAsync(Repository, Logger);
Logger.LogInformation("Evaluating subscription health metrics for {repo}@{branch}", Repository, Branch);
// Get subscriptions that target this repo/branch
Subscriptions = (await remote.GetSubscriptionsAsync(targetRepo: Repository))
.Where(s => s.TargetBranch.Equals(Branch, StringComparison.OrdinalIgnoreCase)).ToList();
// Get the dependencies of the repository/branch. Skip pinned and subscriptions tied to another
// dependency (coherent parent), as well as those not selected by the dependency selector.
try
{
Dependencies = (await remote.GetDependenciesAsync(Repository, Branch))
.Where(d => !d.Pinned && string.IsNullOrEmpty(d.CoherentParentDependencyName))
.Where(d => DependencySelector(d))
.ToList();
}
catch (DependencyFileNotFoundException)
{
// When the dependency file is not found, then we're good as long as this repo is not
// targeted by any subscriptions
if (Subscriptions.Any())
{
MissingVersionDetailsFile = true;
Result = HealthResult.Failed;
return;
}
else
{
Result = HealthResult.Passed;
return;
}
}
Dictionary <string, Subscription> latestAssets = await GetLatestAssetsAndComputeConflicts(remote);
ComputeSubscriptionUse(latestAssets);
// Determine the result. A conflict or missing subscription is an error.
// A non-flowing subscription or unused subscription is a warning
if (DependenciesMissingSubscriptions.Any() ||
ConflictingSubscriptions.Any())
{
Result = HealthResult.Failed;
}
else if (UnusedSubscriptions.Any() || DependenciesThatDoNotFlow.Any())
{
Result = HealthResult.Warning;
}
else
{
Result = HealthResult.Passed;
}
}
private static async Task HandleMasterCopyAndCreateGitDir(RemoteFactory remoteFactory, string repoUrl, string masterGitRepoPath, string masterRepoGitDirPath, string gitDirRedirect, ILogger log)
{
log.LogDebug($"Master .gitdir {masterRepoGitDirPath} does not exist");
// The master folder also doesn't exist. Just clone and set everything up for the first time.
if (!Directory.Exists(masterGitRepoPath))
{
log.LogInformation($"Cloning master copy of {repoUrl} into {masterGitRepoPath} with .gitdir path {masterRepoGitDirPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repoUrl, log);
repoRemote.Clone(repoUrl, null, masterGitRepoPath, masterRepoGitDirPath);
}
// The master folder already exists. We are probably resuming with a different --git-dir-parent setting, or the .gitdir parent was cleaned.
else
{
string masterRepoPossibleGitDirPath = Path.Combine(masterGitRepoPath, ".git");
// The master folder has a full .gitdir. Relocate it to the .gitdir parent directory and update to redirect to that.
if (Directory.Exists(masterRepoPossibleGitDirPath))
{
log.LogDebug($".gitdir {masterRepoPossibleGitDirPath} exists in {masterGitRepoPath}");
// Check if the .gitdir is already where we expect it to be first.
if (Path.GetFullPath(masterRepoPossibleGitDirPath) != Path.GetFullPath(masterRepoGitDirPath))
{
log.LogDebug($"Moving .gitdir {masterRepoPossibleGitDirPath} to expected location {masterRepoGitDirPath}");
Directory.Move(masterRepoPossibleGitDirPath, masterRepoGitDirPath);
File.WriteAllText(masterRepoPossibleGitDirPath, gitDirRedirect);
}
}
// The master folder has a .gitdir redirect. Relocate its .gitdir to where we expect and update the redirect.
else if (File.Exists(masterRepoPossibleGitDirPath))
{
log.LogDebug($"Master repo {masterGitRepoPath} has a .gitdir redirect");
string relocatedGitDirPath = File.ReadAllText(masterRepoPossibleGitDirPath).Substring(GitDirRedirectPrefix.Length);
if (Path.GetFullPath(relocatedGitDirPath) != Path.GetFullPath(masterRepoGitDirPath))
{
log.LogDebug($"Existing .gitdir redirect of {relocatedGitDirPath} does not match expected {masterRepoGitDirPath}, moving .gitdir and updating redirect");
Directory.Move(relocatedGitDirPath, masterRepoGitDirPath);
File.WriteAllText(masterRepoPossibleGitDirPath, gitDirRedirect);
}
}
// This repo is orphaned. Since it's supposed to be our master copy, adopt it.
else
{
log.LogDebug($"Master repo {masterGitRepoPath} is orphaned, adding .gitdir redirect");
File.WriteAllText(masterRepoPossibleGitDirPath, gitDirRedirect);
}
}
}
public override async Task EvaluateAsync()
{
// Build the repository graph starting at the repo + branch and then look for cycles.
// Build without build lookups, toolsets, etc. to minimize time. Toolset dependencies also break
// product dependency cycles, so that eliminates some analysis
DependencyGraphBuildOptions options = new DependencyGraphBuildOptions
{
EarlyBuildBreak = EarlyBreakOn.NoEarlyBreak,
IncludeToolset = false,
LookupBuilds = false,
NodeDiff = NodeDiff.None,
ComputeCyclePaths = true
};
// Evaluate and find out what the latest is on the branch
var remote = await RemoteFactory.GetRemoteAsync(Repository, Logger);
var commit = await remote.GetLatestCommitAsync(Repository, Branch);
if (commit == null)
{
// If there were no commits, then there can be no cycles. This would be typical of newly
// created branches.
Result = HealthResult.Passed;
Cycles = new List <List <string> >();
return;
}
DependencyGraph graph =
await DependencyGraph.BuildRemoteDependencyGraphAsync(RemoteFactory, Repository, commit, options, Logger);
// Check to see whether there are any cycles.
// Boil down the cycles into just the repositories involved
Cycles = graph.Cycles.Select(cycle => cycle.Select(graphNode => graphNode.Repository));
if (Cycles.Any())
{
Result = HealthResult.Failed;
}
else
{
Result = HealthResult.Passed;
}
}
private static async Task HandleMasterCopyWithDefaultGitDir(RemoteFactory remoteFactory, string repoUrl, string masterGitRepoPath, string masterRepoGitDirPath, ILogger log)
{
log.LogDebug($"Starting master copy for {repoUrl} in {masterGitRepoPath} with default .gitdir");
// The master folder doesn't exist. Just clone and set everything up for the first time.
if (!Directory.Exists(masterGitRepoPath))
{
log.LogInformation($"Cloning master copy of {repoUrl} into {masterGitRepoPath}");
IRemote repoRemote = await remoteFactory.GetRemoteAsync(repoUrl, log);
repoRemote.Clone(repoUrl, null, masterGitRepoPath, masterRepoGitDirPath);
}
// The master folder already exists. We are probably resuming with a different --git-dir-parent setting, or the .gitdir parent was cleaned.
else
{
log.LogDebug($"Checking for existing .gitdir in {masterGitRepoPath}");
string masterRepoPossibleGitDirPath = Path.Combine(masterGitRepoPath, ".git");
// This repo is not in good shape for us. It needs to be deleted and recreated.
if (!Directory.Exists(masterRepoPossibleGitDirPath))
{
throw new InvalidOperationException($"Repo {masterGitRepoPath} does not have a .git folder, but no git-dir-parent was specified. Please fix the repo or specify a git-dir-parent.");
}
}
}
public override async Task <int> ExecuteAsync()
{
try
{
IEnumerable <DependencyDetail> rootDependencies = null;
DependencyGraph graph;
RemoteFactory remoteFactory = new RemoteFactory(_options);
if (!_options.Local)
{
NodeDiff diffOption = NodeDiff.None;
// Check node diff options
switch (_options.DeltaFrom.ToLowerInvariant())
{
case "none":
break;
case "newest-in-channel":
diffOption = NodeDiff.LatestInChannel;
break;
case "newest-in-graph":
diffOption = NodeDiff.LatestInGraph;
break;
default:
Console.WriteLine("Unknown --delta-from option, please see help.");
return(Constants.ErrorCode);
}
// If the repo uri and version are set, then call the graph
// build operation based on those. Both should be set in this case.
// If they are not set, then gather the initial set based on the local repository,
// and then call the graph build with that root set.
if (!string.IsNullOrEmpty(_options.RepoUri))
{
if (string.IsNullOrEmpty(_options.Version))
{
Console.WriteLine("If --repo is set, --version should be supplied");
return(Constants.ErrorCode);
}
Console.WriteLine($"Getting root dependencies from {_options.RepoUri}@{_options.Version}...");
// Grab root dependency set. The graph build can do this, but
// if an original asset name is passed, then this will do the initial filtering.
IRemote rootRepoRemote = await remoteFactory.GetRemoteAsync(_options.RepoUri, Logger);
rootDependencies = await rootRepoRemote.GetDependenciesAsync(
_options.RepoUri,
_options.Version,
_options.AssetName);
}
else
{
if (!string.IsNullOrEmpty(_options.Version))
{
Console.WriteLine("If --version is supplied, then --repo is required");
return(Constants.ErrorCode);
}
Console.WriteLine($"Getting root dependencies from local repository...");
// Grab root dependency set from local repo
Local local = new Local(Logger);
rootDependencies = await local.GetDependenciesAsync(
_options.AssetName);
}
Console.WriteLine($"Building repository dependency graph...");
rootDependencies = FilterToolsetDependencies(rootDependencies);
if (!rootDependencies.Any())
{
Console.WriteLine($"No root dependencies found, exiting.");
return(Constants.ErrorCode);
}
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = diffOption != NodeDiff.None || !_options.SkipBuildLookup,
NodeDiff = diffOption
};
// Build graph
graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootDependencies,
_options.RepoUri ?? LocalHelpers.GetRootDir(Logger),
_options.Version ?? LocalHelpers.GetGitCommit(Logger),
graphBuildOptions,
Logger);
}
else
{
Console.WriteLine($"Getting root dependencies from local repository...");
Local local = new Local(Logger);
rootDependencies = await local.GetDependenciesAsync(
_options.AssetName);
rootDependencies = FilterToolsetDependencies(rootDependencies);
if (!rootDependencies.Any())
{
Console.WriteLine($"No root dependencies found, exiting.");
return(Constants.ErrorCode);
}
Console.WriteLine($"Building repository dependency graph from local information...");
DependencyGraphBuildOptions graphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = false,
NodeDiff = NodeDiff.None
};
// Build graph using only local resources
graph = await DependencyGraph.BuildLocalDependencyGraphAsync(
rootDependencies,
graphBuildOptions,
Logger,
LocalHelpers.GetRootDir(Logger),
LocalHelpers.GetGitCommit(Logger),
_options.ReposFolder,
_options.RemotesMap);
}
if (_options.Flat)
{
await LogFlatDependencyGraph(graph);
}
else
{
await LogDependencyGraph(graph);
}
if (!string.IsNullOrEmpty(_options.GraphVizOutputFile))
{
await LogGraphViz(graph);
}
return(Constants.SuccessCode);
}
catch (Exception exc)
{
Logger.LogError(exc, "Something failed while getting the dependency graph.");
return(Constants.ErrorCode);
}
}
/// <summary>
/// Build the list of builds that will need their assets downloaded.
/// </summary>
/// <returns>List of builds to download</returns>
/// <remarks>
/// This can be pretty simple if a full build download is
/// not desired (just determine the root build and return it) or it could
/// be a matter of determining all builds that contributed to all dependencies.
/// </remarks>
private async Task <InputBuilds> GatherBuildsToDownloadAsync()
{
Console.WriteLine("Determining what builds to download...");
// Gather the root build
Build rootBuild = await GetRootBuildAsync();
if (rootBuild == null)
{
return(new InputBuilds {
Successful = false
});
}
Console.WriteLine($"Root build - Build number {rootBuild.AzureDevOpsBuildNumber} of {rootBuild.AzureDevOpsRepository} @ {rootBuild.Commit}");
// If transitive (full tree) was not selected, we're done
if (!_options.Transitive)
{
return(new InputBuilds()
{
Successful = true,
Builds = new List <Build>()
{
rootBuild
}
});
}
HashSet <Build> builds = new HashSet <Build>(new BuildComparer());
builds.Add(rootBuild);
IRemoteFactory remoteFactory = new RemoteFactory(_options);
// Grab dependencies
IRemote rootBuildRemote = await remoteFactory.GetRemoteAsync(rootBuild.AzureDevOpsRepository, Logger);
Console.WriteLine($"Getting dependencies of root build...");
// Flatten for convencience and remove dependencies of types that we don't want if need be.
if (!_options.IncludeToolset)
{
Console.WriteLine("Filtering toolset dependencies from the graph...");
}
DependencyGraphBuildOptions buildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = _options.IncludeToolset,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
Console.WriteLine("Building graph of all dependencies under root build...");
DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync(
remoteFactory,
rootBuild.GitHubRepository ?? rootBuild.AzureDevOpsRepository,
rootBuild.Commit,
buildOptions,
Logger);
Dictionary <DependencyDetail, Build> dependencyCache =
new Dictionary <DependencyDetail, Build>(new DependencyDetailComparer());
// Cache root build's assets
foreach (Asset buildAsset in rootBuild.Assets)
{
dependencyCache.Add(
new DependencyDetail
{
Name = buildAsset.Name,
Version = buildAsset.Version,
Commit = rootBuild.Commit,
},
rootBuild);
}
Console.WriteLine($"There are {graph.UniqueDependencies.Count()} unique dependencies in the graph.");
Console.WriteLine("Full set of builds in graph:");
foreach (var build in graph.ContributingBuilds)
{
Console.WriteLine($" Build - {build.AzureDevOpsBuildNumber} of {build.GitHubRepository ?? build.AzureDevOpsRepository} @ {build.Commit}");
builds.Add(build);
}
if (graph.DependenciesMissingBuilds.Any())
{
Console.WriteLine("Dependencies missing builds:");
foreach (DependencyDetail dependency in graph.DependenciesMissingBuilds)
{
Console.WriteLine($" {dependency.Name}@{dependency.Version} @ ({dependency.RepoUri}@{dependency.Commit})");
}
if (!_options.ContinueOnError)
{
return(new InputBuilds()
{
Successful = false,
Builds = builds
});
}
}
return(new InputBuilds()
{
Successful = true,
Builds = builds.ToList()
});
}