private static async Task DoLatestInChannelGraphNodeDiffAsync(
IRemoteFactory remoteFactory,
ILogger logger,
Dictionary <string, DependencyGraphNode> nodeCache,
Dictionary <string, DependencyGraphNode> visitedRepoUriNodes)
{
logger.LogInformation("Running latest in channel node diff.");
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger);
// Walk each node in the graph and diff against the latest build in the channel
// that was also applied to the node.
Dictionary <string, string> latestCommitCache = new Dictionary <string, string>();
foreach (DependencyGraphNode node in nodeCache.Values)
{
// Start with an unknown diff.
node.DiffFrom = GitDiff.UnknownDiff();
if (node.ContributingBuilds.Any())
{
// Choose latest build of node that has a channel.
Build newestBuildWithChannel = node.ContributingBuilds.OrderByDescending(b => b.DateProduced).FirstOrDefault(
b => b.Channels != null && b.Channels.Any());
// If no build was found (e.g. build was flowed without a channel or channel was removed from
// a build, then no diff from latest.
if (newestBuildWithChannel != null)
{
int channelId = newestBuildWithChannel.Channels.First().Id;
// Just choose the first channel. This algorithm is mostly just heuristic.
string latestCommitKey = $"{node.Repository}@{channelId}";
string latestCommit = null;
if (!latestCommitCache.TryGetValue(latestCommitKey, out latestCommit))
{
// Look up latest build in the channel
var latestBuild = await barOnlyRemote.GetLatestBuildAsync(node.Repository, channelId);
// Could be null, if the only build was removed from the channel
if (latestBuild != null)
{
latestCommit = latestBuild.Commit;
}
// Add to cache
latestCommitCache.Add(latestCommitKey, latestCommit);
}
// Perform diff if there is a latest commit.
if (!string.IsNullOrEmpty(latestCommit))
{
IRemote repoRemote = await remoteFactory.GetRemoteAsync(node.Repository, logger);
// This will return a no-diff if latestCommit == node.Commit
node.DiffFrom = await repoRemote.GitDiffAsync(node.Repository, latestCommit, node.Commit);
}
}
}
}
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory)
{
// Read the current dependency files and grab their locations so that nuget.config can be updated appropriately.
// Update the incoming dependencies with locations.
IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync();
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies);
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
// TODO: This needs to be moved into some consistent handling between local/remote and add/update:
// https://github.com/dotnet/arcade/issues/1095
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null)
{
try
{
IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger);
List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common");
foreach (GitFile file in localEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
file.Operation = GitFileOperation.Delete;
filesToUpdate.Add(file);
}
}
}
catch (Exception exc) when
(exc.Message == "Not Found")
{
_logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " +
"where a packages folder was passed and the commit which generated them is not " +
"yet pushed.");
}
}
// Push on local does not commit.
await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null);
}
/// <summary>
/// Diff each node in the graph against the latest build in
/// the graph.
/// </summary>
/// <param name="remoteFactory"></param>
/// <param name="logger"></param>
/// <param name="nodeCache"></param>
/// <param name="visitedRepoUriNodes"></param>
/// <returns></returns>
private static async Task DoLatestInGraphNodeDiffAsync(
IRemoteFactory remoteFactory,
ILogger logger,
Dictionary <string, DependencyGraphNode> nodeCache,
Dictionary <string, DependencyGraphNode> visitedRepoUriNodes)
{
logger.LogInformation("Running latest in graph node diff.");
// Find the build of each repo in the graph, then
// get the diff info from the latest
foreach (string repo in visitedRepoUriNodes.Keys)
{
// Get all nodes with this value
List <DependencyGraphNode> nodes = nodeCache.Values.Where(n => n.Repository == repo).ToList();
// If only one, determine latest
if (nodes.Count > 1)
{
// Find latest
DependencyGraphNode newestNode = null;
Build newestBuild = null;
foreach (DependencyGraphNode node in nodes)
{
if (newestNode == null)
{
newestNode = node;
if (newestNode.ContributingBuilds.Any())
{
newestBuild = newestNode.ContributingBuilds.OrderByDescending(b => b.DateProduced).First();
}
}
else if (node.ContributingBuilds.Any(b => b.DateProduced > newestBuild?.DateProduced))
{
newestNode = node;
newestBuild = newestNode.ContributingBuilds.OrderByDescending(b => b.DateProduced).First();
}
}
// Compare all other nodes to the latest
foreach (DependencyGraphNode node in nodes)
{
IRemote repoRemote = await remoteFactory.GetRemoteAsync(node.Repository, logger);
// If node == newestNode, returns no diff.
node.DiffFrom = await repoRemote.GitDiffAsync(node.Repository, newestNode.Commit, node.Commit);
}
}
else
{
DependencyGraphNode singleNode = nodes.Single();
singleNode.DiffFrom = GitDiff.NoDiff(singleNode.Commit);
}
}
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory)
{
// TODO: This should use known updaters, but today the updaters for global.json can only
// add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095
/*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote);
* foreach (DependencyDetail dependency in dependencies)
* {
* if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function))
* {
* await (Task)function.DynamicInvoke(_fileManager, _repo, dependency);
* }
* else
* {
* defaultUpdates.Add(dependency);
* }
* }*/
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
// TODO: This needs to be moved into some consistent handling between local/remote and add/update:
// https://github.com/dotnet/arcade/issues/1095
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null)
{
try
{
IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger);
List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common");
foreach (GitFile file in localEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
file.Operation = GitFileOperation.Delete;
filesToUpdate.Add(file);
}
}
}
catch (Exception exc) when
(exc.Message == "Not Found")
{
_logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " +
"where a packages folder was passed and the commit which generated them is not " +
"yet pushed.");
}
}
// Push on local does not commit.
await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null);
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory)
{
// Read the current dependency files and grab their locations so that nuget.config can be updated appropriately.
// Update the incoming dependencies with locations.
IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync();
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies);
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
SemanticVersion targetDotNetVersion = null;
IRemote arcadeRemote = null;
if (arcadeItem != null)
{
arcadeRemote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger);
targetDotNetVersion = await arcadeRemote.GetToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit);
}
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies, targetDotNetVersion);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
if (arcadeItem != null)
{
try
{
List <GitFile> engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common");
foreach (GitFile file in localEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
// This is a file in the repo's eng/common folder that isn't present in Arcade at the
// requested SHA so delete it during the update.
// GitFile instances do not have public setters since we insert/retrieve them from an
// In-memory cache during remote updates and we don't want anything to modify the cached,
// references, so add a copy with a Delete FileOperation.
filesToUpdate.Add(new GitFile(
file.FilePath,
file.Content,
file.ContentEncoding,
file.Mode,
GitFileOperation.Delete));
}
}
}
catch (Exception exc) when
(exc.Message == "Not Found")
{
_logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " +
"where a packages folder was passed and the commit which generated them is not " +
"yet pushed.");
}
}
// Push on local does not commit.
await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null);
}
private static async Task <IEnumerable <DependencyDetail> > GetDependenciesAsync(
IRemoteFactory remoteFactory,
bool remote,
ILogger logger,
string repoUri,
string commit,
bool includeToolset,
IEnumerable <string> remotesMap,
string reposFolder,
string testPath = null)
{
try
{
IEnumerable <DependencyDetail> dependencies = null;
if (!string.IsNullOrEmpty(testPath))
{
testPath = Path.Combine(
testPath,
repoUri,
commit);
if (Directory.Exists(testPath))
{
Local local = new Local(logger, testPath);
dependencies = await local.GetDependenciesAsync();
}
}
else if (remote)
{
IRemote remoteClient = await remoteFactory.GetRemoteAsync(repoUri, logger);
dependencies = await remoteClient.GetDependenciesAsync(
repoUri,
commit);
}
else
{
string repoPath = GetRepoPath(repoUri, commit, remotesMap, reposFolder, logger);
if (!string.IsNullOrEmpty(repoPath))
{
Local local = new Local(logger);
string fileContents = LocalHelpers.GitShow(
repoPath,
commit,
VersionFiles.VersionDetailsXml,
logger);
dependencies = local.GetDependenciesFromFileContents(fileContents);
}
}
if (!includeToolset)
{
dependencies = dependencies.Where(dependency => dependency.Type != DependencyType.Toolset);
}
return(dependencies);
}
catch (DependencyFileNotFoundException)
{
// This is not an error. Dependencies can be specified with explicit shas that
// may not have eng/Version.Details.xml at that point.
logger.LogWarning($"{repoUri}@{commit} does not have an eng/Version.Details.xml.");
return(null);
}
catch (Exception exc)
{
logger.LogError(exc, $"Something failed while trying the fetch the " +
$"dependencies of repo '{repoUri}' at sha " +
$"'{commit}'");
throw;
}
}
