private IEnumerable <DependencyDetail> GetDependencyDetails(XmlDocument document, string branch = null)
{
List <DependencyDetail> dependencyDetails = new List <DependencyDetail>();
if (document != null)
{
BuildDependencies(document.DocumentElement.SelectNodes("//Dependency"));
void BuildDependencies(XmlNodeList dependencies)
{
if (dependencies.Count > 0)
{
foreach (XmlNode dependency in dependencies)
{
if (dependency.NodeType != XmlNodeType.Comment && dependency.NodeType != XmlNodeType.Whitespace)
{
DependencyDetail dependencyDetail = new DependencyDetail
{
Branch = branch,
Name = dependency.Attributes["Name"].Value,
RepoUri = dependency.SelectSingleNode("Uri").InnerText,
Commit = dependency.SelectSingleNode("Sha").InnerText,
Version = dependency.Attributes["Version"].Value
};
dependencyDetails.Add(dependencyDetail);
}
}
}
else
{
_logger.LogWarning("No dependencies defined in file.");
}
}
}
else
{
_logger.LogError($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " +
$"Look for exceptions above.");
}
return(dependencyDetails);
}
/// <summary>
/// Update well-known version files.
/// </summary>
/// <param name="versionProps">Versions.props xml document</param>
/// <param name="token">Global.json document</param>
/// <param name="itemToUpdate">Item that needs an update.</param>
/// <remarks>
/// TODO: https://github.com/dotnet/arcade/issues/1095
/// </remarks>
private void UpdateVersionFiles(XmlDocument versionProps, JToken token, DependencyDetail itemToUpdate)
{
string versionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(itemToUpdate.Name);
string alternateVersionElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName(itemToUpdate.Name);
// Select nodes case insensitively, then update the name.
XmlNode packageVersionNode = versionProps.DocumentElement.SelectSingleNode(
$"//*[translate(local-name(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')=" +
$"'{versionElementName.ToLower()}']");
string foundElementName = versionElementName;
// Find alternate names
if (packageVersionNode == null)
{
packageVersionNode = versionProps.DocumentElement.SelectSingleNode(
$"//*[translate(local-name(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')=" +
$"'{alternateVersionElementName.ToLower()}']");
foundElementName = alternateVersionElementName;
}
if (packageVersionNode != null)
{
packageVersionNode.InnerText = itemToUpdate.Version;
// If the node name was updated, then create a new node with the new name, unlink this node
// and create a new one in the same location.
if (packageVersionNode.LocalName != foundElementName)
{
{
XmlNode parentNode = packageVersionNode.ParentNode;
XmlNode newPackageVersionElement = versionProps.CreateElement(
foundElementName,
versionProps.DocumentElement.NamespaceURI);
newPackageVersionElement.InnerText = itemToUpdate.Version;
parentNode.ReplaceChild(newPackageVersionElement, packageVersionNode);
}
}
}
// Update the global json too, even if there was an element in the props file, in case
// it was listed in both
UpdateVersionGlobalJson(itemToUpdate, token);
}
private async Task CommitFilesForPullRequestAsync(string repoUri, string branch, string assetsProducedInCommit, IEnumerable <DependencyDetail> itemsToUpdate, string pullRequestBaseBranch = null)
{
CheckForValidGitClient();
GitFileContentContainer fileContainer = await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch);
List <GitFile> filesToCommit = fileContainer.GetFilesToCommitMap(pullRequestBaseBranch);
// If there is an arcade asset that we need to update we try to update the script files as well
DependencyDetail arcadeItem = itemsToUpdate.Where(i => i.Name.ToLower().Contains("arcade")).FirstOrDefault();
if (arcadeItem != null &&
repoUri != arcadeItem.RepoUri)
{
List <GitFile> engCommonsFiles = await GetScriptFilesAsync(arcadeItem.RepoUri, assetsProducedInCommit);
filesToCommit.AddRange(engCommonsFiles);
}
await _gitClient.PushFilesAsync(filesToCommit, repoUri, pullRequestBaseBranch, "Updating version files");
}
public async Task AddDependencyToVersionProps(string filePath, DependencyDetail dependency)
{
XmlDocument versionProps = await ReadVersionPropsAsync(filePath, null);
string versionedName = dependency.Name.Replace(".", string.Empty);
versionedName = $"{versionedName}Version";
XmlNode versionNode = versionProps.DocumentElement.SelectNodes($"//PropertyGroup").Item(0);
XmlNode newDependency = versionProps.CreateElement(versionedName);
newDependency.InnerText = dependency.Version;
versionNode.AppendChild(newDependency);
GitFile file = new GitFile(filePath, versionProps);
File.WriteAllText(file.FilePath, file.Content);
_logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.props");
}
public async Task CommitUpdatesAsync(
string repoUri,
string branch,
List <DependencyDetail> itemsToUpdate,
string message)
{
CheckForValidGitClient();
GitFileContentContainer fileContainer =
await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch);
List <GitFile> filesToCommit = fileContainer.GetFilesToCommit();
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = itemsToUpdate.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null && repoUri != arcadeItem.RepoUri)
{
// Files in arcade repository
List <GitFile> engCommonFiles = await GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToCommit.AddRange(engCommonFiles);
// Files in the target repo
string latestCommit = await _gitClient.GetLastCommitShaAsync(_gitClient.GetOwnerAndRepoFromRepoUri(repoUri), branch);
List <GitFile> targetEngCommonFiles = await GetCommonScriptFilesAsync(repoUri, latestCommit);
foreach (GitFile file in targetEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
file.Operation = GitFileOperation.Delete;
filesToCommit.Add(file);
}
}
}
await _gitClient.PushFilesAsync(filesToCommit, repoUri, branch, message);
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemote remote)
{
// TODO: This should use known updaters, but today the updaters for global.json can only
// add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095
/*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote);
* foreach (DependencyDetail dependency in dependencies)
* {
* if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function))
* {
* await (Task)function.DynamicInvoke(_fileManager, _repo, dependency);
* }
* else
* {
* defaultUpdates.Add(dependency);
* }
* }*/
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
// TODO: This needs to be moved into some consistent handling between local/remote and add/update:
// https://github.com/dotnet/arcade/issues/1095
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null)
{
List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
}
// Push on local does not commit.
await _gitClient.PushFilesAsync(filesToUpdate, _repo, null, null);
}
public async Task AddDependencyToVersionDetails(string filePath, DependencyDetail dependency, DependencyType dependencyType)
{
XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(filePath, null);
XmlNode newDependency = versionDetails.CreateElement("Dependency");
XmlAttribute nameAttribute = versionDetails.CreateAttribute("Name");
nameAttribute.Value = dependency.Name;
newDependency.Attributes.Append(nameAttribute);
XmlAttribute versionAttribute = versionDetails.CreateAttribute("Version");
versionAttribute.Value = dependency.Version;
newDependency.Attributes.Append(versionAttribute);
XmlNode uri = versionDetails.CreateElement("Uri");
uri.InnerText = dependency.RepoUri;
newDependency.AppendChild(uri);
XmlNode sha = versionDetails.CreateElement("Sha");
sha.InnerText = dependency.Commit;
newDependency.AppendChild(sha);
XmlNode dependencies = versionDetails.SelectSingleNode($"//{dependencyType}Dependencies");
dependencies.AppendChild(newDependency);
GitFile file = new GitFile(filePath, versionDetails);
File.WriteAllText(file.FilePath, file.Content);
_logger.LogInformation($"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to Version.Details.xml");
}
public void AddChild(DependencyGraphNode newChild, DependencyDetail dependency)
{
Children.Add(newChild);
newChild.Parents.Add(this);
}
/// <summary>
/// Adds a dependency to the dependency files
/// </summary>
/// <returns></returns>
public async Task AddDependencyAsync(DependencyDetail dependency, DependencyType dependencyType)
{
// TODO: https://github.com/dotnet/arcade/issues/1095
// This should be getting back a container and writing the files from here.
await _fileManager.AddDependencyAsync(dependency, dependencyType, _repo, null);
}
private static string GetRepoPath(
DependencyDetail dependency,
IEnumerable <string> remotesMap,
string reposFolder,
ILogger logger)
{
string repoPath = null;
if (remotesMap != null)
{
if (_remotesMapping == null)
{
_remotesMapping = CreateRemotesMapping(remotesMap);
}
if (!_remotesMapping.ContainsKey(repoPath))
{
throw new DarcException($"A key matching '{dependency.RepoUri}' was not " +
$"found in the mapping. Please make sure to include it...");
}
repoPath = _remotesMapping[repoPath];
}
else
{
string folder = null;
if (!string.IsNullOrEmpty(reposFolder))
{
folder = reposFolder;
}
else
{
// If a repo folder or a mapping was not set we use the current parent's
// parent folder.
string gitDir = LocalHelpers.GetGitDir(logger);
string parent = Directory.GetParent(gitDir).FullName;
folder = Directory.GetParent(parent).FullName;
}
// There are cases when the sha is not specified in Version.Details.xml
// since owners want that Maestro++ fills this in. Without a sha we
// cannot walk the graph. We do not fail the process but display/return
// a dependency with no sha and for that graph path that would be the end of the walk
if (string.IsNullOrEmpty(dependency.Commit))
{
return(null);
}
repoPath = LocalHelpers.GetRepoPathFromFolder(folder, dependency.Commit, logger);
if (string.IsNullOrEmpty(repoPath))
{
throw new DarcException($"Commit '{dependency.Commit}' was not found in any " +
$"folder in '{folder}'. Make sure a folder for '{dependency.RepoUri}' exists " +
$"and it has all the latest changes...");
}
}
return(repoPath);
}
/// <summary>
/// Add a dependency to Versions.props. This has the form:
/// <!-- Package names -->
/// <PropertyGroup>
/// <MicrosoftDotNetApiCompatPackage>Microsoft.DotNet.ApiCompat</MicrosoftDotNetApiCompatPackage>
/// </PropertyGroup>
///
/// <!-- Package versions -->
/// <PropertyGroup>
/// <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.18478.5</MicrosoftDotNetApiCompatPackageVersion>
/// </PropertyGroup>
///
/// See https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md for more
/// information.
/// </summary>
/// <param name="repo">Path to Versions.props file</param>
/// <param name="dependency">Dependency information to add.</param>
/// <returns>Async task.</returns>
public async Task AddDependencyToVersionsPropsAsync(string repo, string branch, DependencyDetail dependency)
{
XmlDocument versionProps = await ReadVersionPropsAsync(repo, null);
string documentNamespaceUri = versionProps.DocumentElement.NamespaceURI;
string packageNameElementName = VersionFiles.GetVersionPropsPackageElementName(dependency.Name);
string packageVersionElementName = VersionFiles.GetVersionPropsPackageVersionElementName(dependency.Name);
string packageVersionAlternateElementName = VersionFiles.GetVersionPropsAlternatePackageVersionElementName(
dependency.Name);
// Select elements by local name, since the Project (DocumentElement) element often has a default
// xmlns set.
XmlNodeList propertyGroupNodes = versionProps.DocumentElement.SelectNodes($"//*[local-name()='PropertyGroup']");
XmlNode newPackageNameElement = versionProps.CreateElement(packageNameElementName, documentNamespaceUri);
newPackageNameElement.InnerText = dependency.Name;
bool addedPackageVersionElement = false;
bool addedPackageNameElement = false;
// There can be more than one property group. Find the appropriate one containing an existing element of
// the same type, and add it to the parent.
foreach (XmlNode propertyGroupNode in propertyGroupNodes)
{
if (propertyGroupNode.HasChildNodes)
{
foreach (XmlNode propertyNode in propertyGroupNode.ChildNodes)
{
if (!addedPackageVersionElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsVersionElementSuffix))
{
XmlNode newPackageVersionElement = versionProps.CreateElement(
packageVersionElementName,
documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
propertyGroupNode.AppendChild(newPackageVersionElement);
addedPackageVersionElement = true;
break;
}
// Test for alternate suffixes. This will eventually be replaced by repo-level configuration:
// https://github.com/dotnet/arcade/issues/1095
else if (!addedPackageVersionElement && propertyNode.Name.EndsWith(
VersionFiles.VersionPropsAlternateVersionElementSuffix))
{
XmlNode newPackageVersionElement = versionProps.CreateElement(
packageVersionAlternateElementName,
documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
propertyGroupNode.AppendChild(newPackageVersionElement);
addedPackageVersionElement = true;
break;
}
else if (!addedPackageNameElement && propertyNode.Name.EndsWith(VersionFiles.VersionPropsPackageElementSuffix))
{
propertyGroupNode.AppendChild(newPackageNameElement);
addedPackageNameElement = true;
break;
}
}
if (addedPackageVersionElement && addedPackageNameElement)
{
break;
}
}
}
// Add the property groups if none were present
if (!addedPackageVersionElement)
{
// If the repository doesn't have any package version element, then
// use the non-alternate element name.
XmlNode newPackageVersionElement = versionProps.CreateElement(packageVersionElementName, documentNamespaceUri);
newPackageVersionElement.InnerText = dependency.Version;
XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package versions");
versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
versionProps.DocumentElement.AppendChild(propertyGroupElement);
propertyGroupElement.AppendChild(newPackageVersionElement);
}
if (!addedPackageNameElement)
{
XmlNode propertyGroupElement = versionProps.CreateElement("PropertyGroup", documentNamespaceUri);
XmlNode propertyGroupCommentElement = versionProps.CreateComment("Package names");
versionProps.DocumentElement.AppendChild(propertyGroupCommentElement);
versionProps.DocumentElement.AppendChild(propertyGroupElement);
propertyGroupElement.AppendChild(newPackageNameElement);
}
// TODO: This should not be done here. This should return some kind of generic file container to the caller,
// who will gather up all updates and then call the git client to write the files all at once:
// https://github.com/dotnet/arcade/issues/1095. Today this is only called from the Local interface so
// it's okay for now.
var file = new GitFile(VersionFiles.VersionProps, versionProps);
await _gitClient.PushFilesAsync(new List <GitFile> {
file
}, repo, branch, $"Add {dependency} to " +
$"'{VersionFiles.VersionProps}'");
_logger.LogInformation(
$"Dependency '{dependency.Name}' with version '{dependency.Version}' successfully added to " +
$"'{VersionFiles.VersionProps}'");
}
public DependencyGraphNode(DependencyDetail dependencyDetail)
: this(
dependencyDetail,
new HashSet <string>())
{
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory)
{
// TODO: This should use known updaters, but today the updaters for global.json can only
// add, not actually update. This needs a fix. https://github.com/dotnet/arcade/issues/1095
/*List<DependencyDetail> defaultUpdates = new List<DependencyDetail>(, IRemote remote);
* foreach (DependencyDetail dependency in dependencies)
* {
* if (DependencyOperations.TryGetKnownUpdater(dependency.Name, out Delegate function))
* {
* await (Task)function.DynamicInvoke(_fileManager, _repo, dependency);
* }
* else
* {
* defaultUpdates.Add(dependency);
* }
* }*/
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
// TODO: This needs to be moved into some consistent handling between local/remote and add/update:
// https://github.com/dotnet/arcade/issues/1095
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null)
{
try
{
IRemote remote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger);
List <GitFile> engCommonFiles = await remote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common");
foreach (GitFile file in localEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
file.Operation = GitFileOperation.Delete;
filesToUpdate.Add(file);
}
}
}
catch (Exception exc) when
(exc.Message == "Not Found")
{
_logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " +
"where a packages folder was passed and the commit which generated them is not " +
"yet pushed.");
}
}
// Push on local does not commit.
await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null);
}
public static async Task UpdateGlobalJson(GitFileManager fileManager, string repository, DependencyDetail dependency)
{
Dictionary <string, string> dependencyMapping = new Dictionary <string, string>
{
{ "Microsoft.DotNet.Arcade.Sdk", "msbuild-sdks" },
{ "dotnet", "tools" }
};
if (!dependencyMapping.ContainsKey(dependency.Name))
{
throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined.");
}
string parent = dependencyMapping[dependency.Name];
await fileManager.AddDependencyToGlobalJson(Path.Combine(repository, VersionFilePath.GlobalJson), parent, dependency.Name, dependency.Version);
await fileManager.AddDependencyToVersionDetails(Path.Combine(repository, VersionFilePath.VersionDetailsXml), dependency, DependencyType.Toolset);
}
public bool Equals(Asset x, DependencyDetail y)
{
return(x.Name.Equals(y.Name, StringComparison.OrdinalIgnoreCase) &&
x.Version == y.Version);
}
/// <summary>
/// Updates existing dependencies in the dependency files
/// </summary>
/// <param name="dependencies">Dependencies that need updates.</param>
/// <param name="remote">Remote instance for gathering eng/common script updates.</param>
/// <returns></returns>
public async Task UpdateDependenciesAsync(List <DependencyDetail> dependencies, IRemoteFactory remoteFactory)
{
// Read the current dependency files and grab their locations so that nuget.config can be updated appropriately.
// Update the incoming dependencies with locations.
IEnumerable <DependencyDetail> oldDependencies = await GetDependenciesAsync();
IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(_logger);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(oldDependencies);
await barOnlyRemote.AddAssetLocationToDependenciesAsync(dependencies);
// If we are updating the arcade sdk we need to update the eng/common files as well
DependencyDetail arcadeItem = dependencies.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
SemanticVersion targetDotNetVersion = null;
IRemote arcadeRemote = null;
if (arcadeItem != null)
{
arcadeRemote = await remoteFactory.GetRemoteAsync(arcadeItem.RepoUri, _logger);
targetDotNetVersion = await arcadeRemote.GetToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit);
}
var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, _repo, null, oldDependencies, targetDotNetVersion);
List <GitFile> filesToUpdate = fileContainer.GetFilesToCommit();
if (arcadeItem != null)
{
try
{
List <GitFile> engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit);
filesToUpdate.AddRange(engCommonFiles);
List <GitFile> localEngCommonFiles = await _gitClient.GetFilesAtCommitAsync(null, null, "eng/common");
foreach (GitFile file in localEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
// This is a file in the repo's eng/common folder that isn't present in Arcade at the
// requested SHA so delete it during the update.
// GitFile instances do not have public setters since we insert/retrieve them from an
// In-memory cache during remote updates and we don't want anything to modify the cached,
// references, so add a copy with a Delete FileOperation.
filesToUpdate.Add(new GitFile(
file.FilePath,
file.Content,
file.ContentEncoding,
file.Mode,
GitFileOperation.Delete));
}
}
}
catch (Exception exc) when
(exc.Message == "Not Found")
{
_logger.LogWarning("Could not update 'eng/common'. Most likely this is a scenario " +
"where a packages folder was passed and the commit which generated them is not " +
"yet pushed.");
}
}
// Push on local does not commit.
await _gitClient.CommitFilesAsync(filesToUpdate, _repo, null, null);
}
private IEnumerable <DependencyDetail> GetDependencyDetails(XmlDocument document, string branch = null, bool includePinned = true)
{
List <DependencyDetail> dependencyDetails = new List <DependencyDetail>();
if (document != null)
{
BuildDependencies(document.DocumentElement.SelectNodes("//Dependency"));
void BuildDependencies(XmlNodeList dependencies)
{
if (dependencies.Count > 0)
{
foreach (XmlNode dependency in dependencies)
{
if (dependency.NodeType != XmlNodeType.Comment && dependency.NodeType != XmlNodeType.Whitespace)
{
DependencyType type;
switch (dependency.ParentNode.Name)
{
case "ProductDependencies":
type = DependencyType.Product;
break;
case "ToolsetDependencies":
type = DependencyType.Toolset;
break;
default:
throw new DarcException($"Unknown dependency type '{dependency.ParentNode.Name}'");
}
bool isPinned = false;
// If the 'Pinned' attribute does not exist or if it is set to false we just not update it
if (dependency.Attributes[VersionFiles.PinnedAttributeName] != null)
{
if (!bool.TryParse(dependency.Attributes[VersionFiles.PinnedAttributeName].Value, out isPinned))
{
throw new DarcException($"The '{VersionFiles.PinnedAttributeName}' attribute is set but the value " +
$"'{dependency.Attributes[VersionFiles.PinnedAttributeName].Value}' " +
$"is not a valid boolean...");
}
}
DependencyDetail dependencyDetail = new DependencyDetail
{
Name = dependency.Attributes[VersionFiles.NameAttributeName].Value,
RepoUri = dependency.SelectSingleNode(VersionFiles.UriElementName).InnerText,
Commit = dependency.SelectSingleNode(VersionFiles.ShaElementName)?.InnerText,
Version = dependency.Attributes[VersionFiles.VersionAttributeName].Value,
CoherentParentDependencyName = dependency.Attributes[VersionFiles.CoherentParentAttributeName]?.Value,
Pinned = isPinned,
Type = type
};
dependencyDetails.Add(dependencyDetail);
}
}
}
}
}
else
{
_logger.LogError($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " +
$"Look for exceptions above.");
}
if (includePinned)
{
return(dependencyDetails);
}
return(dependencyDetails.Where(d => !d.Pinned));
}
public async Task CommitUpdatesAsync(
string repoUri,
string branch,
List <DependencyDetail> itemsToUpdate,
string message)
{
CheckForValidGitClient();
GitFileContentContainer fileContainer =
await _fileManager.UpdateDependencyFiles(itemsToUpdate, repoUri, branch);
List <GitFile> filesToCommit = new List <GitFile>();
// If we are updating the arcade sdk we need to update the eng/common files and the dotnet versions as well
DependencyDetail arcadeItem = itemsToUpdate.FirstOrDefault(
i => string.Equals(i.Name, "Microsoft.DotNet.Arcade.Sdk", StringComparison.OrdinalIgnoreCase));
if (arcadeItem != null && repoUri != arcadeItem.RepoUri)
{
// Files in arcade repository. All Arcade items have a GitHub repo URI by default so we need to change the
// URI from we are getting the eng/common files. If in an AzDO context we change the URI to that of
// dotnet-arcade in dnceng
string arcadeRepoUri = arcadeItem.RepoUri;
if (Uri.TryCreate(repoUri, UriKind.Absolute, out Uri parsedUri))
{
if (parsedUri.Host == "dev.azure.com" || parsedUri.Host.EndsWith("visualstudio.com"))
{
arcadeRepoUri = "https://dev.azure.com/dnceng/internal/_git/dotnet-arcade";
}
}
SemanticVersion arcadeDotnetVersion = await GetToolsDotnetVersionAsync(arcadeRepoUri, arcadeItem.Commit);
if (arcadeDotnetVersion != null)
{
fileContainer.GlobalJson = UpdateDotnetVersionGlobalJson(arcadeDotnetVersion, fileContainer.GlobalJson);
}
List <GitFile> engCommonFiles = await GetCommonScriptFilesAsync(arcadeRepoUri, arcadeItem.Commit);
filesToCommit.AddRange(engCommonFiles);
// Files in the target repo
string latestCommit = await _gitClient.GetLastCommitShaAsync(repoUri, branch);
List <GitFile> targetEngCommonFiles = await GetCommonScriptFilesAsync(repoUri, latestCommit);
foreach (GitFile file in targetEngCommonFiles)
{
if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
{
file.Operation = GitFileOperation.Delete;
filesToCommit.Add(file);
}
}
}
filesToCommit.AddRange(fileContainer.GetFilesToCommit());
await _gitClient.CommitFilesAsync(filesToCommit, repoUri, branch, message);
}
/// <summary>
/// Determine what dependencies need to be updated given an input set of assets.
/// </summary>
/// <param name="sourceCommit">Commit the assets come from.</param>
/// <param name="assets">Assets as inputs for the update.</param>
/// <param name="dependencies">Current set of the dependencies.</param>
/// <param name="sourceRepoUri">Repository the assets came from.</param>
/// <returns>Map of existing dependency->updated dependency</returns>
public Task <List <DependencyUpdate> > GetRequiredNonCoherencyUpdatesAsync(
string sourceRepoUri,
string sourceCommit,
IEnumerable <AssetData> assets,
IEnumerable <DependencyDetail> dependencies)
{
Dictionary <DependencyDetail, DependencyDetail> toUpdate = new Dictionary <DependencyDetail, DependencyDetail>();
// Walk the assets, finding the dependencies that don't have coherency markers.
// those must be updated in a second pass.
foreach (AssetData asset in assets)
{
DependencyDetail matchingDependencyByName =
dependencies.FirstOrDefault(d => d.Name.Equals(asset.Name, StringComparison.OrdinalIgnoreCase) &&
string.IsNullOrEmpty(d.CoherentParentDependencyName));
if (matchingDependencyByName == null)
{
continue;
}
// If the dependency is pinned, don't touch it.
if (matchingDependencyByName.Pinned)
{
continue;
}
// Build might contain multiple assets of the same name
if (toUpdate.ContainsKey(matchingDependencyByName))
{
continue;
}
// Check if an update is actually needed.
// Case-sensitive compare as case-correction is desired.
if (matchingDependencyByName.Name == asset.Name &&
matchingDependencyByName.Version == asset.Version &&
matchingDependencyByName.Commit == sourceCommit &&
matchingDependencyByName.RepoUri == sourceRepoUri)
{
continue;
}
DependencyDetail newDependency = new DependencyDetail(matchingDependencyByName)
{
Commit = sourceCommit,
RepoUri = sourceRepoUri,
Version = asset.Version,
Name = asset.Name
};
toUpdate.Add(matchingDependencyByName, newDependency);
}
List <DependencyUpdate> dependencyUpdates = toUpdate.Select(kv => new DependencyUpdate
{
From = kv.Key,
To = kv.Value
}).ToList();
return(Task.FromResult(dependencyUpdates));
}
/// <summary>
/// Get updates required by coherency constraints.
/// </summary>
/// <param name="dependencies">Current set of dependencies.</param>
/// <param name="remoteFactory">Remote factory for remote queries.</param>
/// <returns>Dependencies with updates.</returns>
public async Task <List <DependencyUpdate> > GetRequiredCoherencyUpdatesAsync(
IEnumerable <DependencyDetail> dependencies,
IRemoteFactory remoteFactory)
{
List <DependencyUpdate> toUpdate = new List <DependencyUpdate>();
IEnumerable <DependencyDetail> leavesOfCoherencyTrees =
CalculateLeavesOfCoherencyTrees(dependencies);
if (!leavesOfCoherencyTrees.Any())
{
// Nothing to do.
return(toUpdate);
}
DependencyGraphBuildOptions dependencyGraphBuildOptions = new DependencyGraphBuildOptions()
{
IncludeToolset = true,
LookupBuilds = true,
NodeDiff = NodeDiff.None
};
// Now make a walk over coherent dependencies. Note that coherent dependencies could make
// a chain (A->B->C). In all cases we need to walk to the head of the chain, keeping track
// of all elements in the chain. Also note that we are walking all dependencies here, not
// just those that match the incoming AssetData and aligning all of these based on the coherency data.
Dictionary <string, DependencyGraphNode> nodeCache = new Dictionary <string, DependencyGraphNode>();
HashSet <DependencyDetail> visited = new HashSet <DependencyDetail>();
foreach (DependencyDetail dependency in leavesOfCoherencyTrees)
{
// If the dependency was already updated, then skip it (could have been part of a larger
// dependency chain)
if (visited.Contains(dependency))
{
continue;
}
// Walk to head of dependency tree, keeping track of elements along the way.
// If we hit a pinned dependency in the walk, that means we can't move
// the dependency and therefore it is effectively the "head" of the subtree.
// We will still visit all the elements in the chain eventually in this algorithm:
// Consider A->B(pinned)->C(pinned)->D.
List <DependencyDetail> updateList = new List <DependencyDetail>();
DependencyDetail currentDependency = dependency;
while (!string.IsNullOrEmpty(currentDependency.CoherentParentDependencyName) && !currentDependency.Pinned)
{
updateList.Add(currentDependency);
DependencyDetail parentCoherentDependency = dependencies.FirstOrDefault(d =>
d.Name.Equals(currentDependency.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase));
currentDependency = parentCoherentDependency ?? throw new DarcException($"Dependency {currentDependency.Name} has non-existent parent " +
$"dependency {currentDependency.CoherentParentDependencyName}");
}
DependencyGraphNode rootNode = null;
// Build the graph to find the assets if we don't have the root in the cache.
// The graph build is automatically broken when
// all the desired assets are found (breadth first search). This means the cache may or
// may not contain a complete graph for a given node. So, we first search the cache for the desired assets,
// then if not found (or if there was no cache), we then build the graph from that node.
bool nodeFromCache = nodeCache.TryGetValue($"{currentDependency.RepoUri}@{currentDependency.Commit}", out rootNode);
if (!nodeFromCache)
{
_logger.LogInformation($"Node not found in cache, starting graph build at " +
$"{currentDependency.RepoUri}@{currentDependency.Commit}");
rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, updateList, nodeCache);
}
List <DependencyDetail> leftToFind = new List <DependencyDetail>(updateList);
// Now do the lookup to find the element in the tree for each item in the update list
foreach (DependencyDetail dependencyInUpdateChain in updateList)
{
(Asset coherentAsset, Build buildForAsset) =
FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode);
// If we originally got the root node from the cache the graph may be incomplete.
// Rebuild to attempt to find all the assets we have left to find. If we still can't find, or if
// the root node did not come the cache, then we're in an invalid state.
if (coherentAsset == null && nodeFromCache)
{
_logger.LogInformation($"Asset {dependencyInUpdateChain.Name} was not found in cached graph, rebuilding from " +
$"{currentDependency.RepoUri}@{currentDependency.Commit}");
rootNode = await BuildGraphAtDependency(remoteFactory, currentDependency, leftToFind, nodeCache);
// And attempt to find again.
(coherentAsset, buildForAsset) =
FindAssetInBuildTree(dependencyInUpdateChain.Name, rootNode);
}
if (coherentAsset == null)
{
// This is an invalid state. We can't satisfy the
// constraints so they should either be removed or pinned.
throw new DarcException($"Unable to update {dependencyInUpdateChain.Name} to have coherency with " +
$"parent {dependencyInUpdateChain.CoherentParentDependencyName}. No matching asset found in tree. " +
$"Either remove the coherency attribute or mark as pinned.");
}
else
{
leftToFind.Remove(dependencyInUpdateChain);
}
string buildRepoUri = buildForAsset.GitHubRepository ?? buildForAsset.AzureDevOpsRepository;
if (dependencyInUpdateChain.Name == coherentAsset.Name &&
dependencyInUpdateChain.Version == coherentAsset.Version &&
dependencyInUpdateChain.Commit == buildForAsset.Commit &&
dependencyInUpdateChain.RepoUri == buildRepoUri)
{
continue;
}
DependencyDetail updatedDependency = new DependencyDetail(dependencyInUpdateChain)
{
Name = coherentAsset.Name,
Version = coherentAsset.Version,
RepoUri = buildRepoUri,
Commit = buildForAsset.Commit
};
toUpdate.Add(new DependencyUpdate
{
From = dependencyInUpdateChain,
To = updatedDependency
});
visited.Add(dependencyInUpdateChain);
}
}
return(toUpdate);
}
