private static async Task DoLatestInChannelGraphNodeDiffAsync( IRemoteFactory remoteFactory, ILogger logger, Dictionary <string, DependencyGraphNode> nodeCache, Dictionary <string, DependencyGraphNode> visitedRepoUriNodes) { logger.LogInformation("Running latest in channel node diff."); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(logger); // Walk each node in the graph and diff against the latest build in the channel // that was also applied to the node. Dictionary <string, string> latestCommitCache = new Dictionary <string, string>(); foreach (DependencyGraphNode node in nodeCache.Values) { // Start with an unknown diff. node.DiffFrom = GitDiff.UnknownDiff(); if (node.ContributingBuilds.Any()) { // Choose latest build of node that has a channel. Build newestBuildWithChannel = node.ContributingBuilds.OrderByDescending(b => b.DateProduced).FirstOrDefault( b => b.Channels != null && b.Channels.Any()); // If no build was found (e.g. build was flowed without a channel or channel was removed from // a build, then no diff from latest. if (newestBuildWithChannel != null) { int channelId = newestBuildWithChannel.Channels.First().Id; // Just choose the first channel. This algorithm is mostly just heuristic. string latestCommitKey = $"{node.Repository}@{channelId}"; string latestCommit = null; if (!latestCommitCache.TryGetValue(latestCommitKey, out latestCommit)) { // Look up latest build in the channel var latestBuild = await barOnlyRemote.GetLatestBuildAsync(node.Repository, channelId); // Could be null, if the only build was removed from the channel if (latestBuild != null) { latestCommit = latestBuild.Commit; } // Add to cache latestCommitCache.Add(latestCommitKey, latestCommit); } // Perform diff if there is a latest commit. if (!string.IsNullOrEmpty(latestCommit)) { IRemote repoRemote = await remoteFactory.GetRemoteAsync(node.Repository, logger); // This will return a no-diff if latestCommit == node.Commit node.DiffFrom = await repoRemote.GitDiffAsync(node.Repository, latestCommit, node.Commit); } } } } }
/// <summary> /// Update local dependencies based on a specific channel. /// </summary> /// <param name="options">Command line options</param> /// <returns>Process exit code.</returns> public override async Task <int> ExecuteAsync() { try { DarcSettings darcSettings = darcSettings = LocalSettings.GetDarcSettings(_options, Logger); // TODO: PAT only used for pulling the Arcade eng/common dir, // so hardcoded to GitHub PAT right now. Must be more generic in the future. darcSettings.GitType = GitRepoType.GitHub; LocalSettings localSettings = LocalSettings.LoadSettingsFile(_options); darcSettings.GitRepoPersonalAccessToken = localSettings != null && !string.IsNullOrEmpty(localSettings.GitHubToken) ? localSettings.GitHubToken : _options.GitHubPat; IRemoteFactory remoteFactory = new RemoteFactory(_options); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger); Local local = new Local(Logger); List <DependencyDetail> dependenciesToUpdate = new List <DependencyDetail>(); bool someUpToDate = false; string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'."; // First we need to figure out what to query for. Load Version.Details.xml and // find all repository uris, optionally restricted by the input dependency parameter. IEnumerable <DependencyDetail> localDependencies = await local.GetDependenciesAsync(_options.Name, false); // If the source repository was specified, filter away any local dependencies not from that // source repository. if (!string.IsNullOrEmpty(_options.SourceRepository)) { localDependencies = localDependencies.Where( dependency => dependency.RepoUri.Contains(_options.SourceRepository, StringComparison.OrdinalIgnoreCase)); } if (!localDependencies.Any()) { Console.WriteLine("Found no dependencies to update."); return(Constants.ErrorCode); } List <DependencyDetail> currentDependencies = localDependencies.ToList(); if (!string.IsNullOrEmpty(_options.Name) && !string.IsNullOrEmpty(_options.Version)) { DependencyDetail dependency = currentDependencies.First(); dependency.Version = _options.Version; dependenciesToUpdate.Add(dependency); Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{_options.Version}'"); finalMessage = $"Local dependency {_options.Name} updated to version '{_options.Version}'."; } else if (!string.IsNullOrEmpty(_options.PackagesFolder)) { try { dependenciesToUpdate.AddRange(GetDependenciesFromPackagesFolder(_options.PackagesFolder, currentDependencies)); } catch (DarcException exc) { Logger.LogError(exc, $"Error: Failed to update dependencies based on folder '{_options.PackagesFolder}'"); return(Constants.ErrorCode); } finalMessage = $"Local dependencies updated based on packages folder {_options.PackagesFolder}."; } else if (_options.BARBuildId > 0) { try { if (!_options.CoherencyOnly) { Console.WriteLine($"Looking up build with BAR id {_options.BARBuildId}"); var specificBuild = await barOnlyRemote.GetBuildAsync(_options.BARBuildId); int nonCoherencyResult = await NonCoherencyUpdatesForBuildAsync(specificBuild, barOnlyRemote, currentDependencies, dependenciesToUpdate) .ConfigureAwait(false); if (nonCoherencyResult != Constants.SuccessCode) { Console.WriteLine("Error: Failed to update non-coherent parent tied dependencies."); return(nonCoherencyResult); } string sourceRepo = specificBuild.GitHubRepository ?? specificBuild.AzureDevOpsRepository; string sourceBranch = specificBuild.GitHubBranch ?? specificBuild.AzureDevOpsBranch; finalMessage = $"Local dependencies updated based on build with BAR id {_options.BARBuildId} " + $"({specificBuild.AzureDevOpsBuildNumber} from {sourceRepo}@{sourceBranch})"; } int coherencyResult = await CoherencyUpdatesAsync(barOnlyRemote, remoteFactory, currentDependencies, dependenciesToUpdate) .ConfigureAwait(false); if (coherencyResult != Constants.SuccessCode) { Console.WriteLine("Error: Failed to update coherent parent tied dependencies."); return(coherencyResult); } finalMessage = string.IsNullOrEmpty(finalMessage) ? "Local dependencies successfully updated." : finalMessage; } catch (RestApiException e) when(e.Response.Status == 404) { Console.WriteLine($"Could not find build with BAR id '{_options.BARBuildId}'."); return(Constants.ErrorCode); } } else { if (!_options.CoherencyOnly) { if (string.IsNullOrEmpty(_options.Channel)) { Console.WriteLine($"Please supply either a channel name (--channel), a packages folder (--packages-folder) " + "a BAR build id (--id), or a specific dependency name and version (--name and --version)."); return(Constants.ErrorCode); } // Start channel query. Task <Channel> channel = barOnlyRemote.GetChannelAsync(_options.Channel); // Limit the number of BAR queries by grabbing the repo URIs and making a hash set. // We gather the latest build for any dependencies that aren't marked with coherent parent // dependencies, as those will be updated based on additional queries. HashSet <string> repositoryUrisForQuery = currentDependencies .Where(dependency => string.IsNullOrEmpty(dependency.CoherentParentDependencyName)) .Select(dependency => dependency.RepoUri) .ToHashSet(); ConcurrentDictionary <string, Task <Build> > getLatestBuildTaskDictionary = new ConcurrentDictionary <string, Task <Build> >(); Channel channelInfo = await channel; if (channelInfo == null) { Console.WriteLine($"Could not find a channel named '{_options.Channel}'."); return(Constants.ErrorCode); } foreach (string repoToQuery in repositoryUrisForQuery) { Console.WriteLine($"Looking up latest build of {repoToQuery} on {_options.Channel}"); var latestBuild = barOnlyRemote.GetLatestBuildAsync(repoToQuery, channelInfo.Id); getLatestBuildTaskDictionary.TryAdd(repoToQuery, latestBuild); } // For each build, first go through and determine the required updates, // updating the "live" dependency information as we go. // Then run a second pass where we update any assets based on coherency information. foreach (KeyValuePair <string, Task <Build> > buildKvPair in getLatestBuildTaskDictionary) { string repoUri = buildKvPair.Key; Build build = await buildKvPair.Value; if (build == null) { Logger.LogTrace($"No build of '{repoUri}' found on channel '{_options.Channel}'."); continue; } int nonCoherencyResult = await NonCoherencyUpdatesForBuildAsync(build, barOnlyRemote, currentDependencies, dependenciesToUpdate) .ConfigureAwait(false); if (nonCoherencyResult != Constants.SuccessCode) { Console.WriteLine("Error: Failed to update non-coherent parent tied dependencies."); return(nonCoherencyResult); } } } int coherencyResult = await CoherencyUpdatesAsync(barOnlyRemote, remoteFactory, currentDependencies, dependenciesToUpdate) .ConfigureAwait(false); if (coherencyResult != Constants.SuccessCode) { Console.WriteLine("Error: Failed to update coherent parent tied dependencies."); return(coherencyResult); } } if (!dependenciesToUpdate.Any()) { // If we found some dependencies already up to date, // then we consider this a success. Otherwise, we didn't even // find matching dependencies so we should let the user know. if (someUpToDate) { Console.WriteLine($"All dependencies are up to date."); return(Constants.SuccessCode); } else { Console.WriteLine($"Found no dependencies to update."); return(Constants.ErrorCode); } } if (_options.DryRun) { return(Constants.SuccessCode); } // Now call the local updater to run the update. await local.UpdateDependenciesAsync(dependenciesToUpdate, remoteFactory); Console.WriteLine(finalMessage); return(Constants.SuccessCode); } catch (AuthenticationException e) { Console.WriteLine(e.Message); return(Constants.ErrorCode); } catch (Exception e) { Logger.LogError(e, "Error: Failed to update dependencies."); return(Constants.ErrorCode); } }
/// <summary> /// Obtain the root build. /// </summary> /// <returns>Root build to start with.</returns> private async Task <Build> GetRootBuildAsync() { if (!ValidateRootBuildOptions()) { return(null); } IRemote remote = RemoteFactory.GetBarOnlyRemote(_options, Logger); string repoUri = GetRepoUri(); if (_options.RootBuildId != 0) { Console.WriteLine($"Looking up build by id {_options.RootBuildId}"); Build rootBuild = await remote.GetBuildAsync(_options.RootBuildId); if (rootBuild == null) { Console.WriteLine($"No build found with id {_options.RootBuildId}"); return(null); } return(rootBuild); } else if (!string.IsNullOrEmpty(repoUri)) { if (!string.IsNullOrEmpty(_options.Channel)) { IEnumerable <Channel> channels = await remote.GetChannelsAsync(); IEnumerable <Channel> desiredChannels = channels.Where(channel => channel.Name.Contains(_options.Channel, StringComparison.OrdinalIgnoreCase)); if (desiredChannels.Count() != 1) { Console.WriteLine($"Channel name {_options.Channel} did not match a unique channel. Available channels:"); foreach (var channel in channels) { Console.WriteLine($" {channel.Name}"); } return(null); } Channel targetChannel = desiredChannels.First(); Console.WriteLine($"Looking up latest build of '{repoUri}' on channel '{targetChannel.Name}'"); Build rootBuild = await remote.GetLatestBuildAsync(repoUri, targetChannel.Id); if (rootBuild == null) { Console.WriteLine($"No build of '{repoUri}' found on channel '{targetChannel.Name}'"); return(null); } return(rootBuild); } else if (!string.IsNullOrEmpty(_options.Commit)) { Console.WriteLine($"Looking up builds of {_options.RepoUri}@{_options.Commit}"); IEnumerable <Build> builds = await remote.GetBuildsAsync(_options.RepoUri, _options.Commit); // If more than one is available, print them with their IDs. if (builds.Count() > 1) { Console.WriteLine($"There were {builds.Count()} potential root builds. Please select one and pass it with --id"); foreach (var build in builds) { Console.WriteLine($" {build.Id}: {build.AzureDevOpsBuildNumber} @ {build.DateProduced.ToLocalTime()}"); } return(null); } Build rootBuild = builds.SingleOrDefault(); if (rootBuild == null) { Console.WriteLine($"No builds were found of {_options.RepoUri}@{_options.Commit}"); } return(rootBuild); } } // Shouldn't get here if ValidateRootBuildOptions is correct. throw new DarcException("Options for root builds were not validated properly. Please contact @dnceng"); }
/// <summary> /// Gets the latest build for a repo /// </summary> /// <returns>Process exit code.</returns> public override async Task <int> ExecuteAsync() { try { IRemote remote = RemoteFactory.GetBarOnlyRemote(_options, Logger); // Calculate out possible repos based on the input strings. // Today the DB has no way of searching for builds by substring, so for now // grab source/targets repos of subscriptions matched on substring, // and then add the explicit repo from the options. // Then search channels by substring // Then run GetLatestBuild for each permutation. var subscriptions = await remote.GetSubscriptionsAsync(); var possibleRepos = subscriptions .SelectMany(subscription => new List <string> { subscription.SourceRepository, subscription.TargetRepository }) .Where(r => r.Contains(_options.Repo, StringComparison.OrdinalIgnoreCase)) .ToHashSet(StringComparer.OrdinalIgnoreCase); possibleRepos.Add(_options.Repo); var channels = (await remote.GetChannelsAsync()) .Where(c => string.IsNullOrEmpty(_options.Channel) || c.Name.Contains(_options.Channel, StringComparison.OrdinalIgnoreCase)); if (!channels.Any()) { Console.WriteLine($"Could not find a channel with name containing '{_options.Channel}'"); return(Constants.ErrorCode); } bool foundBuilds = false; foreach (string possibleRepo in possibleRepos) { foreach (Channel channel in channels) { Build latestBuild = await remote.GetLatestBuildAsync(possibleRepo, channel.Id); if (latestBuild != null) { if (foundBuilds) { Console.WriteLine(); } foundBuilds = true; Console.Write(UxHelpers.GetTextBuildDescription(latestBuild)); } } } if (!foundBuilds) { Console.WriteLine("No latest build found matching the specified criteria"); return(Constants.ErrorCode); } return(Constants.SuccessCode); } catch (AuthenticationException e) { Console.WriteLine(e.Message); return(Constants.ErrorCode); } catch (Exception e) { Logger.LogError(e, "Error: Failed to retrieve latest build."); return(Constants.ErrorCode); } }
/// <summary> /// Get the latest assets that were produced by each subscription /// and compute any conflicts between subscriptionss /// </summary> /// <returns>Mapping of assets to subscriptions that produce them.</returns> private async Task <Dictionary <string, Subscription> > GetLatestAssetsAndComputeConflicts(IRemote remote) { // Populate the latest build task for each of these. The search for assets would be N*M*A where N is the number of // dependencies, M is the number of subscriptions, and A is average the number of assets per build. // Because this could add up pretty quickly, we build up a dictionary of assets->List<(subscription, build)> // instead. Dictionary <string, Subscription> assetsToLatestInSubscription = new Dictionary <string, Subscription>(StringComparer.OrdinalIgnoreCase); Dictionary <string, SubscriptionConflict> subscriptionConflicts = new Dictionary <string, SubscriptionConflict>(); foreach (Subscription subscription in Subscriptions) { // Look up the latest build and add it to the dictionary. Build latestBuild = await remote.GetLatestBuildAsync(subscription.SourceRepository, subscription.Channel.Id); if (latestBuild != null) { foreach (var asset in latestBuild.Assets) { string assetName = asset.Name; if (assetsToLatestInSubscription.TryGetValue(assetName, out Subscription otherSubscription)) { // Repos can publish the same asset twice for the same build, so filter out those cases, // as well as cases where the subscription is functionally the same (e.g. you have a twice daily // and weekly subscription). Basically cases where the source repo and source channels are the same. if (otherSubscription.SourceRepository.Equals(subscription.SourceRepository, StringComparison.OrdinalIgnoreCase) && otherSubscription.Channel.Id == subscription.Channel.Id) { continue; } // While technically this asset would need to be utilized in the dependencies // to cause an issue, it's an issue waiting to happen, so stick this in the conflicting subscriptions. if (subscriptionConflicts.TryGetValue(assetName, out SubscriptionConflict conflict)) { conflict.Subscriptions.Add(subscription); } else { SubscriptionConflict newConflict = new SubscriptionConflict(assetName, new List <Subscription>() { otherSubscription, subscription }, Dependencies.Any(d => d.Name.Equals(assetName, StringComparison.OrdinalIgnoreCase))); subscriptionConflicts.Add(assetName, newConflict); } } else { assetsToLatestInSubscription.Add(assetName, subscription); } } } } // Now there is a complete accounting of the conflicts. ConflictingSubscriptions = subscriptionConflicts.Values.ToList(); return(assetsToLatestInSubscription); }
/// <summary> /// Update local dependencies based on a specific channel. /// </summary> /// <param name="options">Command line options</param> /// <returns>Process exit code.</returns> public override async Task <int> ExecuteAsync() { try { DarcSettings darcSettings = darcSettings = LocalSettings.GetDarcSettings(_options, Logger); // TODO: PAT only used for pulling the arcade eng/common dir, // so hardcoded to GitHub PAT right now. Must be more generic in the future. darcSettings.GitType = GitRepoType.GitHub; LocalSettings localSettings = LocalSettings.LoadSettingsFile(_options); darcSettings.GitRepoPersonalAccessToken = localSettings != null && !string.IsNullOrEmpty(localSettings.GitHubToken) ? localSettings.GitHubToken : _options.GitHubPat; IRemoteFactory remoteFactory = new RemoteFactory(_options); IRemote barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger); Local local = new Local(Logger); List <DependencyDetail> dependenciesToUpdate = new List <DependencyDetail>(); bool someUpToDate = false; string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'."; // First we need to figure out what to query for. Load Version.Details.xml and // find all repository uris, optionally restricted by the input dependency parameter. IEnumerable <DependencyDetail> localDependencies = await local.GetDependenciesAsync(_options.Name, false); // If the source repository was specified, filter away any local dependencies not from that // source repository. if (!string.IsNullOrEmpty(_options.SourceRepository)) { localDependencies = localDependencies.Where( dependency => dependency.RepoUri.Contains(_options.SourceRepository, StringComparison.OrdinalIgnoreCase)); } if (!localDependencies.Any()) { Console.WriteLine("Found no dependencies to update."); return(Constants.ErrorCode); } List <DependencyDetail> currentDependencies = localDependencies.ToList(); if (!string.IsNullOrEmpty(_options.Name) && !string.IsNullOrEmpty(_options.Version)) { DependencyDetail dependency = currentDependencies.First(); dependency.Version = _options.Version; dependenciesToUpdate.Add(dependency); Console.WriteLine($"Updating '{dependency.Name}': '{dependency.Version}' => '{_options.Version}'"); finalMessage = $"Local dependency {_options.Name} updated to version '{_options.Version}'."; } else if (!string.IsNullOrEmpty(_options.PackagesFolder)) { try { dependenciesToUpdate.AddRange(GetDependenciesFromPackagesFolder(_options.PackagesFolder, currentDependencies)); } catch (DarcException exc) { Logger.LogError(exc, $"Error: Failed to update dependencies based on folder '{_options.PackagesFolder}'"); return(Constants.ErrorCode); } finalMessage = $"Local dependencies updated based on packages folder {_options.PackagesFolder}."; } else { if (!_options.CoherencyOnly) { if (string.IsNullOrEmpty(_options.Channel)) { Console.WriteLine($"Please supply either a channel name (--channel), a packages folder (--packages-folder) " + $"or a specific dependency name and version (--name and --version)."); return(Constants.ErrorCode); } // Start channel query. Task <Channel> channel = barOnlyRemote.GetChannelAsync(_options.Channel); // Limit the number of BAR queries by grabbing the repo URIs and making a hash set. // We gather the latest build for any dependencies that aren't marked with coherent parent // dependencies, as those will be updated based on additional queries. HashSet <string> repositoryUrisForQuery = currentDependencies .Where(dependency => string.IsNullOrEmpty(dependency.CoherentParentDependencyName)) .Select(dependency => dependency.RepoUri) .ToHashSet(); ConcurrentDictionary <string, Task <Build> > getLatestBuildTaskDictionary = new ConcurrentDictionary <string, Task <Build> >(); Channel channelInfo = await channel; if (channelInfo == null) { Console.WriteLine($"Could not find a channel named '{_options.Channel}'."); return(Constants.ErrorCode); } foreach (string repoToQuery in repositoryUrisForQuery) { Console.WriteLine($"Looking up latest build of {repoToQuery} on {_options.Channel}"); var latestBuild = barOnlyRemote.GetLatestBuildAsync(repoToQuery, channelInfo.Id); getLatestBuildTaskDictionary.TryAdd(repoToQuery, latestBuild); } // For each build, first go through and determine the required updates, // updating the "live" dependency information as we go. // Then run a second pass where we update any assets based on coherency information. foreach (KeyValuePair <string, Task <Build> > buildKvPair in getLatestBuildTaskDictionary) { string repoUri = buildKvPair.Key; Build build = await buildKvPair.Value; if (build == null) { Logger.LogTrace($"No build of '{repoUri}' found on channel '{_options.Channel}'."); continue; } IEnumerable <AssetData> assetData = build.Assets.Select( a => new AssetData(a.NonShipping) { Name = a.Name, Version = a.Version }); // Now determine what needs to be updated. List <DependencyUpdate> updates = await barOnlyRemote.GetRequiredNonCoherencyUpdatesAsync( repoUri, build.Commit, assetData, currentDependencies); foreach (DependencyUpdate update in updates) { DependencyDetail from = update.From; DependencyDetail to = update.To; // Print out what we are going to do. Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}'" + $" (from build '{build.AzureDevOpsBuildNumber}' of '{repoUri}')"); // Final list of dependencies to update dependenciesToUpdate.Add(to); // Replace in the current dependencies list so the correct data is fed into the coherency pass. currentDependencies.Remove(from); currentDependencies.Add(to); } } } Console.WriteLine("Checking for coherency updates..."); // Now run a coherency update based on the current set of dependencies updated // from the previous pass. List <DependencyUpdate> coherencyUpdates = await barOnlyRemote.GetRequiredCoherencyUpdatesAsync(currentDependencies, remoteFactory); foreach (DependencyUpdate dependencyUpdate in coherencyUpdates) { DependencyDetail from = dependencyUpdate.From; DependencyDetail to = dependencyUpdate.To; DependencyDetail coherencyParent = currentDependencies.First(d => d.Name.Equals(from.CoherentParentDependencyName, StringComparison.OrdinalIgnoreCase)); // Print out what we are going to do. Console.WriteLine($"Updating '{from.Name}': '{from.Version}' => '{to.Version}' " + $"to ensure coherency with {from.CoherentParentDependencyName}@{coherencyParent.Version}"); // Final list of dependencies to update dependenciesToUpdate.Add(to); } } if (!dependenciesToUpdate.Any()) { // If we found some dependencies already up to date, // then we consider this a success. Otherwise, we didn't even // find matching dependencies so we should let the user know. if (someUpToDate) { Console.WriteLine($"All dependencies are up to date."); return(Constants.SuccessCode); } else { Console.WriteLine($"Found no dependencies to update."); return(Constants.ErrorCode); } } if (_options.DryRun) { return(Constants.SuccessCode); } // Now call the local updater to run the update. await local.UpdateDependenciesAsync(dependenciesToUpdate, remoteFactory); Console.WriteLine(finalMessage); return(Constants.SuccessCode); } catch (Exception e) { Logger.LogError(e, $"Error: Failed to update dependencies to channel {_options.Channel}"); return(Constants.ErrorCode); } }