public async Task UpdateLongestBuildPathAsync(CancellationToken cancellationToken)
{
using (Logger.BeginScope($"Updating Longest Build Path table"))
{
List <Channel> channels = Context.Channels.Select(c => new Channel()
{
Id = c.Id, Name = c.Name
}).ToList();
// Get the flow graph
IRemote barOnlyRemote = await RemoteFactory.GetBarOnlyRemoteAsync(Logger);
List <Microsoft.DotNet.Maestro.Client.Models.DefaultChannel> defaultChannels = (await barOnlyRemote.GetDefaultChannelsAsync()).ToList();
List <Microsoft.DotNet.Maestro.Client.Models.Subscription> subscriptions = (await barOnlyRemote.GetSubscriptionsAsync()).ToList();
IEnumerable <string> frequencies = new[] { "everyWeek", "twiceDaily", "everyDay", "everyBuild", "none", };
Logger.LogInformation($"Will update '{channels.Count}' channels");
foreach (var channel in channels)
{
// Build, then prune out what we don't want to see if the user specified channels.
DependencyFlowGraph flowGraph = await DependencyFlowGraph.BuildAsync(defaultChannels, subscriptions, barOnlyRemote, 30);
flowGraph.PruneGraph(
node => DependencyFlowGraph.IsInterestingNode(channel.Name, node),
edge => DependencyFlowGraph.IsInterestingEdge(edge, false, frequencies));
if (flowGraph.Nodes.Count > 0)
{
flowGraph.MarkBackEdges();
flowGraph.CalculateLongestBuildPaths();
flowGraph.MarkLongestBuildPath();
// Get the nodes on the longest path and order them by path time so that the
// contributing repos are in the right order
List <DependencyFlowNode> longestBuildPathNodes = flowGraph.Nodes
.Where(n => n.OnLongestBuildPath)
.OrderByDescending(n => n.BestCasePathTime)
.ToList();
LongestBuildPath lbp = new LongestBuildPath()
{
ChannelId = channel.Id,
BestCaseTimeInMinutes = longestBuildPathNodes.Max(n => n.BestCasePathTime),
WorstCaseTimeInMinutes = longestBuildPathNodes.Max(n => n.WorstCasePathTime),
ContributingRepositories = String.Join(';', longestBuildPathNodes.Select(n => $"{n.Repository}@{n.Branch}").ToArray()),
ReportDate = DateTimeOffset.UtcNow,
};
Logger.LogInformation($"Will update {channel.Name} to best case time {lbp.BestCaseTimeInMinutes} and worst case time {lbp.WorstCaseTimeInMinutes}");
await Context.LongestBuildPaths.AddAsync(lbp);
}
}
await Context.SaveChangesAsync();
}
}
public override async Task <int> ExecuteAsync()
{
try
{
RemoteFactory remoteFactory = new RemoteFactory(_options);
var barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger);
List <DefaultChannel> defaultChannels = (await barOnlyRemote.GetDefaultChannelsAsync()).ToList();
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "master",
Channel = await barOnlyRemote.GetChannelAsync(".NET Tools - Latest")
}
);
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "release/3.x",
Channel = await barOnlyRemote.GetChannelAsync(".NET 3 Tools")
}
);
List <Subscription> subscriptions = (await barOnlyRemote.GetSubscriptionsAsync()).ToList();
// Build, then prune out what we don't want to see if the user specified
// channels.
DependencyFlowGraph flowGraph = DependencyFlowGraph.Build(defaultChannels, subscriptions);
Channel targetChannel = null;
if (!string.IsNullOrEmpty(_options.Channel))
{
// Resolve the channel.
targetChannel = await UxHelpers.ResolveSingleChannel(barOnlyRemote, _options.Channel);
if (targetChannel == null)
{
return(Constants.ErrorCode);
}
}
if (targetChannel != null)
{
flowGraph.PruneGraph(node => IsInterestingNode(targetChannel, node), edge => IsInterestingEdge(edge));
}
await LogGraphViz(targetChannel, flowGraph);
return(Constants.SuccessCode);
}
catch (Exception exc)
{
Logger.LogError(exc, "Something failed while getting the dependency graph.");
return(Constants.ErrorCode);
}
}
public async Task ShouldNotAddLongestBuildPathRowWhenGraphIsEmpty()
{
var graph = new DependencyFlowGraph(
new List <DependencyFlowNode>(),
new List <DependencyFlowEdge>());
SetupRemote(
(ChannelId: 1, Graph: graph));
var updater = ActivatorUtilities.CreateInstance <DependencyUpdater>(Scope.ServiceProvider);
await updater.UpdateLongestBuildPathAsync(CancellationToken.None);
var longestBuildPaths = Context.LongestBuildPaths.ToList();
longestBuildPaths.Should().BeEmpty();
}
/// <summary>
/// Log the graph in graphviz (dot) format.
/// </summary>
/// <param name="graph">Graph to log</param>
/// <remarks>
/// Example of a graphviz graph description
///
/// digraph graphname {
/// a -> b -> c;
/// b -> d;
/// }
///
/// For more info see https://www.graphviz.org/
/// </remarks>
/// <returns>Async task</returns>
private async Task LogGraphViz(Channel targetChannel, DependencyFlowGraph graph)
{
StringBuilder subgraphClusterWriter = null;
bool writeToSubgraphCluster = targetChannel != null;
if (writeToSubgraphCluster)
{
subgraphClusterWriter = new StringBuilder();
subgraphClusterWriter.AppendLine($" subgraph cluster_{OutputHelpers.CalculateGraphVizNodeName(targetChannel.Name)} {{");
subgraphClusterWriter.AppendLine($" label = \"{targetChannel.Name}\"");
}
using (StreamWriter writer = OutputHelpers.GetOutputFileStreamOrConsole(_options.GraphVizOutputFile))
{
await writer.WriteLineAsync("digraph repositoryGraph {");
await writer.WriteLineAsync(" node [shape=record]");
foreach (DependencyFlowNode node in graph.Nodes)
{
StringBuilder nodeBuilder = new StringBuilder();
// First add the node name
nodeBuilder.Append($" {OutputHelpers.CalculateGraphVizNodeName(node)}");
// Then add the label. label looks like [label="<info here>"]
nodeBuilder.Append("[label=\"");
// Append friendly repo name
nodeBuilder.Append(OutputHelpers.GetSimpleRepoName(node.Repository));
nodeBuilder.Append(@"\n");
// Append branch name
nodeBuilder.Append(node.Branch);
// Append end of label and end of node.
nodeBuilder.Append("\"];");
// If highlighting a specific channel, Add those nodes to a subgraph cluster
// if they output to the subgraph cluster.
if (writeToSubgraphCluster && node.OutputChannels.Any(c => c == targetChannel.Name))
{
subgraphClusterWriter.AppendLine($" {OutputHelpers.CalculateGraphVizNodeName(node)}");
}
// Write it out.
await writer.WriteLineAsync(nodeBuilder.ToString());
}
// Now write all the edges
foreach (DependencyFlowEdge edge in graph.Edges)
{
string fromNode = OutputHelpers.CalculateGraphVizNodeName(edge.From);
string toNode = OutputHelpers.CalculateGraphVizNodeName(edge.To);
string label = $"{edge.Subscription.Channel.Name} ({edge.Subscription.Policy.UpdateFrequency})";
if (writeToSubgraphCluster && edge.Subscription.Channel.Name == targetChannel.Name)
{
subgraphClusterWriter.AppendLine($" {fromNode} -> {toNode} [{GetEdgeStyle(edge)}]");
}
else
{
await writer.WriteLineAsync($" {fromNode} -> {toNode} [{GetEdgeStyle(edge)}]");
}
}
if (writeToSubgraphCluster)
{
await writer.WriteLineAsync(subgraphClusterWriter.ToString());
await writer.WriteLineAsync(" }");
}
// Write a legend
await writer.WriteLineAsync(" subgraph cluster1 {");
await writer.WriteLineAsync(" rankdir=RL;");
await writer.WriteLineAsync(" label = \"Legend\"");
await writer.WriteLineAsync(" shape = rectangle;");
await writer.WriteLineAsync(" color = black;");
await writer.WriteLineAsync(" a[style = invis];");
await writer.WriteLineAsync(" b[style = invis];");
await writer.WriteLineAsync(" c[style = invis];");
await writer.WriteLineAsync(" d[style = invis];");
await writer.WriteLineAsync(" e[style = invis];");
await writer.WriteLineAsync(" f[style = invis];");
await writer.WriteLineAsync(" c->d[label = \"Updated Every Build\", style = bold];");
await writer.WriteLineAsync(" a->b[label = \"Updated Every Day\", style = dashed];");
await writer.WriteLineAsync(" e->f[label = \"Disabled/Updated On-demand\", style = dotted];");
await writer.WriteLineAsync(" }");
await writer.WriteLineAsync("}");
}
}
/// <summary>
/// Log the graph in graphviz (dot) format.
/// </summary>
/// <param name="graph">Graph to log</param>
/// <remarks>
/// Example of a graphviz graph description
///
/// digraph graphname {
/// a -> b -> c;
/// b -> d;
/// }
///
/// For more info see https://www.graphviz.org/
/// </remarks>
/// <returns>Async task</returns>
private async Task LogGraphVizAsync(Channel targetChannel, DependencyFlowGraph graph, bool includeBuildTimes)
{
StringBuilder subgraphClusterWriter = null;
bool writeToSubgraphCluster = targetChannel != null;
if (writeToSubgraphCluster)
{
subgraphClusterWriter = new StringBuilder();
subgraphClusterWriter.AppendLine($" subgraph cluster_{UxHelpers.CalculateGraphVizNodeName(targetChannel.Name)} {{");
subgraphClusterWriter.AppendLine($" label = \"{targetChannel.Name}\"");
}
using (StreamWriter writer = UxHelpers.GetOutputFileStreamOrConsole(_options.GraphVizOutputFile))
{
await writer.WriteLineAsync("digraph repositoryGraph {");
await writer.WriteLineAsync(" node [shape=record]");
foreach (DependencyFlowNode node in graph.Nodes)
{
StringBuilder nodeBuilder = new StringBuilder();
string style = node.OnLongestBuildPath ? "style=\"diagonals,bold\" color=red" : "";
// First add the node name
nodeBuilder.Append($" {UxHelpers.CalculateGraphVizNodeName(node)}");
// Then add the label. label looks like [label="<info here>"]
nodeBuilder.Append($"[{style}\nlabel=\"");
// Append friendly repo name
nodeBuilder.Append(UxHelpers.GetSimpleRepoName(node.Repository));
nodeBuilder.Append(@"\n");
// Append branch name
nodeBuilder.Append(node.Branch);
if (includeBuildTimes)
{
// Append best case
nodeBuilder.Append(@"\n");
nodeBuilder.Append($"Best Case: {Math.Round(node.BestCasePathTime, 2, MidpointRounding.AwayFromZero)} min");
nodeBuilder.Append(@"\n");
// Append worst case
nodeBuilder.Append($"Worst Case: {Math.Round(node.WorstCasePathTime, 2, MidpointRounding.AwayFromZero)} min");
nodeBuilder.Append(@"\n");
// Append build times
nodeBuilder.Append($"Official Build Time: {Math.Round(node.OfficialBuildTime, 2, MidpointRounding.AwayFromZero)} min");
nodeBuilder.Append(@"\n");
nodeBuilder.Append($"PR Build Time: {Math.Round(node.PrBuildTime, 2, MidpointRounding.AwayFromZero)} min");
}
// Append end of label and end of node.
nodeBuilder.Append("\"];");
// If highlighting a specific channel, Add those nodes to a subgraph cluster
// if they output to the subgraph cluster.
if (writeToSubgraphCluster && node.OutputChannels.Any(c => c == targetChannel.Name))
{
subgraphClusterWriter.AppendLine($" {UxHelpers.CalculateGraphVizNodeName(node)}");
}
// Write it out.
await writer.WriteLineAsync(nodeBuilder.ToString());
}
// Now write all the edges
foreach (DependencyFlowEdge edge in graph.Edges)
{
string fromNode = UxHelpers.CalculateGraphVizNodeName(edge.From);
string toNode = UxHelpers.CalculateGraphVizNodeName(edge.To);
string label = $"{edge.Subscription.Channel.Name} ({edge.Subscription.Policy.UpdateFrequency})";
if (writeToSubgraphCluster && edge.Subscription.Channel.Name == targetChannel.Name)
{
subgraphClusterWriter.AppendLine($" {fromNode} -> {toNode} [{GetEdgeStyle(edge)}]");
}
else
{
await writer.WriteLineAsync($" {fromNode} -> {toNode} [{GetEdgeStyle(edge)}]");
}
}
if (writeToSubgraphCluster)
{
await writer.WriteLineAsync(subgraphClusterWriter.ToString());
await writer.WriteLineAsync(" }");
}
// Write a legend
await writer.WriteLineAsync(" subgraph cluster1 {");
await writer.WriteLineAsync(" rankdir=RL;");
await writer.WriteLineAsync(" label = \"Legend\"");
await writer.WriteLineAsync(" shape = rectangle;");
await writer.WriteLineAsync(" color = black;");
await writer.WriteLineAsync(" a[style = invis];");
await writer.WriteLineAsync(" b[style = invis];");
await writer.WriteLineAsync(" c[style = invis];");
await writer.WriteLineAsync(" d[style = invis];");
await writer.WriteLineAsync(" e[style = invis];");
await writer.WriteLineAsync(" f[style = invis];");
await writer.WriteLineAsync(" g[style = \"diagonals,bold\" color=red];");
await writer.WriteLineAsync(" h[style = \"diagonals,bold\" color=red];");
await writer.WriteLineAsync(" c->d[label = \"Updated Every Build\", style = bold];");
await writer.WriteLineAsync(" a->b[label = \"Updated Every Day\", style = dashed];");
await writer.WriteLineAsync(" e->f[label = \"Disabled/Updated On-demand\", style = dotted];");
await writer.WriteLineAsync(" g->h[label = \"Longest Build Path\", color=\"red:invis:red\"];");
await writer.WriteLineAsync(" }");
await writer.WriteLineAsync(" subgraph cluster2{");
await writer.WriteLineAsync(" rankdir=BT;");
await writer.WriteLineAsync(" style=invis;");
await writer.WriteLineAsync(" note[shape=plaintext label=\"Best Case: Time through the graph assuming no dependency flow\nWorst Case: Time through the graph with dependency flow (PRs)\"];");
await writer.WriteLineAsync(" }");
await writer.WriteLineAsync(" d->note[lhead=cluster2, ltail=cluster1, style=invis];");
await writer.WriteLineAsync("}");
}
}
public async Task <IActionResult> GetFlowGraphAsync(
int channelId = 0,
bool includeDisabledSubscriptions = false,
#pragma warning disable API0001 // Versioned API methods should not expose non-versioned types.
IEnumerable <string> includedFrequencies = default,
#pragma warning restore API0001 // Versioned API methods should not expose non-versioned types.
bool includeBuildTimes = false,
int days = 7,
bool includeArcade = true)
{
var barOnlyRemote = await _remoteFactory.GetBarOnlyRemoteAsync(Logger);
Microsoft.DotNet.Maestro.Client.Models.Channel engLatestChannel = await barOnlyRemote.GetChannelAsync(EngLatestChannelId);
Microsoft.DotNet.Maestro.Client.Models.Channel eng3Channel = await barOnlyRemote.GetChannelAsync(Eng3ChannelId);
List <Microsoft.DotNet.Maestro.Client.Models.DefaultChannel> defaultChannels = (await barOnlyRemote.GetDefaultChannelsAsync()).ToList();
if (includeArcade)
{
if (engLatestChannel != null)
{
defaultChannels.Add(
new Microsoft.DotNet.Maestro.Client.Models.DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "master",
Channel = engLatestChannel
}
);
}
if (eng3Channel != null)
{
defaultChannels.Add(
new Microsoft.DotNet.Maestro.Client.Models.DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "release/3.x",
Channel = eng3Channel
}
);
}
}
List <Microsoft.DotNet.Maestro.Client.Models.Subscription> subscriptions = (await barOnlyRemote.GetSubscriptionsAsync()).ToList();
// Build, then prune out what we don't want to see if the user specified
// channels.
DependencyFlowGraph flowGraph = await DependencyFlowGraph.BuildAsync(defaultChannels, subscriptions, barOnlyRemote, days);
IEnumerable <string> frequencies = includedFrequencies == default || includedFrequencies.Count() == 0 ?
new string[] { "everyWeek", "twiceDaily", "everyDay", "everyBuild", "none", } :
includedFrequencies;
Microsoft.DotNet.Maestro.Client.Models.Channel targetChannel = null;
if (channelId != 0)
{
targetChannel = await barOnlyRemote.GetChannelAsync((int)channelId);
}
if (targetChannel != null)
{
flowGraph.PruneGraph(
node => DependencyFlowGraph.IsInterestingNode(targetChannel.Name, node),
edge => DependencyFlowGraph.IsInterestingEdge(edge, includeDisabledSubscriptions, frequencies));
}
if (includeBuildTimes)
{
flowGraph.MarkBackEdges();
flowGraph.CalculateLongestBuildPaths();
flowGraph.MarkLongestBuildPath();
}
// Convert flow graph to correct return type
return(Ok(FlowGraph.Create(flowGraph)));
}
public static FlowGraph Create(DependencyFlowGraph other)
{
return(new FlowGraph(other.Nodes.Select(n => FlowRef.Create(n)).ToList(), other.Edges.Select(e => FlowEdge.Create(e)).ToList()));
}
public async Task <DependencyFlowGraph> GetDependencyFlowGraphAsync(
int channelId,
int days,
bool includeArcade,
bool includeBuildTimes,
bool includeDisabledSubscriptions,
IReadOnlyList <string> includedFrequencies)
{
var engLatestChannel = await GetChannelAsync(EngLatestChannelId);
var eng3Channel = await GetChannelAsync(Eng3ChannelId);
var defaultChannels = (await GetDefaultChannelsAsync()).ToList();
if (includeArcade)
{
if (engLatestChannel != null)
{
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "master",
Channel = engLatestChannel
}
);
}
if (eng3Channel != null)
{
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "release/3.x",
Channel = eng3Channel
}
);
}
}
var subscriptions = (await GetSubscriptionsAsync()).ToList();
// Build, then prune out what we don't want to see if the user specified
// channels.
DependencyFlowGraph flowGraph = await DependencyFlowGraph.BuildAsync(
defaultChannels,
subscriptions,
this,
days);
IEnumerable <string> frequencies
= includedFrequencies == default || includedFrequencies.Count() == 0
? new string[] { "everyWeek", "twiceDaily", "everyDay", "everyBuild", "none", }
: includedFrequencies;
Channel targetChannel = null;
if (channelId != 0)
{
targetChannel = await GetChannelAsync(channelId);
}
if (targetChannel != null)
{
flowGraph.PruneGraph(
node => DependencyFlowGraph.IsInterestingNode(targetChannel.Name, node),
edge => DependencyFlowGraph.IsInterestingEdge(edge, includeDisabledSubscriptions, frequencies));
}
if (includeBuildTimes)
{
var edgesWithLastBuild = flowGraph.Edges
.Where(e => e.Subscription.LastAppliedBuild != null);
foreach (var edge in edgesWithLastBuild)
{
edge.IsToolingOnly = !_context.IsProductDependency(
edge.From.Repository,
edge.From.Branch,
edge.To.Repository,
edge.To.Branch);
}
flowGraph.MarkBackEdges();
flowGraph.CalculateLongestBuildPaths();
flowGraph.MarkLongestBuildPath();
}
return(flowGraph);
}
public override async Task <int> ExecuteAsync()
{
try
{
RemoteFactory remoteFactory = new RemoteFactory(_options);
var barOnlyRemote = await remoteFactory.GetBarOnlyRemoteAsync(Logger);
Channel engLatestChannel = await barOnlyRemote.GetChannelAsync(engLatestChannelId);
Channel eng3Channel = await barOnlyRemote.GetChannelAsync(eng3ChannelId);
List <DefaultChannel> defaultChannels = (await barOnlyRemote.GetDefaultChannelsAsync()).ToList();
if (engLatestChannel != null)
{
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "master",
Channel = engLatestChannel
}
);
}
if (eng3Channel != null)
{
defaultChannels.Add(
new DefaultChannel(0, "https://github.com/dotnet/arcade", true)
{
Branch = "release/3.x",
Channel = eng3Channel
}
);
}
List <Subscription> subscriptions = (await barOnlyRemote.GetSubscriptionsAsync()).ToList();
// Build, then prune out what we don't want to see if the user specified
// channels.
DependencyFlowGraph flowGraph = await DependencyFlowGraph.BuildAsync(defaultChannels, subscriptions, barOnlyRemote, _options.Days);
Channel targetChannel = null;
if (!string.IsNullOrEmpty(_options.Channel))
{
// Resolve the channel.
targetChannel = await UxHelpers.ResolveSingleChannel(barOnlyRemote, _options.Channel);
if (targetChannel == null)
{
return(Constants.ErrorCode);
}
}
if (targetChannel != null)
{
flowGraph.PruneGraph(
node => DependencyFlowGraph.IsInterestingNode(targetChannel.Name, node),
edge => DependencyFlowGraph.IsInterestingEdge(edge, _options.IncludeDisabledSubscriptions, _options.IncludedFrequencies));
}
if (_options.IncludeBuildTimes)
{
flowGraph.MarkBackEdges();
flowGraph.CalculateLongestBuildPaths();
flowGraph.MarkLongestBuildPath();
}
await LogGraphVizAsync(targetChannel, flowGraph, _options.IncludeBuildTimes);
return(Constants.SuccessCode);
}
catch (Exception exc)
{
Logger.LogError(exc, "Something failed while getting the dependency graph.");
return(Constants.ErrorCode);
}
}
