/// <summary>
/// Uploads results to GitHub/Azure Table Storage
/// </summary>
/// <param name="scorecardFiles">List of paths to scorecard CSV files</param>
/// <param name="config">Parsed Config object representing config file</param>
/// <param name="githubClient">An authenticated Octokit.GitHubClient instance</param>
/// <param name="storageAccountKey">A secret bundle containing the key to the rollout scorecards storage account</param>
/// <returns>Exit code (0 = success, 1 = failure)</returns>
public async static Task <int> UploadResultsAsync(List <string> scorecardFiles, GitHubClient githubClient, string storageAccountKey, ILogger log = null, bool skipPr = false)
{
try
{
await UploadResultsAsync(new List <Scorecard>(
await Task.WhenAll(scorecardFiles.Select(
file => Scorecard.ParseScorecardFromCsvAsync(file, StandardConfig.DefaultConfig)
))), githubClient, storageAccountKey, StandardConfig.DefaultConfig.GithubConfig, skipPr);
}
catch (IOException e)
{
Utilities.WriteError($"File could not be opened for writing; do you have it open in Excel?", log);
Utilities.WriteError(e.Message, log);
return(1);
}
string successMessage = $"Successfully uploaded:\n\t{string.Join("\n\t", scorecardFiles)}";
if (log == null)
{
Console.WriteLine(successMessage);
}
else
{
log.LogInformation(successMessage);
}
return(0);
}
/// <summary>
/// Parses a Scorecard object from a given CSV file
/// </summary>
/// <param name="filePath">Path to CSV file</param>
/// <param name="config">Config object to use during parsing</param>
/// <returns></returns>
public async static Task <Scorecard> ParseScorecardFromCsvAsync(string filePath, Config config)
{
using (StreamReader file = new StreamReader(filePath))
{
Scorecard scorecard = new Scorecard
{
RolloutWeightConfig = config.RolloutWeightConfig
};
string[] rolloutInfo = (await file.ReadLineAsync()).Split(',');
scorecard.Repo = config.RepoConfigs.Find(r => r.Repo == rolloutInfo[0]);
scorecard.Date = DateTimeOffset.Parse(rolloutInfo[1]);
await file.ReadLineAsync();
await file.ReadLineAsync();
string[] rolloutScorecardSummary = (await file.ReadLineAsync()).Split(',');
scorecard.TimeToRollout = TimeSpan.Parse(rolloutScorecardSummary[0]);
scorecard.CriticalIssues = int.Parse(rolloutScorecardSummary[1]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[2]));
scorecard.Hotfixes = int.Parse(rolloutScorecardSummary[3]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[4]));
scorecard.Rollbacks = int.Parse(rolloutScorecardSummary[5]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[6]));
scorecard.Downtime = TimeSpan.Parse(rolloutScorecardSummary[7]);
scorecard.Failure = bool.Parse(rolloutScorecardSummary[8]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[9]));
await file.ReadLineAsync();
await file.ReadLineAsync();
string[] buildBreakdownLines = (await file.ReadToEndAsync()).Split('\n');
foreach (string breakdownLine in buildBreakdownLines)
{
if (breakdownLine.Length == 0)
{
break;
}
BuildSummary buildSummary = new BuildSummary();
string[] breakdownSummary = breakdownLine.Split(',');
buildSummary.BuildNumber = breakdownSummary[0];
buildSummary.Links.WebLink.Href = breakdownSummary[1];
ScorecardBuildBreakdown buildBreakdown = new ScorecardBuildBreakdown(buildSummary);
buildBreakdown.Score.TimeToRollout = TimeSpan.Parse(breakdownSummary[2]);
buildBreakdown.Score.CriticalIssues = int.Parse(breakdownSummary[3]);
buildBreakdown.Score.Hotfixes = int.Parse(breakdownSummary[4]);
buildBreakdown.Score.Rollbacks = int.Parse(breakdownSummary[5]);
buildBreakdown.Score.Downtime = TimeSpan.Parse(breakdownSummary[6]);
scorecard.BuildBreakdowns.Add(buildBreakdown);
}
return(scorecard);
}
}
private static RepoMarkdown CreateRepoMarkdown(Scorecard scorecard)
{
string summary = $"## {scorecard.Repo.Repo}\n\n" +
$"| Metric | Value | Target | Score |\n" +
$"|:--------------------------------:|:--------:|:--------:|:---------:|\n" +
$"| Time to Rollout | {scorecard.TimeToRollout} | {TimeSpan.FromMinutes(scorecard.Repo.ExpectedTime)} | {scorecard.TimeToRolloutScore} |\n" +
$"| Critical/blocking issues created | {scorecard.CriticalIssues} | 0 | {scorecard.CriticalIssueScore} |\n" +
$"| Hotfixes | {scorecard.Hotfixes} | 0 | {scorecard.HotfixScore} |\n" +
$"| Rollbacks | {scorecard.Rollbacks} | 0 | {scorecard.RollbackScore} |\n" +
$"| Service downtime | {scorecard.Downtime} | 00:00:00 | {scorecard.DowntimeScore} |\n" +
$"| Failed to rollout | {scorecard.Failure.ToString().ToUpperInvariant()} | FALSE | {(scorecard.Failure ? scorecard.RolloutWeightConfig.FailurePoints : 0)} |\n" +
$"| Total | | | **{scorecard.TotalScore}** |\n\n" +
$"{CreateGithubIssueUrisMarkdown(scorecard.GithubIssueUris)}";
string breakdown = "";
if (scorecard.BuildBreakdowns.Count > 0)
{
string breakdownTableHeader = "| Metric |";
string breakdownTableColumns = "|:-----:|";
string breakdownTimeToRolloutRow = "| Time to Rollout |";
string breakdownCriticalIssuesRow = "| Critical/blocking issues created |";
string breakdownHotfixesRow = "| Hotfixes |";
string breakdownRollbacksRow = "| Rollbacks |";
string breakdownDowntime = "| Service downtime |";
foreach (ScorecardBuildBreakdown scorecardBreakdown in scorecard.BuildBreakdowns)
{
breakdownTableHeader += $" [{scorecardBreakdown.BuildSummary.BuildNumber}]({scorecardBreakdown.BuildSummary.WebLink}) |";
breakdownTableColumns += ":-----:|";
breakdownTimeToRolloutRow += $" {scorecardBreakdown.Score.TimeToRollout} |";
breakdownCriticalIssuesRow += $" {scorecardBreakdown.Score.CriticalIssues} |";
breakdownHotfixesRow += $" {scorecardBreakdown.Score.Hotfixes} |";
breakdownRollbacksRow += $" {scorecardBreakdown.Score.Rollbacks} |";
breakdownDowntime += $" {scorecardBreakdown.Score.Downtime} |";
}
breakdown = $"## {scorecard.Repo.Repo}\n\n" +
$"{breakdownTableHeader}\n" +
$"{breakdownTableColumns}\n" +
$"{breakdownTimeToRolloutRow}\n" +
$"{breakdownCriticalIssuesRow}\n" +
$"{breakdownHotfixesRow}\n" +
$"{breakdownRollbacksRow}\n" +
$"{breakdownDowntime}\n\n";
}
return(new RepoMarkdown {
Summary = summary, Breakdown = breakdown
});
}
public async static Task <Scorecard> CreateScorecardAsync(RolloutScorer rolloutScorer)
{
(int numHotfixes, int numRollbacks) = await rolloutScorer.CalculateNumHotfixesAndRollbacksFromAzdoAsync();
List <Issue> githubIssues = await rolloutScorer.GetRolloutIssuesFromGithubAsync();
string repoLabel = rolloutScorer.RepoConfig.GithubIssueLabel;
Scorecard scorecard = new Scorecard
{
Repo = rolloutScorer.RepoConfig,
Date = rolloutScorer.RolloutStartDate,
TimeToRollout = await rolloutScorer.CalculateTimeToRolloutAsync(),
CriticalIssues = githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log)),
Hotfixes = numHotfixes + githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log)),
Rollbacks = numRollbacks + githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log)),
Downtime = await rolloutScorer.CalculateDowntimeAsync(githubIssues) + rolloutScorer.Downtime,
Failure = rolloutScorer.DetermineFailure() || rolloutScorer.Failed,
BuildBreakdowns = rolloutScorer.BuildBreakdowns,
RolloutWeightConfig = rolloutScorer.RolloutWeightConfig,
GithubIssues = githubIssues,
};
// Critical issues and manual hotfixes/rollbacks need to be included in the build breakdowns, but it isn't possible to determine which
// builds they belong to; so we'll just append the issues to the first build and the hotfixes/rollbacks to the last
if (scorecard.BuildBreakdowns.Count > 0)
{
scorecard.BuildBreakdowns.Sort((x, y) => x.BuildSummary.BuildNumber.CompareTo(y.BuildSummary.BuildNumber));
// Critical issues are assumed to have been caused by the first deployment
ScorecardBuildBreakdown firstDeployment = scorecard.BuildBreakdowns.First();
firstDeployment.Score.CriticalIssues += scorecard.CriticalIssues;
firstDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues
.Where(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log)));
// Hotfixes & rollbacks are assumed to have taken place in the last deployment
// This is likely incorrect given >2 deployments but can be manually adjusted if necessary
ScorecardBuildBreakdown lastDeployment = scorecard.BuildBreakdowns.Last();
lastDeployment.Score.Hotfixes += rolloutScorer.ManualHotfixes;
lastDeployment.Score.Rollbacks += rolloutScorer.ManualRollbacks;
lastDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues
.Where(issue =>
Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log) ||
Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log)));
}
return(scorecard);
}
public async Task <int> InvokeAsync(IEnumerable <string> arguments)
{
Options.Parse(arguments);
if (_showHelp)
{
Options.WriteOptionDescriptions(CommandSet.Out);
return(0);
}
if (string.IsNullOrEmpty(_rolloutScorer.OutputFile))
{
_rolloutScorer.OutputFile = Path.Combine(Directory.GetCurrentDirectory(),
$"{_rolloutScorer.Repo}-{_rolloutScorer.RolloutStartDate.Date.ToShortDateString().Replace("/","-")}-scorecard.csv");
}
_rolloutScorer.RolloutWeightConfig = StandardConfig.DefaultConfig.RolloutWeightConfig;
_rolloutScorer.GithubConfig = StandardConfig.DefaultConfig.GithubConfig;
// If they haven't told us to upload but they also haven't specified a repo & rollout start date, we need to throw
if (string.IsNullOrEmpty(_rolloutScorer.Repo) || (_rolloutScorer.RolloutStartDate == null))
{
Utilities.WriteError($"ERROR: One or both of required parameters 'repo' and 'rollout-start-date' were not specified.");
return(1);
}
_rolloutScorer.RepoConfig = StandardConfig.DefaultConfig.RepoConfigs.Find(r => r.Repo == _rolloutScorer.Repo);
if (_rolloutScorer.RepoConfig == null)
{
Utilities.WriteError($"ERROR: Provided repo '{_rolloutScorer.Repo}' does not exist in config file");
return(1);
}
_rolloutScorer.AzdoConfig = StandardConfig.DefaultConfig.AzdoInstanceConfigs.Find(a => a.Name == _rolloutScorer.RepoConfig.AzdoInstance);
if (_rolloutScorer.AzdoConfig == null)
{
Utilities.WriteError($"ERROR: Configuration file is invalid; repo '{_rolloutScorer.RepoConfig.Repo}' " +
$"references unknown AzDO instance '{_rolloutScorer.RepoConfig.AzdoInstance}'");
return(1);
}
// Get the AzDO & GitHub PATs from key vault
AzureServiceTokenProvider tokenProvider = new AzureServiceTokenProvider();
SecretBundle githubPat;
SecretBundle storageAccountConnectionString;
using (KeyVaultClient kv = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(tokenProvider.KeyVaultTokenCallback)))
{
Console.WriteLine("Fetching PATs from key vault.");
_rolloutScorer.SetupHttpClient((await kv.GetSecretAsync(_rolloutScorer.AzdoConfig.KeyVaultUri, _rolloutScorer.AzdoConfig.PatSecretName)).Value);
githubPat = await kv.GetSecretAsync(Utilities.KeyVaultUri, Utilities.GitHubPatSecretName);
_rolloutScorer.SetupGithubClient(githubPat.Value);
storageAccountConnectionString = await kv.GetSecretAsync(Utilities.KeyVaultUri, ScorecardsStorageAccount.KeySecretName);
}
try
{
await _rolloutScorer.InitAsync();
}
catch (ArgumentException e)
{
Utilities.WriteError(e.Message);
return(1);
}
Scorecard scorecard = await Scorecard.CreateScorecardAsync(_rolloutScorer);
string expectedTimeToRollout = TimeSpan.FromMinutes(_rolloutScorer.RepoConfig.ExpectedTime).ToString();
Console.WriteLine($"The {_rolloutScorer.Repo} {_rolloutScorer.RolloutStartDate.Date.ToShortDateString()} rollout score is {scorecard.TotalScore}.\n");
Console.WriteLine($"| Metric | Value | Target | Score |");
Console.WriteLine($"|:--------------------------------:|:--------:|:--------:|:---------:|");
Console.WriteLine($"| Time to Rollout | {scorecard.TimeToRollout} | {expectedTimeToRollout} | {scorecard.TimeToRolloutScore} |");
Console.WriteLine($"| Critical/blocking issues created | {scorecard.CriticalIssues} | 0 | {scorecard.CriticalIssueScore} |");
Console.WriteLine($"| Hotfixes | {scorecard.Hotfixes} | 0 | {scorecard.HotfixScore} |");
Console.WriteLine($"| Rollbacks | {scorecard.Rollbacks} | 0 | {scorecard.RollbackScore} |");
Console.WriteLine($"| Service downtime | {scorecard.Downtime} | 00:00:00 | {scorecard.DowntimeScore} |");
Console.WriteLine($"| Failed to rollout | {scorecard.Failure.ToString().ToUpperInvariant()} | FALSE | {(scorecard.Failure ? StandardConfig.DefaultConfig.RolloutWeightConfig.FailurePoints : 0)} |");
Console.WriteLine($"| Total | | | **{scorecard.TotalScore}** |");
if (_rolloutScorer.Upload)
{
Console.WriteLine("Directly uploading results.");
await RolloutUploader.UploadResultsAsync(new List <Scorecard> {
scorecard
}, Utilities.GetGithubClient(githubPat.Value), storageAccountConnectionString.Value, _rolloutScorer.GithubConfig);
}
if (_rolloutScorer.SkipOutput)
{
Console.WriteLine("Skipping output step.");
}
else
{
if (await scorecard.Output(_rolloutScorer.OutputFile) != 0)
{
return(1);
}
Console.WriteLine($"Wrote output to file {_rolloutScorer.OutputFile}");
}
return(0);
}
public ScorecardBuildBreakdown(BuildSummary buildSummary)
{
BuildSummary = buildSummary;
Score = new Scorecard();
}
