/// <summary> /// Uploads results to GitHub/Azure Table Storage /// </summary> /// <param name="scorecardFiles">List of paths to scorecard CSV files</param> /// <param name="config">Parsed Config object representing config file</param> /// <param name="githubClient">An authenticated Octokit.GitHubClient instance</param> /// <param name="storageAccountKey">A secret bundle containing the key to the rollout scorecards storage account</param> /// <returns>Exit code (0 = success, 1 = failure)</returns> public async static Task <int> UploadResultsAsync(List <string> scorecardFiles, GitHubClient githubClient, string storageAccountKey, ILogger log = null, bool skipPr = false) { try { await UploadResultsAsync(new List <Scorecard>( await Task.WhenAll(scorecardFiles.Select( file => Scorecard.ParseScorecardFromCsvAsync(file, StandardConfig.DefaultConfig) ))), githubClient, storageAccountKey, StandardConfig.DefaultConfig.GithubConfig, skipPr); } catch (IOException e) { Utilities.WriteError($"File could not be opened for writing; do you have it open in Excel?", log); Utilities.WriteError(e.Message, log); return(1); } string successMessage = $"Successfully uploaded:\n\t{string.Join("\n\t", scorecardFiles)}"; if (log == null) { Console.WriteLine(successMessage); } else { log.LogInformation(successMessage); } return(0); }
/// <summary> /// Parses a Scorecard object from a given CSV file /// </summary> /// <param name="filePath">Path to CSV file</param> /// <param name="config">Config object to use during parsing</param> /// <returns></returns> public async static Task <Scorecard> ParseScorecardFromCsvAsync(string filePath, Config config) { using (StreamReader file = new StreamReader(filePath)) { Scorecard scorecard = new Scorecard { RolloutWeightConfig = config.RolloutWeightConfig }; string[] rolloutInfo = (await file.ReadLineAsync()).Split(','); scorecard.Repo = config.RepoConfigs.Find(r => r.Repo == rolloutInfo[0]); scorecard.Date = DateTimeOffset.Parse(rolloutInfo[1]); await file.ReadLineAsync(); await file.ReadLineAsync(); string[] rolloutScorecardSummary = (await file.ReadLineAsync()).Split(','); scorecard.TimeToRollout = TimeSpan.Parse(rolloutScorecardSummary[0]); scorecard.CriticalIssues = int.Parse(rolloutScorecardSummary[1]); scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[2])); scorecard.Hotfixes = int.Parse(rolloutScorecardSummary[3]); scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[4])); scorecard.Rollbacks = int.Parse(rolloutScorecardSummary[5]); scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[6])); scorecard.Downtime = TimeSpan.Parse(rolloutScorecardSummary[7]); scorecard.Failure = bool.Parse(rolloutScorecardSummary[8]); scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[9])); await file.ReadLineAsync(); await file.ReadLineAsync(); string[] buildBreakdownLines = (await file.ReadToEndAsync()).Split('\n'); foreach (string breakdownLine in buildBreakdownLines) { if (breakdownLine.Length == 0) { break; } BuildSummary buildSummary = new BuildSummary(); string[] breakdownSummary = breakdownLine.Split(','); buildSummary.BuildNumber = breakdownSummary[0]; buildSummary.Links.WebLink.Href = breakdownSummary[1]; ScorecardBuildBreakdown buildBreakdown = new ScorecardBuildBreakdown(buildSummary); buildBreakdown.Score.TimeToRollout = TimeSpan.Parse(breakdownSummary[2]); buildBreakdown.Score.CriticalIssues = int.Parse(breakdownSummary[3]); buildBreakdown.Score.Hotfixes = int.Parse(breakdownSummary[4]); buildBreakdown.Score.Rollbacks = int.Parse(breakdownSummary[5]); buildBreakdown.Score.Downtime = TimeSpan.Parse(breakdownSummary[6]); scorecard.BuildBreakdowns.Add(buildBreakdown); } return(scorecard); } }
private static RepoMarkdown CreateRepoMarkdown(Scorecard scorecard) { string summary = $"## {scorecard.Repo.Repo}\n\n" + $"| Metric | Value | Target | Score |\n" + $"|:--------------------------------:|:--------:|:--------:|:---------:|\n" + $"| Time to Rollout | {scorecard.TimeToRollout} | {TimeSpan.FromMinutes(scorecard.Repo.ExpectedTime)} | {scorecard.TimeToRolloutScore} |\n" + $"| Critical/blocking issues created | {scorecard.CriticalIssues} | 0 | {scorecard.CriticalIssueScore} |\n" + $"| Hotfixes | {scorecard.Hotfixes} | 0 | {scorecard.HotfixScore} |\n" + $"| Rollbacks | {scorecard.Rollbacks} | 0 | {scorecard.RollbackScore} |\n" + $"| Service downtime | {scorecard.Downtime} | 00:00:00 | {scorecard.DowntimeScore} |\n" + $"| Failed to rollout | {scorecard.Failure.ToString().ToUpperInvariant()} | FALSE | {(scorecard.Failure ? scorecard.RolloutWeightConfig.FailurePoints : 0)} |\n" + $"| Total | | | **{scorecard.TotalScore}** |\n\n" + $"{CreateGithubIssueUrisMarkdown(scorecard.GithubIssueUris)}"; string breakdown = ""; if (scorecard.BuildBreakdowns.Count > 0) { string breakdownTableHeader = "| Metric |"; string breakdownTableColumns = "|:-----:|"; string breakdownTimeToRolloutRow = "| Time to Rollout |"; string breakdownCriticalIssuesRow = "| Critical/blocking issues created |"; string breakdownHotfixesRow = "| Hotfixes |"; string breakdownRollbacksRow = "| Rollbacks |"; string breakdownDowntime = "| Service downtime |"; foreach (ScorecardBuildBreakdown scorecardBreakdown in scorecard.BuildBreakdowns) { breakdownTableHeader += $" [{scorecardBreakdown.BuildSummary.BuildNumber}]({scorecardBreakdown.BuildSummary.WebLink}) |"; breakdownTableColumns += ":-----:|"; breakdownTimeToRolloutRow += $" {scorecardBreakdown.Score.TimeToRollout} |"; breakdownCriticalIssuesRow += $" {scorecardBreakdown.Score.CriticalIssues} |"; breakdownHotfixesRow += $" {scorecardBreakdown.Score.Hotfixes} |"; breakdownRollbacksRow += $" {scorecardBreakdown.Score.Rollbacks} |"; breakdownDowntime += $" {scorecardBreakdown.Score.Downtime} |"; } breakdown = $"## {scorecard.Repo.Repo}\n\n" + $"{breakdownTableHeader}\n" + $"{breakdownTableColumns}\n" + $"{breakdownTimeToRolloutRow}\n" + $"{breakdownCriticalIssuesRow}\n" + $"{breakdownHotfixesRow}\n" + $"{breakdownRollbacksRow}\n" + $"{breakdownDowntime}\n\n"; } return(new RepoMarkdown { Summary = summary, Breakdown = breakdown }); }
public async static Task <Scorecard> CreateScorecardAsync(RolloutScorer rolloutScorer) { (int numHotfixes, int numRollbacks) = await rolloutScorer.CalculateNumHotfixesAndRollbacksFromAzdoAsync(); List <Issue> githubIssues = await rolloutScorer.GetRolloutIssuesFromGithubAsync(); string repoLabel = rolloutScorer.RepoConfig.GithubIssueLabel; Scorecard scorecard = new Scorecard { Repo = rolloutScorer.RepoConfig, Date = rolloutScorer.RolloutStartDate, TimeToRollout = await rolloutScorer.CalculateTimeToRolloutAsync(), CriticalIssues = githubIssues .Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log)), Hotfixes = numHotfixes + githubIssues .Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log)), Rollbacks = numRollbacks + githubIssues .Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log)), Downtime = await rolloutScorer.CalculateDowntimeAsync(githubIssues) + rolloutScorer.Downtime, Failure = rolloutScorer.DetermineFailure() || rolloutScorer.Failed, BuildBreakdowns = rolloutScorer.BuildBreakdowns, RolloutWeightConfig = rolloutScorer.RolloutWeightConfig, GithubIssues = githubIssues, }; // Critical issues and manual hotfixes/rollbacks need to be included in the build breakdowns, but it isn't possible to determine which // builds they belong to; so we'll just append the issues to the first build and the hotfixes/rollbacks to the last if (scorecard.BuildBreakdowns.Count > 0) { scorecard.BuildBreakdowns.Sort((x, y) => x.BuildSummary.BuildNumber.CompareTo(y.BuildSummary.BuildNumber)); // Critical issues are assumed to have been caused by the first deployment ScorecardBuildBreakdown firstDeployment = scorecard.BuildBreakdowns.First(); firstDeployment.Score.CriticalIssues += scorecard.CriticalIssues; firstDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues .Where(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log))); // Hotfixes & rollbacks are assumed to have taken place in the last deployment // This is likely incorrect given >2 deployments but can be manually adjusted if necessary ScorecardBuildBreakdown lastDeployment = scorecard.BuildBreakdowns.Last(); lastDeployment.Score.Hotfixes += rolloutScorer.ManualHotfixes; lastDeployment.Score.Rollbacks += rolloutScorer.ManualRollbacks; lastDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues .Where(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log) || Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log))); } return(scorecard); }
public async Task <int> InvokeAsync(IEnumerable <string> arguments) { Options.Parse(arguments); if (_showHelp) { Options.WriteOptionDescriptions(CommandSet.Out); return(0); } if (string.IsNullOrEmpty(_rolloutScorer.OutputFile)) { _rolloutScorer.OutputFile = Path.Combine(Directory.GetCurrentDirectory(), $"{_rolloutScorer.Repo}-{_rolloutScorer.RolloutStartDate.Date.ToShortDateString().Replace("/","-")}-scorecard.csv"); } _rolloutScorer.RolloutWeightConfig = StandardConfig.DefaultConfig.RolloutWeightConfig; _rolloutScorer.GithubConfig = StandardConfig.DefaultConfig.GithubConfig; // If they haven't told us to upload but they also haven't specified a repo & rollout start date, we need to throw if (string.IsNullOrEmpty(_rolloutScorer.Repo) || (_rolloutScorer.RolloutStartDate == null)) { Utilities.WriteError($"ERROR: One or both of required parameters 'repo' and 'rollout-start-date' were not specified."); return(1); } _rolloutScorer.RepoConfig = StandardConfig.DefaultConfig.RepoConfigs.Find(r => r.Repo == _rolloutScorer.Repo); if (_rolloutScorer.RepoConfig == null) { Utilities.WriteError($"ERROR: Provided repo '{_rolloutScorer.Repo}' does not exist in config file"); return(1); } _rolloutScorer.AzdoConfig = StandardConfig.DefaultConfig.AzdoInstanceConfigs.Find(a => a.Name == _rolloutScorer.RepoConfig.AzdoInstance); if (_rolloutScorer.AzdoConfig == null) { Utilities.WriteError($"ERROR: Configuration file is invalid; repo '{_rolloutScorer.RepoConfig.Repo}' " + $"references unknown AzDO instance '{_rolloutScorer.RepoConfig.AzdoInstance}'"); return(1); } // Get the AzDO & GitHub PATs from key vault AzureServiceTokenProvider tokenProvider = new AzureServiceTokenProvider(); SecretBundle githubPat; SecretBundle storageAccountConnectionString; using (KeyVaultClient kv = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(tokenProvider.KeyVaultTokenCallback))) { Console.WriteLine("Fetching PATs from key vault."); _rolloutScorer.SetupHttpClient((await kv.GetSecretAsync(_rolloutScorer.AzdoConfig.KeyVaultUri, _rolloutScorer.AzdoConfig.PatSecretName)).Value); githubPat = await kv.GetSecretAsync(Utilities.KeyVaultUri, Utilities.GitHubPatSecretName); _rolloutScorer.SetupGithubClient(githubPat.Value); storageAccountConnectionString = await kv.GetSecretAsync(Utilities.KeyVaultUri, ScorecardsStorageAccount.KeySecretName); } try { await _rolloutScorer.InitAsync(); } catch (ArgumentException e) { Utilities.WriteError(e.Message); return(1); } Scorecard scorecard = await Scorecard.CreateScorecardAsync(_rolloutScorer); string expectedTimeToRollout = TimeSpan.FromMinutes(_rolloutScorer.RepoConfig.ExpectedTime).ToString(); Console.WriteLine($"The {_rolloutScorer.Repo} {_rolloutScorer.RolloutStartDate.Date.ToShortDateString()} rollout score is {scorecard.TotalScore}.\n"); Console.WriteLine($"| Metric | Value | Target | Score |"); Console.WriteLine($"|:--------------------------------:|:--------:|:--------:|:---------:|"); Console.WriteLine($"| Time to Rollout | {scorecard.TimeToRollout} | {expectedTimeToRollout} | {scorecard.TimeToRolloutScore} |"); Console.WriteLine($"| Critical/blocking issues created | {scorecard.CriticalIssues} | 0 | {scorecard.CriticalIssueScore} |"); Console.WriteLine($"| Hotfixes | {scorecard.Hotfixes} | 0 | {scorecard.HotfixScore} |"); Console.WriteLine($"| Rollbacks | {scorecard.Rollbacks} | 0 | {scorecard.RollbackScore} |"); Console.WriteLine($"| Service downtime | {scorecard.Downtime} | 00:00:00 | {scorecard.DowntimeScore} |"); Console.WriteLine($"| Failed to rollout | {scorecard.Failure.ToString().ToUpperInvariant()} | FALSE | {(scorecard.Failure ? StandardConfig.DefaultConfig.RolloutWeightConfig.FailurePoints : 0)} |"); Console.WriteLine($"| Total | | | **{scorecard.TotalScore}** |"); if (_rolloutScorer.Upload) { Console.WriteLine("Directly uploading results."); await RolloutUploader.UploadResultsAsync(new List <Scorecard> { scorecard }, Utilities.GetGithubClient(githubPat.Value), storageAccountConnectionString.Value, _rolloutScorer.GithubConfig); } if (_rolloutScorer.SkipOutput) { Console.WriteLine("Skipping output step."); } else { if (await scorecard.Output(_rolloutScorer.OutputFile) != 0) { return(1); } Console.WriteLine($"Wrote output to file {_rolloutScorer.OutputFile}"); } return(0); }
public ScorecardBuildBreakdown(BuildSummary buildSummary) { BuildSummary = buildSummary; Score = new Scorecard(); }