public bool DetermineFailure()
{
Utilities.WriteDebug($"Determining failure for {Repo}...", Log, LogLevel);
if (BuildBreakdowns.Count == 0)
{
Utilities.WriteDebug($"No builds found for {Repo} this rollout; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
ScorecardBuildBreakdown lastBuild = BuildBreakdowns.Last();
Utilities.WriteDebug($"Last build is for {Repo} is {lastBuild.BuildSummary.BuildNumber} ({lastBuild.BuildSummary.WebLink})", Log, LogLevel);
if (lastBuild.Score.Rollbacks == 1)
{
Utilities.WriteDebug($"Last build ({lastBuild.BuildSummary.BuildNumber}) was a rollback; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
string lastBuildResult = lastBuild.BuildSummary.Result;
Utilities.WriteDebug($"Build {lastBuild.BuildSummary.BuildNumber} has result '{lastBuildResult}'", Log, LogLevel);
switch (lastBuildResult)
{
case "succeeded":
case "partiallySucceeded":
Utilities.WriteDebug($"Last build determined successful.", Log, LogLevel);
return(false);
default:
Utilities.WriteDebug($"Last build determined unsuccessful; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
}
/// <summary>
/// Parses a Scorecard object from a given CSV file
/// </summary>
/// <param name="filePath">Path to CSV file</param>
/// <param name="config">Config object to use during parsing</param>
/// <returns></returns>
public async static Task <Scorecard> ParseScorecardFromCsvAsync(string filePath, Config config)
{
using (StreamReader file = new StreamReader(filePath))
{
Scorecard scorecard = new Scorecard
{
RolloutWeightConfig = config.RolloutWeightConfig
};
string[] rolloutInfo = (await file.ReadLineAsync()).Split(',');
scorecard.Repo = config.RepoConfigs.Find(r => r.Repo == rolloutInfo[0]);
scorecard.Date = DateTimeOffset.Parse(rolloutInfo[1]);
await file.ReadLineAsync();
await file.ReadLineAsync();
string[] rolloutScorecardSummary = (await file.ReadLineAsync()).Split(',');
scorecard.TimeToRollout = TimeSpan.Parse(rolloutScorecardSummary[0]);
scorecard.CriticalIssues = int.Parse(rolloutScorecardSummary[1]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[2]));
scorecard.Hotfixes = int.Parse(rolloutScorecardSummary[3]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[4]));
scorecard.Rollbacks = int.Parse(rolloutScorecardSummary[5]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[6]));
scorecard.Downtime = TimeSpan.Parse(rolloutScorecardSummary[7]);
scorecard.Failure = bool.Parse(rolloutScorecardSummary[8]);
scorecard._githubIssuesUris.AddRange(GetIssueLinksFromString(rolloutScorecardSummary[9]));
await file.ReadLineAsync();
await file.ReadLineAsync();
string[] buildBreakdownLines = (await file.ReadToEndAsync()).Split('\n');
foreach (string breakdownLine in buildBreakdownLines)
{
if (breakdownLine.Length == 0)
{
break;
}
BuildSummary buildSummary = new BuildSummary();
string[] breakdownSummary = breakdownLine.Split(',');
buildSummary.BuildNumber = breakdownSummary[0];
buildSummary.Links.WebLink.Href = breakdownSummary[1];
ScorecardBuildBreakdown buildBreakdown = new ScorecardBuildBreakdown(buildSummary);
buildBreakdown.Score.TimeToRollout = TimeSpan.Parse(breakdownSummary[2]);
buildBreakdown.Score.CriticalIssues = int.Parse(breakdownSummary[3]);
buildBreakdown.Score.Hotfixes = int.Parse(breakdownSummary[4]);
buildBreakdown.Score.Rollbacks = int.Parse(breakdownSummary[5]);
buildBreakdown.Score.Downtime = TimeSpan.Parse(breakdownSummary[6]);
scorecard.BuildBreakdowns.Add(buildBreakdown);
}
return(scorecard);
}
}
public async Task CollectStages(ScorecardBuildBreakdown buildBreakdown)
{
string timelineLinkWithAttempts = $"{buildBreakdown.BuildSummary.TimelineLink}?changeId=1";
Utilities.WriteDebug($"Querying AzDO API: {timelineLinkWithAttempts}", Log, LogLevel);
JObject jsonTimelineResponse = await GetAzdoApiResponseAsync(timelineLinkWithAttempts);
BuildTimeline timeline = jsonTimelineResponse.ToObject <BuildTimeline>();
if (timeline.Records != null)
{
// We're going to use this to store previous attempts as we find them
Dictionary <string, BuildTimeline> previousAttemptTimelines = new Dictionary <string, BuildTimeline>();
foreach (BuildTimelineEntry record in timeline.Records)
{
// We measure times at the stage level because this is the simplest thing to do
// By taking the min start time and max end time of all the stages except for the ones we exclude,
// we can determine a pretty accurate measurement of how long it took to rollout
if ((record.Type == "Checkpoint.Approval" || record.Type == "Stage") && !RepoConfig.ExcludeStages.Any(s => s == record.Name))
{
buildBreakdown.BuildSummary.Stages.Add(record);
if (record.PreviousAttempts.Count > 0)
{
// we're going to just add these attempts as additional stages
foreach (PreviousAttempt attempt in record.PreviousAttempts)
{
if (!previousAttemptTimelines.ContainsKey(attempt.TimelineId))
{
previousAttemptTimelines.Add(attempt.TimelineId,
(await GetAzdoApiResponseAsync($"{buildBreakdown.BuildSummary.TimelineLink}/{attempt.TimelineId}")).ToObject <BuildTimeline>());
}
if (previousAttemptTimelines[attempt.TimelineId] != null)
{
buildBreakdown.BuildSummary.Stages.Add(previousAttemptTimelines[attempt.TimelineId].Records
.Where(t => (t.Type == "Checkpoint.Approval" || t.Type == "Stage") && t.Name == record.Name).First());
}
}
}
}
}
if (buildBreakdown.BuildSummary.Stages.Any(s => s.Name.StartsWith("deploy", StringComparison.InvariantCultureIgnoreCase) && !string.IsNullOrEmpty(s.EndTime)))
{
buildBreakdown.BuildSummary.DeploymentReached = true;
Utilities.WriteDebug($"Build {buildBreakdown.BuildSummary.BuildNumber} determined to have reached deployment.", Log, LogLevel);
}
else
{
Utilities.WriteDebug($"Build {buildBreakdown.BuildSummary.BuildNumber} determined to NOT have reached deployment.", Log, LogLevel);
}
}
}
public async static Task <Scorecard> CreateScorecardAsync(RolloutScorer rolloutScorer)
{
(int numHotfixes, int numRollbacks) = await rolloutScorer.CalculateNumHotfixesAndRollbacksFromAzdoAsync();
List <Issue> githubIssues = await rolloutScorer.GetRolloutIssuesFromGithubAsync();
string repoLabel = rolloutScorer.RepoConfig.GithubIssueLabel;
Scorecard scorecard = new Scorecard
{
Repo = rolloutScorer.RepoConfig,
Date = rolloutScorer.RolloutStartDate,
TimeToRollout = await rolloutScorer.CalculateTimeToRolloutAsync(),
CriticalIssues = githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log)),
Hotfixes = numHotfixes + githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log)),
Rollbacks = numRollbacks + githubIssues
.Count(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log)),
Downtime = await rolloutScorer.CalculateDowntimeAsync(githubIssues) + rolloutScorer.Downtime,
Failure = rolloutScorer.DetermineFailure() || rolloutScorer.Failed,
BuildBreakdowns = rolloutScorer.BuildBreakdowns,
RolloutWeightConfig = rolloutScorer.RolloutWeightConfig,
GithubIssues = githubIssues,
};
// Critical issues and manual hotfixes/rollbacks need to be included in the build breakdowns, but it isn't possible to determine which
// builds they belong to; so we'll just append the issues to the first build and the hotfixes/rollbacks to the last
if (scorecard.BuildBreakdowns.Count > 0)
{
scorecard.BuildBreakdowns.Sort((x, y) => x.BuildSummary.BuildNumber.CompareTo(y.BuildSummary.BuildNumber));
// Critical issues are assumed to have been caused by the first deployment
ScorecardBuildBreakdown firstDeployment = scorecard.BuildBreakdowns.First();
firstDeployment.Score.CriticalIssues += scorecard.CriticalIssues;
firstDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues
.Where(issue => Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.IssueLabel, repoLabel, rolloutScorer.Log)));
// Hotfixes & rollbacks are assumed to have taken place in the last deployment
// This is likely incorrect given >2 deployments but can be manually adjusted if necessary
ScorecardBuildBreakdown lastDeployment = scorecard.BuildBreakdowns.Last();
lastDeployment.Score.Hotfixes += rolloutScorer.ManualHotfixes;
lastDeployment.Score.Rollbacks += rolloutScorer.ManualRollbacks;
lastDeployment.Score.GithubIssues.AddRange(scorecard.GithubIssues
.Where(issue =>
Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.HotfixLabel, repoLabel, rolloutScorer.Log) ||
Utilities.IssueContainsRelevantLabels(issue, GithubLabelNames.RollbackLabel, repoLabel, rolloutScorer.Log)));
}
return(scorecard);
}
public bool DetermineFailure(List <Issue> githubIssues)
{
Utilities.WriteDebug($"Determining failure for {Repo}...", Log, LogLevel);
if (githubIssues.Any(i => Utilities.IssueContainsRelevantLabels(i, GithubLabelNames.FailureLabel, RepoConfig.GithubIssueLabel, Log, LogLevel)))
{
Utilities.WriteDebug($"Issue with failure tag found for {Repo}; rollout marked as FAILED", Log, LogLevel);
return(true);
}
if (BuildBreakdowns.Count == 0)
{
Utilities.WriteDebug($"No builds found for {Repo} this rollout; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
ScorecardBuildBreakdown lastBuild = BuildBreakdowns.Last();
Utilities.WriteDebug($"Last build is for {Repo} is {lastBuild.BuildSummary.BuildNumber} ({lastBuild.BuildSummary.WebLink})", Log, LogLevel);
if (lastBuild.Score.Rollbacks == 1)
{
Utilities.WriteDebug($"Last build ({lastBuild.BuildSummary.BuildNumber}) was a rollback; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
string lastBuildResult = lastBuild.BuildSummary.Result;
Utilities.WriteDebug($"Build {lastBuild.BuildSummary.BuildNumber} has result '{lastBuildResult}'", Log, LogLevel);
switch (lastBuildResult)
{
case "succeeded":
case "partiallySucceeded":
Utilities.WriteDebug($"Last build determined successful.", Log, LogLevel);
return(false);
default:
Utilities.WriteDebug($"Last build determined unsuccessful; rollout marked as FAILED.", Log, LogLevel);
return(true);
}
}