/// <summary>
/// Updates and closes existing issues based on regressions found.
/// </summary>
/// <returns>
/// The remaining issues that haven't been reported yet.
/// </returns>
private static async Task <IEnumerable <Regression> > UpdateIssues(IEnumerable <Regression> regressions, Source source, string template)
{
if (!regressions.Any())
{
return(regressions);
}
if (_options.Debug)
{
return(regressions);
}
var issues = await GetRecentIssues(source);
Console.WriteLine($"Downloaded {issues.Count()} issues");
// The list of regressions that remain to be reported
var regressionsToReport = new List <Regression>(regressions).ToDictionary(x => x.Identifier, x => x);
foreach (var issue in issues)
{
if (_options.Verbose)
{
Console.WriteLine($"Checking issue: {issue.HtmlUrl}");
}
if (String.IsNullOrWhiteSpace(issue.Body))
{
continue;
}
// For each issue, extract the regressions and update their status (recovered).
// If all regressions are recovered, close the issue.
var existingRegressions = ExtractRegressionsBlock(issue.Body)?.ToDictionary(x => x.Identifier, x => x);
if (existingRegressions == null)
{
continue;
}
// Find all regressions that are reported in this issue, and check if they have recovered
var issueNeedsUpdate = false;
// Update local regressions that have recovered
foreach (var r in regressions)
{
if (existingRegressions.TryGetValue(r.Identifier, out var localRegression))
{
// If the issue has been reported, exclude it
if (regressionsToReport.Remove(r.Identifier))
{
Console.WriteLine($"Issue already reported {r.CurrentResult.Description} at {r.CurrentResult.DateTimeUtc}");
}
if (!localRegression.HasRecovered && r.HasRecovered)
{
Console.WriteLine($"Found update for {r.Identifier}");
existingRegressions.Remove(r.Identifier);
existingRegressions.Add(r.Identifier, r);
issueNeedsUpdate = true;
}
}
}
if (issueNeedsUpdate)
{
Console.WriteLine("Updating issue...");
var update = issue.ToUpdate();
update.Body = await CreateIssueBody(existingRegressions.Values, template);
// If all regressions have recovered, close it
if (existingRegressions.Values.All(x => x.HasRecovered))
{
Console.WriteLine("All regressions have recovered, closing the issue");
update.State = ItemState.Closed;
}
if (!_options.ReadOnly)
{
await GitHubHelper.GetClient().Issue.Update(_options.RepositoryId, issue.Number, update);
}
}
else
{
Console.WriteLine("Issue doesn't need to be updated");
}
}
return(regressionsToReport.Values);
}
/// <summary>
/// This method finds regressions for a give source.
/// Steps:
/// - Query the table for the latest rows specified in the source
/// - Group records by Scenario + Description (descriptor)
/// - For each unique descriptor
/// - Find matching rules from the source
/// - Evaluate the source's probes for each record
/// - Calculates the std deviation
/// - Look for 2 consecutive deviations
/// </summary>
private static async IAsyncEnumerable <Regression> FindRegression(Source source)
{
if (source.Regressions == null)
{
yield break;
}
var loadStartDateTimeUtc = DateTime.UtcNow.AddDays(0 - source.DaysToLoad);
var detectionMaxDateTimeUtc = DateTime.UtcNow.AddDays(0 - source.DaysToSkip);
var allResults = new List <BenchmarksResult>();
// Load latest records
Console.Write("Loading records... ");
using (var connection = new SqlConnection(_options.ConnectionString))
{
using (var command = new SqlCommand(String.Format(Queries.Latest, source.Table), connection))
{
command.Parameters.AddWithValue("@startDate", loadStartDateTimeUtc);
await connection.OpenAsync();
var reader = await command.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
allResults.Add(new BenchmarksResult
{
Id = Convert.ToInt32(reader["Id"]),
Excluded = Convert.ToBoolean(reader["Excluded"]),
DateTimeUtc = (DateTimeOffset)reader["DateTimeUtc"],
Session = Convert.ToString(reader["Session"]),
Scenario = Convert.ToString(reader["Scenario"]),
Description = Convert.ToString(reader["Description"]),
Document = Convert.ToString(reader["Document"]),
});
}
}
}
Console.WriteLine($"{allResults.Count} found");
// Reorder results chronologically
allResults.Reverse();
// Compute standard deviation
var resultsByScenario = allResults
.GroupBy(x => x.Scenario + ":" + x.Description)
.ToDictionary(x => x.Key, x => x.ToArray())
;
foreach (var descriptor in resultsByScenario.Keys)
{
// Does the descriptor match a rule?
if (!source.Include(descriptor))
{
continue;
}
var rules = source.Match(descriptor);
// Should regressions be ignored for this descriptor?
var lastIgnoreRegressionRule = rules.LastOrDefault(x => x.IgnoreRegressions != null);
if (lastIgnoreRegressionRule != null && lastIgnoreRegressionRule.IgnoreRegressions.Value)
{
if (_options.Verbose)
{
Console.WriteLine("Regressions ignored");
}
continue;
}
// Resolve path for the metric
var results = resultsByScenario[descriptor];
foreach (var probe in source.Regressions.Probes)
{
if (_options.Verbose)
{
Console.WriteLine();
Console.WriteLine($"Evaluating probe {descriptor} {probe.Path} with {results.Count()} results");
Console.WriteLine("=============================================================================================");
Console.WriteLine();
}
var resultSet = results
.Select(x => new { Result = x, Token = x.Data.SelectTokens(probe.Path).FirstOrDefault() })
.Where(x => x.Token != null)
.Select(x => new { Result = x.Result, Value = Convert.ToDouble(x.Token) })
.ToArray();
// Find regressions
// Can't find a regression if there are less than 5 value
if (resultSet.Length < 5)
{
if (_options.Verbose)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Not enough data ({resultSet.Length})");
Console.ResetColor();
}
continue;
}
if (_options.Verbose)
{
Console.WriteLine($"Values: {JsonConvert.SerializeObject(resultSet.Select(x => x.Value).ToArray())}");
}
var values = resultSet.Select(x => x.Value).ToArray();
// Look for 2 consecutive values that are outside of the threshold,
// subsequent to 3 consecutive values that are inside the threshold.
// 5 is the number of data points necessary to detect a threshold
for (var i = 0; i < resultSet.Length - 5; i++)
{
// Skip the measurement if it's too recent
if (resultSet[i].Result.DateTimeUtc >= detectionMaxDateTimeUtc)
{
continue;
}
if (_options.Verbose)
{
Console.WriteLine($"Checking {resultSet[i + 3].Value} at {resultSet[i + 3].Result.DateTimeUtc} with values {JsonConvert.SerializeObject(values.Skip(i).Take(5).ToArray())}");
}
// Measure stdev by picking the StdevCount results before the currently checked one
var stdevs = values.Take(i + 1).TakeLast(source.StdevCount).ToArray();
if (stdevs.Length < source.StdevCount && probe.Unit == ThresholdUnits.StDev)
{
Console.WriteLine($"Not enough values to build a standard deviation: {JsonConvert.SerializeObject(stdevs)}");
continue;
}
// Calculate the stdev from all values up to the verified window
double average = stdevs.Average();
double sumOfSquaresOfDifferences = stdevs.Sum(val => (val - average) * (val - average));
double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / stdevs.Length);
if (_options.Verbose)
{
Console.WriteLine($"Building stdev ({standardDeviation}) from last {source.StdevCount} values {JsonConvert.SerializeObject(stdevs)}");
}
/* checked value (included in stdev)
* ^ ______/i+3---------i+4---------
* (stdev results) ----i---------i+1---------i+2/
*
* <- value1 ->
* <------- value2 ------->
* <--------------- value3 --------->
* <------------------------ value4 ------------->
*/
if (standardDeviation == 0)
{
// We skip measurement with stdev of zero since it could induce divisions by zero, and any change will trigger
// a regression
Console.WriteLine($"Ignoring measurement with stdev = 0");
continue;
}
var value1 = values[i + 1] - values[i];
var value2 = values[i + 2] - values[i];
var value3 = values[i + 3] - values[i];
var value4 = values[i + 4] - values[i];
if (_options.Verbose)
{
Console.WriteLine($"Next values: {values[i + 0]} {values[i + 1]} {values[i + 2]} {values[i + 3]} {values[i + 4]}");
Console.WriteLine($"Deviations: {value1:n0} {value2:n0} {value3:n0} {value4:n0} Allowed deviation: {standardDeviation * probe.Threshold:n0}");
}
var hasRegressed = false;
switch (probe.Unit)
{
case ThresholdUnits.StDev:
// factor of standard deviation
hasRegressed = Math.Abs(value1) < probe.Threshold * standardDeviation &&
Math.Abs(value2) < probe.Threshold * standardDeviation &&
Math.Abs(value3) >= probe.Threshold * standardDeviation &&
Math.Abs(value4) >= probe.Threshold * standardDeviation &&
Math.Sign(value3) == Math.Sign(value4);
break;
case ThresholdUnits.Percent:
// percentage of the average of values
hasRegressed = Math.Abs(value1) < average * (probe.Threshold / 100) &&
Math.Abs(value2) < average * (probe.Threshold / 100) &&
Math.Abs(value3) >= average * (probe.Threshold / 100) &&
Math.Abs(value4) >= average * (probe.Threshold / 100) &&
Math.Sign(value3) == Math.Sign(value4);
break;
case ThresholdUnits.Absolute:
// absolute deviation
hasRegressed = Math.Abs(value1) < probe.Threshold &&
Math.Abs(value2) < probe.Threshold &&
Math.Abs(value3) >= probe.Threshold &&
Math.Abs(value4) >= probe.Threshold &&
Math.Sign(value3) == Math.Sign(value4);
break;
default:
break;
}
if (hasRegressed)
{
var regression = new Regression
{
PreviousResult = resultSet[i + 2].Result,
CurrentResult = resultSet[i + 3].Result,
Change = value3,
StandardDeviation = standardDeviation,
Average = average
};
if (_options.Verbose)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"Regression detected: {values[i + 2]:n0} to {values[i + 3]:n0} for {regression.Identifier}");
Console.ResetColor();
}
foreach (var rule in rules)
{
foreach (var label in rule.Labels)
{
regression.Labels.Add(label);
}
foreach (var owner in rule.Owners)
{
regression.Owners.Add(owner);
}
}
foreach (var label in source.Regressions.Labels)
{
regression.Labels.Add(label);
}
foreach (var owner in source.Regressions.Owners)
{
regression.Owners.Add(owner);
}
// If there are subsequent measurements, detect if the benchmark has
// recovered
// - if the delta is inside the threshold limits
// - the delta is outside the threshold limits but in the opposite sign
for (var j = i + 5; j < resultSet.Length; j++)
{
var nextValue = values[j] - values[i];
var hasRecovered = false;
// It has recovered if the difference between the first measurement and the current one
// are within the threashold boundaries, or if the value is better (opposite sign).
switch (probe.Unit)
{
case ThresholdUnits.StDev:
// factor of standard deviation
hasRecovered = Math.Abs(nextValue) < probe.Threshold * standardDeviation || Math.Sign(nextValue) != Math.Sign(value4);
break;
case ThresholdUnits.Percent:
// percentage of the average of values
hasRecovered = Math.Abs(nextValue) < average * (probe.Threshold / 100) || Math.Sign(nextValue) != Math.Sign(value4);
;
break;
case ThresholdUnits.Absolute:
// absolute deviation
hasRecovered = Math.Abs(nextValue) < probe.Threshold || Math.Sign(nextValue) != Math.Sign(value4);
break;
default:
break;
}
if (hasRecovered)
{
regression.RecoveredResult = resultSet[j].Result;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"Recovered on {regression.RecoveredResult.DateTimeUtc}");
Console.ResetColor();
break;
}
}
regression.ComputeChanges();
yield return(regression);
}
}
}
}
}