/// <summary> /// Updates and closes existing issues based on regressions found. /// </summary> /// <returns> /// The remaining issues that haven't been reported yet. /// </returns> private static async Task <IEnumerable <Regression> > UpdateIssues(IEnumerable <Regression> regressions, Source source, string template) { if (!regressions.Any()) { return(regressions); } if (_options.Debug) { return(regressions); } var issues = await GetRecentIssues(source); Console.WriteLine($"Downloaded {issues.Count()} issues"); // The list of regressions that remain to be reported var regressionsToReport = new List <Regression>(regressions).ToDictionary(x => x.Identifier, x => x); foreach (var issue in issues) { if (_options.Verbose) { Console.WriteLine($"Checking issue: {issue.HtmlUrl}"); } if (String.IsNullOrWhiteSpace(issue.Body)) { continue; } // For each issue, extract the regressions and update their status (recovered). // If all regressions are recovered, close the issue. var existingRegressions = ExtractRegressionsBlock(issue.Body)?.ToDictionary(x => x.Identifier, x => x); if (existingRegressions == null) { continue; } // Find all regressions that are reported in this issue, and check if they have recovered var issueNeedsUpdate = false; // Update local regressions that have recovered foreach (var r in regressions) { if (existingRegressions.TryGetValue(r.Identifier, out var localRegression)) { // If the issue has been reported, exclude it if (regressionsToReport.Remove(r.Identifier)) { Console.WriteLine($"Issue already reported {r.CurrentResult.Description} at {r.CurrentResult.DateTimeUtc}"); } if (!localRegression.HasRecovered && r.HasRecovered) { Console.WriteLine($"Found update for {r.Identifier}"); existingRegressions.Remove(r.Identifier); existingRegressions.Add(r.Identifier, r); issueNeedsUpdate = true; } } } if (issueNeedsUpdate) { Console.WriteLine("Updating issue..."); var update = issue.ToUpdate(); update.Body = await CreateIssueBody(existingRegressions.Values, template); // If all regressions have recovered, close it if (existingRegressions.Values.All(x => x.HasRecovered)) { Console.WriteLine("All regressions have recovered, closing the issue"); update.State = ItemState.Closed; } if (!_options.ReadOnly) { await GitHubHelper.GetClient().Issue.Update(_options.RepositoryId, issue.Number, update); } } else { Console.WriteLine("Issue doesn't need to be updated"); } } return(regressionsToReport.Values); }
/// <summary> /// This method finds regressions for a give source. /// Steps: /// - Query the table for the latest rows specified in the source /// - Group records by Scenario + Description (descriptor) /// - For each unique descriptor /// - Find matching rules from the source /// - Evaluate the source's probes for each record /// - Calculates the std deviation /// - Look for 2 consecutive deviations /// </summary> private static async IAsyncEnumerable <Regression> FindRegression(Source source) { if (source.Regressions == null) { yield break; } var loadStartDateTimeUtc = DateTime.UtcNow.AddDays(0 - source.DaysToLoad); var detectionMaxDateTimeUtc = DateTime.UtcNow.AddDays(0 - source.DaysToSkip); var allResults = new List <BenchmarksResult>(); // Load latest records Console.Write("Loading records... "); using (var connection = new SqlConnection(_options.ConnectionString)) { using (var command = new SqlCommand(String.Format(Queries.Latest, source.Table), connection)) { command.Parameters.AddWithValue("@startDate", loadStartDateTimeUtc); await connection.OpenAsync(); var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { allResults.Add(new BenchmarksResult { Id = Convert.ToInt32(reader["Id"]), Excluded = Convert.ToBoolean(reader["Excluded"]), DateTimeUtc = (DateTimeOffset)reader["DateTimeUtc"], Session = Convert.ToString(reader["Session"]), Scenario = Convert.ToString(reader["Scenario"]), Description = Convert.ToString(reader["Description"]), Document = Convert.ToString(reader["Document"]), }); } } } Console.WriteLine($"{allResults.Count} found"); // Reorder results chronologically allResults.Reverse(); // Compute standard deviation var resultsByScenario = allResults .GroupBy(x => x.Scenario + ":" + x.Description) .ToDictionary(x => x.Key, x => x.ToArray()) ; foreach (var descriptor in resultsByScenario.Keys) { // Does the descriptor match a rule? if (!source.Include(descriptor)) { continue; } var rules = source.Match(descriptor); // Should regressions be ignored for this descriptor? var lastIgnoreRegressionRule = rules.LastOrDefault(x => x.IgnoreRegressions != null); if (lastIgnoreRegressionRule != null && lastIgnoreRegressionRule.IgnoreRegressions.Value) { if (_options.Verbose) { Console.WriteLine("Regressions ignored"); } continue; } // Resolve path for the metric var results = resultsByScenario[descriptor]; foreach (var probe in source.Regressions.Probes) { if (_options.Verbose) { Console.WriteLine(); Console.WriteLine($"Evaluating probe {descriptor} {probe.Path} with {results.Count()} results"); Console.WriteLine("============================================================================================="); Console.WriteLine(); } var resultSet = results .Select(x => new { Result = x, Token = x.Data.SelectTokens(probe.Path).FirstOrDefault() }) .Where(x => x.Token != null) .Select(x => new { Result = x.Result, Value = Convert.ToDouble(x.Token) }) .ToArray(); // Find regressions // Can't find a regression if there are less than 5 value if (resultSet.Length < 5) { if (_options.Verbose) { Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine($"Not enough data ({resultSet.Length})"); Console.ResetColor(); } continue; } if (_options.Verbose) { Console.WriteLine($"Values: {JsonConvert.SerializeObject(resultSet.Select(x => x.Value).ToArray())}"); } var values = resultSet.Select(x => x.Value).ToArray(); // Look for 2 consecutive values that are outside of the threshold, // subsequent to 3 consecutive values that are inside the threshold. // 5 is the number of data points necessary to detect a threshold for (var i = 0; i < resultSet.Length - 5; i++) { // Skip the measurement if it's too recent if (resultSet[i].Result.DateTimeUtc >= detectionMaxDateTimeUtc) { continue; } if (_options.Verbose) { Console.WriteLine($"Checking {resultSet[i + 3].Value} at {resultSet[i + 3].Result.DateTimeUtc} with values {JsonConvert.SerializeObject(values.Skip(i).Take(5).ToArray())}"); } // Measure stdev by picking the StdevCount results before the currently checked one var stdevs = values.Take(i + 1).TakeLast(source.StdevCount).ToArray(); if (stdevs.Length < source.StdevCount && probe.Unit == ThresholdUnits.StDev) { Console.WriteLine($"Not enough values to build a standard deviation: {JsonConvert.SerializeObject(stdevs)}"); continue; } // Calculate the stdev from all values up to the verified window double average = stdevs.Average(); double sumOfSquaresOfDifferences = stdevs.Sum(val => (val - average) * (val - average)); double standardDeviation = Math.Sqrt(sumOfSquaresOfDifferences / stdevs.Length); if (_options.Verbose) { Console.WriteLine($"Building stdev ({standardDeviation}) from last {source.StdevCount} values {JsonConvert.SerializeObject(stdevs)}"); } /* checked value (included in stdev) * ^ ______/i+3---------i+4--------- * (stdev results) ----i---------i+1---------i+2/ * * <- value1 -> * <------- value2 -------> * <--------------- value3 ---------> * <------------------------ value4 -------------> */ if (standardDeviation == 0) { // We skip measurement with stdev of zero since it could induce divisions by zero, and any change will trigger // a regression Console.WriteLine($"Ignoring measurement with stdev = 0"); continue; } var value1 = values[i + 1] - values[i]; var value2 = values[i + 2] - values[i]; var value3 = values[i + 3] - values[i]; var value4 = values[i + 4] - values[i]; if (_options.Verbose) { Console.WriteLine($"Next values: {values[i + 0]} {values[i + 1]} {values[i + 2]} {values[i + 3]} {values[i + 4]}"); Console.WriteLine($"Deviations: {value1:n0} {value2:n0} {value3:n0} {value4:n0} Allowed deviation: {standardDeviation * probe.Threshold:n0}"); } var hasRegressed = false; switch (probe.Unit) { case ThresholdUnits.StDev: // factor of standard deviation hasRegressed = Math.Abs(value1) < probe.Threshold * standardDeviation && Math.Abs(value2) < probe.Threshold * standardDeviation && Math.Abs(value3) >= probe.Threshold * standardDeviation && Math.Abs(value4) >= probe.Threshold * standardDeviation && Math.Sign(value3) == Math.Sign(value4); break; case ThresholdUnits.Percent: // percentage of the average of values hasRegressed = Math.Abs(value1) < average * (probe.Threshold / 100) && Math.Abs(value2) < average * (probe.Threshold / 100) && Math.Abs(value3) >= average * (probe.Threshold / 100) && Math.Abs(value4) >= average * (probe.Threshold / 100) && Math.Sign(value3) == Math.Sign(value4); break; case ThresholdUnits.Absolute: // absolute deviation hasRegressed = Math.Abs(value1) < probe.Threshold && Math.Abs(value2) < probe.Threshold && Math.Abs(value3) >= probe.Threshold && Math.Abs(value4) >= probe.Threshold && Math.Sign(value3) == Math.Sign(value4); break; default: break; } if (hasRegressed) { var regression = new Regression { PreviousResult = resultSet[i + 2].Result, CurrentResult = resultSet[i + 3].Result, Change = value3, StandardDeviation = standardDeviation, Average = average }; if (_options.Verbose) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine($"Regression detected: {values[i + 2]:n0} to {values[i + 3]:n0} for {regression.Identifier}"); Console.ResetColor(); } foreach (var rule in rules) { foreach (var label in rule.Labels) { regression.Labels.Add(label); } foreach (var owner in rule.Owners) { regression.Owners.Add(owner); } } foreach (var label in source.Regressions.Labels) { regression.Labels.Add(label); } foreach (var owner in source.Regressions.Owners) { regression.Owners.Add(owner); } // If there are subsequent measurements, detect if the benchmark has // recovered // - if the delta is inside the threshold limits // - the delta is outside the threshold limits but in the opposite sign for (var j = i + 5; j < resultSet.Length; j++) { var nextValue = values[j] - values[i]; var hasRecovered = false; // It has recovered if the difference between the first measurement and the current one // are within the threashold boundaries, or if the value is better (opposite sign). switch (probe.Unit) { case ThresholdUnits.StDev: // factor of standard deviation hasRecovered = Math.Abs(nextValue) < probe.Threshold * standardDeviation || Math.Sign(nextValue) != Math.Sign(value4); break; case ThresholdUnits.Percent: // percentage of the average of values hasRecovered = Math.Abs(nextValue) < average * (probe.Threshold / 100) || Math.Sign(nextValue) != Math.Sign(value4); ; break; case ThresholdUnits.Absolute: // absolute deviation hasRecovered = Math.Abs(nextValue) < probe.Threshold || Math.Sign(nextValue) != Math.Sign(value4); break; default: break; } if (hasRecovered) { regression.RecoveredResult = resultSet[j].Result; Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine($"Recovered on {regression.RecoveredResult.DateTimeUtc}"); Console.ResetColor(); break; } } regression.ComputeChanges(); yield return(regression); } } } } }